ATLAS-1369 : Optimize Gremlin queries generated by DSL translator

common/src/main/java/org/apache/atlas/groovy/AbstractFunctionExpression.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.groovy;
+
+/**
+ * Base class for all expression that can have a caller.
+ */
+public abstract class AbstractFunctionExpression extends AbstractGroovyExpression {
+
+    // null for global functions
+    private GroovyExpression caller;
+    private TraversalStepType type = TraversalStepType.NONE;
+
+    public AbstractFunctionExpression(GroovyExpression target) {
+        this.caller = target;
+    }
+
+    public AbstractFunctionExpression(TraversalStepType type, GroovyExpression target) {
+        this.caller = target;
+        this.type = type;
+    }
+
+    public  GroovyExpression getCaller() {
+        return caller;
+    }
+
+    public void setCaller(GroovyExpression expr) {
+        caller = expr;
+    }
+
+
+    public void setType(TraversalStepType type) {
+        this.type = type;
+    }
+
+    @Override
+    public TraversalStepType getType() {
+        return type;
+    }
+
+
+}

common/src/main/java/org/apache/atlas/groovy/AbstractGroovyExpression.java

@@ -33,4 +33,13 @@ public abstract class AbstractGroovyExpression implements GroovyExpression {
         return ctx.getQuery();
     }
 
+    @Override
+    public TraversalStepType getType() {
+        return TraversalStepType.NONE;
+    }
+
+    @Override
+    public GroovyExpression copy() {
+        return copy(getChildren());
+    }
 }

common/src/main/java/org/apache/atlas/groovy/ArithmeticExpression.java

@@ -17,6 +17,8 @@
  */
 package org.apache.atlas.groovy;
 
+import java.util.List;
+
 import org.apache.atlas.AtlasException;
 
 /**
@@ -56,4 +58,14 @@ public class ArithmeticExpression extends BinaryExpression {
     public ArithmeticExpression(GroovyExpression left, ArithmeticOperator op, GroovyExpression right) {
         super(left, op.getGroovyValue(), right);
     }
+
+    private ArithmeticExpression(GroovyExpression left, String op, GroovyExpression right) {
+        super(left, op, right);
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 2;
+        return new ArithmeticExpression(newChildren.get(0), op, newChildren.get(1));
+    }
 }

common/src/main/java/org/apache/atlas/groovy/BinaryExpression.java

@@ -18,6 +18,9 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * Represents any kind of binary expression.  This could
  * be an arithmetic expression, such as a + 3, a boolean
@@ -30,7 +33,7 @@ public abstract class BinaryExpression extends AbstractGroovyExpression {
 
     private GroovyExpression left;
     private GroovyExpression right;
-    private String op;
+    protected String op;
 
     public BinaryExpression(GroovyExpression left, String op, GroovyExpression right) {
         this.left = left;
@@ -48,4 +51,8 @@ public abstract class BinaryExpression extends AbstractGroovyExpression {
         right.generateGroovy(context);
     }
 
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Arrays.asList(left, right);
+    }
 }

common/src/main/java/org/apache/atlas/groovy/CastExpression.java

@@ -18,6 +18,9 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.Collections;
+import java.util.List;
+
 /**
  * Groovy expression that represents a cast.
  */
@@ -28,7 +31,7 @@ public class CastExpression extends AbstractGroovyExpression {
 
     public CastExpression(GroovyExpression expr, String className) {
         this.expr = expr;
-        this.className  =className;
+        this.className  = className;
     }
 
     @Override
@@ -41,4 +44,13 @@ public class CastExpression extends AbstractGroovyExpression {
         context.append(")");
     }
 
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.singletonList(expr);
+    }
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 1;
+        return new CastExpression(newChildren.get(0), className);
+    }
 }

common/src/main/java/org/apache/atlas/groovy/ClosureExpression.java

@@ -20,6 +20,7 @@ package org.apache.atlas.groovy;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -28,28 +29,79 @@ import java.util.List;
  */
 public class ClosureExpression extends AbstractGroovyExpression {
 
-    private List<String> varNames = new ArrayList<>();
-    private GroovyExpression body;
+    /**
+     * Variable declaration in a closure.
+     */
+    public static class VariableDeclaration {
+        private String type;
+        private String varName;
 
-    public ClosureExpression(GroovyExpression body, String... varNames) {
-        this.body = body;
-        this.varNames.addAll(Arrays.asList(varNames));
+        public VariableDeclaration(String type, String varName) {
+            super();
+            this.type = type;
+            this.varName = varName;
+        }
+
+        public VariableDeclaration(String varName) {
+            this.varName = varName;
+        }
+
+        public void append(GroovyGenerationContext context) {
+            if (type != null) {
+                context.append(type);
+                context.append(" ");
+            }
+            context.append(varName);
+        }
+    }
+    private List<VariableDeclaration> vars = new ArrayList<>();
+    private StatementListExpression body = new StatementListExpression();
+
+    public ClosureExpression(String... varNames) {
+        this(null, varNames);
+    }
+
+    public ClosureExpression(GroovyExpression initialStmt, String... varNames) {
+        this(Arrays.asList(varNames), initialStmt);
+    }
+
+    public ClosureExpression(List<String> varNames, GroovyExpression initialStmt) {
+        if (initialStmt != null) {
+            this.body.addStatement(initialStmt);
+        }
+        for (String varName : varNames) {
+            vars.add(new VariableDeclaration(varName));
+        }
     }
 
-    public ClosureExpression(List<String> varNames, GroovyExpression body) {
-        this.body = body;
-        this.varNames.addAll(varNames);
+    public ClosureExpression(GroovyExpression initialStmt, List<VariableDeclaration> varNames) {
+        if (initialStmt != null) {
+            this.body.addStatement(initialStmt);
+        }
+        vars.addAll(varNames);
+    }
+
+    public void addStatement(GroovyExpression expr) {
+        body.addStatement(expr);
+    }
+
+    public void addStatements(List<GroovyExpression> exprs) {
+        body.addStatements(exprs);
+    }
+
+    public void replaceStatement(int index, GroovyExpression newExpr) {
+        body.replaceStatement(index, newExpr);
     }
 
     @Override
     public void generateGroovy(GroovyGenerationContext context) {
 
         context.append("{");
-        if (!varNames.isEmpty()) {
-            Iterator<String> varIt = varNames.iterator();
+        if (!vars.isEmpty()) {
+            Iterator<VariableDeclaration> varIt = vars.iterator();
             while(varIt.hasNext()) {
-                String varName = varIt.next();
-                context.append(varName);
+                VariableDeclaration var = varIt.next();
+                var.append(context);
                 if (varIt.hasNext()) {
                     context.append(", ");
                 }
@@ -58,6 +110,20 @@ public class ClosureExpression extends AbstractGroovyExpression {
         }
         body.generateGroovy(context);
         context.append("}");
+    }
+
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.<GroovyExpression>singletonList(body);
+    }
 
+    public List<GroovyExpression> getStatements() {
+        return body.getStatements();
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 1;
+        return new ClosureExpression(newChildren.get(0), vars);
     }
 }

common/src/main/java/org/apache/atlas/groovy/ComparisonExpression.java

@@ -17,6 +17,8 @@
  */
 package org.apache.atlas.groovy;
 
+import java.util.List;
+
 import org.apache.atlas.AtlasException;
 
 /**
@@ -61,4 +63,14 @@ public class ComparisonExpression extends BinaryExpression {
     public ComparisonExpression(GroovyExpression left, ComparisonOperator op, GroovyExpression right) {
         super(left, op.getGroovyValue(), right);
     }
+
+    private ComparisonExpression(GroovyExpression left, String op, GroovyExpression right) {
+        super(left, op, right);
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 2;
+        return new ComparisonExpression(newChildren.get(0), op, newChildren.get(1));
+    }
 }

common/src/main/java/org/apache/atlas/groovy/ComparisonOperatorExpression.java

@@ -17,6 +17,8 @@
  */
 package org.apache.atlas.groovy;
 
+import java.util.List;
+
 /**
  * Represents an expression that compares two expressions using
  * the Groovy "spaceship" operator.  This is basically the
@@ -29,4 +31,10 @@ public class ComparisonOperatorExpression extends BinaryExpression {
     public ComparisonOperatorExpression(GroovyExpression left, GroovyExpression right) {
         super(left, "<=>", right);
     }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 2;
+        return new ComparisonOperatorExpression(newChildren.get(0), newChildren.get(1));
+    }
 }

common/src/main/java/org/apache/atlas/groovy/FieldExpression.java

@@ -18,27 +18,38 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.Collections;
+import java.util.List;
+
 /**
  * Groovy expression that accesses a field in an object.
  */
-public class FieldExpression extends AbstractGroovyExpression {
+public class FieldExpression extends AbstractFunctionExpression {
 
-    private GroovyExpression target;
     private String fieldName;
 
     public FieldExpression(GroovyExpression target, String fieldName) {
-        this.target = target;
+        super(target);
         this.fieldName = fieldName;
     }
 
     @Override
     public void generateGroovy(GroovyGenerationContext context) {
-
-        target.generateGroovy(context);
+        getCaller().generateGroovy(context);
         context.append(".'");
 
         context.append(fieldName);
         context.append("'");
     }
 
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.singletonList(getCaller());
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 1;
+        return new FieldExpression(newChildren.get(0), fieldName);
+    }
 }

common/src/main/java/org/apache/atlas/groovy/FunctionCallExpression.java

@@ -20,41 +20,52 @@ package org.apache.atlas.groovy;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
 /**
  * Groovy expression that calls a method on an object.
  */
-public class FunctionCallExpression extends AbstractGroovyExpression {
-
-    // null for global functions
-    private GroovyExpression target;
+public class FunctionCallExpression extends AbstractFunctionExpression {
 
     private String functionName;
     private List<GroovyExpression> arguments = new ArrayList<>();
 
-    public FunctionCallExpression(String functionName, List<? extends GroovyExpression> arguments) {
-        this.target = null;
+    public FunctionCallExpression(TraversalStepType type, String functionName, GroovyExpression... arguments) {
+        super(type, null);
         this.functionName = functionName;
-        this.arguments.addAll(arguments);
+        this.arguments.addAll(Arrays.asList(arguments));
     }
 
-    public FunctionCallExpression(GroovyExpression target, String functionName,
-            List<? extends GroovyExpression> arguments) {
-        this.target = target;
+    public FunctionCallExpression(String functionName, GroovyExpression... arguments) {
+        super(null);
+        this.functionName = functionName;
+        this.arguments.addAll(Arrays.asList(arguments));
+    }
+
+    public FunctionCallExpression(TraversalStepType type, String functionName, List<GroovyExpression> arguments) {
+        super(type, null);
         this.functionName = functionName;
         this.arguments.addAll(arguments);
     }
 
-    public FunctionCallExpression(String functionName, GroovyExpression... arguments) {
-        this.target = null;
+    public FunctionCallExpression(GroovyExpression target, String functionName, GroovyExpression... arguments) {
+        super(target);
         this.functionName = functionName;
         this.arguments.addAll(Arrays.asList(arguments));
     }
 
-    public FunctionCallExpression(GroovyExpression target, String functionName, GroovyExpression... arguments) {
-        this.target = target;
+    public FunctionCallExpression(TraversalStepType type, GroovyExpression target, String functionName,
+                                  List<? extends GroovyExpression> arguments) {
+        super(type, target);
+        this.functionName = functionName;
+        this.arguments.addAll(arguments);
+    }
+
+    public FunctionCallExpression(TraversalStepType type, GroovyExpression target, String functionName,
+                                  GroovyExpression... arguments) {
+        super(type, target);
         this.functionName = functionName;
         this.arguments.addAll(Arrays.asList(arguments));
     }
@@ -63,11 +74,20 @@ public class FunctionCallExpression extends AbstractGroovyExpression {
         arguments.add(expr);
     }
 
+    public List<GroovyExpression> getArguments() {
+        return Collections.unmodifiableList(arguments);
+    }
+
+
+    public String getFunctionName() {
+        return functionName;
+    }
+
     @Override
     public void generateGroovy(GroovyGenerationContext context) {
 
-        if (target != null) {
-            target.generateGroovy(context);
+        if (getCaller() != null) {
+            getCaller().generateGroovy(context);
             context.append(".");
         }
         context.append(functionName);
@@ -77,10 +97,44 @@ public class FunctionCallExpression extends AbstractGroovyExpression {
             GroovyExpression expr = it.next();
             expr.generateGroovy(context);
             if (it.hasNext()) {
-                context.append(", ");
+                context.append(",");
             }
         }
         context.append(")");
     }
 
+    @Override
+    public List<GroovyExpression> getChildren() {
+        List<GroovyExpression> result = new ArrayList<>(arguments.size() + 1);
+        if (getCaller() != null) {
+            result.add(getCaller());
+        }
+        result.addAll(arguments);
+        return result;
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+
+        if (getCaller() == null) {
+            return new FunctionCallExpression(getType(), functionName, newChildren);
+        }
+
+        GroovyExpression newTarget = newChildren.get(0);
+        List<GroovyExpression> args = null;
+        if (newChildren.size() > 1) {
+            args = newChildren.subList(1, newChildren.size());
+        } else {
+            args = Collections.emptyList();
+        }
+        return new FunctionCallExpression(getType(), newTarget, functionName, args);
+
+    }
+
+    public void setArgument(int index, GroovyExpression value) {
+        if (index < 0 || index >= arguments.size()) {
+            throw new IllegalArgumentException("Invalid argIndex " + index);
+        }
+        arguments.set(index, value);
+    }
 }

common/src/main/java/org/apache/atlas/groovy/GroovyExpression.java

@@ -18,17 +18,55 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.List;
+
 /**
  * Represents an expression in the Groovy programming language, which
  * is the language that Gremlin scripts are written and interpreted in.
  */
 public interface GroovyExpression  {
-
     /**
-     * Generates a groovy script from the expression.
+     * Generates a Groovy script from the expression.
      *
      * @param context
      */
     void generateGroovy(GroovyGenerationContext context);
 
+    /**
+     * Gets all of the child expressions of this expression.
+     * s
+     * @return
+     */
+    List<GroovyExpression> getChildren();
+
+    /**
+     * Makes a copy of the expression, keeping everything the
+     * same except its child expressions.  These are replaced
+     * with the provided children.  The order of the children
+     * is important.  It is expected that the children provided
+     * here are updated versions of the children returned by
+     * getChildren().  The order of the children must be the
+     * same as the order in which the children were returned
+     * by getChildren()
+     *
+     * @param newChildren
+     * @return
+     */
+    GroovyExpression copy(List<GroovyExpression> newChildren);
+
+    /**
+     * Makes a shallow copy of the GroovyExpression.  This
+     * is equivalent to copy(getChildren());
+     *
+     * @return
+     */
+    GroovyExpression copy();
+
+    /**
+     * Gets the type of traversal step represented by this
+     * expression (or TraversalStepType.NONE if it is not part of a graph traversal).
+     *
+     * @return
+     */
+    TraversalStepType getType();
 }

common/src/main/java/org/apache/atlas/groovy/IdentifierExpression.java

@@ -18,18 +18,28 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.Collections;
+import java.util.List;
+
 /**
  * Groovy expression that references the variable with the given name.
  *
  */
 public class IdentifierExpression extends AbstractGroovyExpression {
 
+    private TraversalStepType type = TraversalStepType.NONE;
     private String varName;
 
     public IdentifierExpression(String varName) {
         this.varName = varName;
     }
 
+    public IdentifierExpression(TraversalStepType type, String varName) {
+        this.varName = varName;
+        this.type = type;
+    }
+
+
     public String getVariableName() {
         return varName;
     }
@@ -39,4 +49,25 @@ public class IdentifierExpression extends AbstractGroovyExpression {
         context.append(varName);
     }
 
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.isEmpty();
+        IdentifierExpression result =  new IdentifierExpression(varName);
+        result.setType(type);
+        return result;
+    }
+
+    public void setType(TraversalStepType type) {
+        this.type = type;
+    }
+
+    @Override
+    public TraversalStepType getType() {
+        return type;
+    }
 }

common/src/main/java/org/apache/atlas/groovy/LabeledExpression.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.groovy;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents a Groovy expression that has a label.
+ */
+public class LabeledExpression extends AbstractGroovyExpression {
+
+    private String label;
+    private GroovyExpression expr;
+
+    public LabeledExpression(String label, GroovyExpression expr) {
+        this.label = label;
+        this.expr = expr;
+    }
+
+    @Override
+    public void generateGroovy(GroovyGenerationContext context) {
+        context.append(label);
+        context.append(":");
+        expr.generateGroovy(context);
+    }
+
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.singletonList(expr);
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 1;
+        return new LabeledExpression(label, newChildren.get(0));
+    }
+}

common/src/main/java/org/apache/atlas/groovy/ListExpression.java

@@ -20,6 +20,7 @@ package org.apache.atlas.groovy;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -57,4 +58,15 @@ public class ListExpression extends AbstractGroovyExpression {
         context.append("]");
     }
 
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.unmodifiableList(values);
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        return new ListExpression(newChildren);
+    }
+
+
 }

common/src/main/java/org/apache/atlas/groovy/LiteralExpression.java

@@ -18,13 +18,15 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.Collections;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
  * Represents a literal value.
  */
-public class LiteralExpression implements GroovyExpression {
+public class LiteralExpression extends AbstractGroovyExpression {
 
     public static final LiteralExpression TRUE = new LiteralExpression(true);
     public static final LiteralExpression FALSE = new LiteralExpression(false);
@@ -40,6 +42,12 @@ public class LiteralExpression implements GroovyExpression {
         this.addTypeSuffix = addTypeSuffix;
     }
 
+    public LiteralExpression(Object value, boolean addTypeSuffix, boolean translateToParameter) {
+        this.value = value;
+        this.translateToParameter = translateToParameter;
+        this.addTypeSuffix = addTypeSuffix;
+    }
+
     public LiteralExpression(Object value) {
         this.value = value;
         this.translateToParameter = value instanceof String;
@@ -86,6 +94,10 @@ public class LiteralExpression implements GroovyExpression {
 
     }
 
+    public Object getValue() {
+        return value;
+    }
+
     private String getEscapedValue() {
         String escapedValue = (String)value;
         escapedValue = escapedValue.replaceAll(Pattern.quote("\\"), Matcher.quoteReplacement("\\\\"));
@@ -96,4 +108,15 @@ public class LiteralExpression implements GroovyExpression {
     public void setTranslateToParameter(boolean translateToParameter) {
         this.translateToParameter = translateToParameter;
     }
+
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 0;
+        return new LiteralExpression(value, addTypeSuffix, translateToParameter);
+    }
 }

common/src/main/java/org/apache/atlas/groovy/LogicalExpression.java

@@ -17,6 +17,8 @@
  */
 package org.apache.atlas.groovy;
 
+import java.util.List;
+
 /**
  * Represents a logical (and/or) expression.
  *
@@ -43,4 +45,14 @@ public class LogicalExpression extends BinaryExpression {
     public LogicalExpression(GroovyExpression left, LogicalOperator op, GroovyExpression right) {
         super(left, op.getGroovyValue(), right);
     }
+
+    private LogicalExpression(GroovyExpression left, String op, GroovyExpression right) {
+        super(left, op, right);
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 2;
+        return new LogicalExpression(newChildren.get(0), op, newChildren.get(1));
+    }
 }

common/src/main/java/org/apache/atlas/groovy/RangeExpression.java

@@ -18,28 +18,68 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.Collections;
+import java.util.List;
+
 /**
  * Represents an "exclusive" range expression, e.g. [0..<10].
  */
-public class RangeExpression extends AbstractGroovyExpression {
+public class RangeExpression extends AbstractFunctionExpression {
 
-    private GroovyExpression parent;
-    private int offset;
-    private int count;
+    private TraversalStepType stepType;
+    private int startIndex;
+    private int endIndex;
 
-    public RangeExpression(GroovyExpression parent, int offset, int count) {
-        this.parent = parent;
-        this.offset = offset;
-        this.count = count;
+    public RangeExpression(TraversalStepType stepType, GroovyExpression parent, int offset, int count) {
+        super(parent);
+        this.startIndex = offset;
+        this.endIndex = count;
+        this.stepType = stepType;
     }
 
     @Override
     public void generateGroovy(GroovyGenerationContext context) {
-        parent.generateGroovy(context);
+        getCaller().generateGroovy(context);
         context.append(" [");
-        new LiteralExpression(offset).generateGroovy(context);
+        new LiteralExpression(startIndex).generateGroovy(context);
         context.append("..<");
-        new LiteralExpression(count).generateGroovy(context);
+        new LiteralExpression(endIndex).generateGroovy(context);
         context.append("]");
     }
+
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.singletonList(getCaller());
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 1;
+        return new RangeExpression(stepType, newChildren.get(0), startIndex, endIndex);
+    }
+
+    @Override
+    public TraversalStepType getType() {
+        return stepType;
+    }
+
+    public int getStartIndex() {
+
+        return startIndex;
+    }
+
+    public void setStartIndex(int startIndex) {
+
+        this.startIndex = startIndex;
+    }
+
+    public int getEndIndex() {
+
+        return endIndex;
+    }
+
+    public void setEndIndex(int endIndex) {
+
+        this.endIndex = endIndex;
+    }
 }

common/src/main/java/org/apache/atlas/groovy/CodeBlockExpression.java → common/src/main/java/org/apache/atlas/groovy/StatementListExpression.java

@@ -19,35 +19,46 @@
 package org.apache.atlas.groovy;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
 /**
- * Groovy expression that represents a block of code
- * that contains 0 or more statements that are delimited
- * by semicolons.
+ * Represents a semi-colon delimited list of Groovy expressions.
  */
-public class CodeBlockExpression extends AbstractGroovyExpression {
+public class StatementListExpression extends AbstractGroovyExpression {
 
-    private List<GroovyExpression> body = new ArrayList<>();
+    private List<GroovyExpression> stmts = new ArrayList<>();
+
+    public StatementListExpression() {
+
+    }
+
+    /**
+     * @param newChildren
+     */
+    public StatementListExpression(List<GroovyExpression> newChildren) {
+        stmts.addAll(newChildren);
+    }
 
     public void addStatement(GroovyExpression expr) {
-        body.add(expr);
+        if (expr instanceof StatementListExpression) {
+            stmts.addAll(((StatementListExpression)expr).getStatements());
+        } else {
+            stmts.add(expr);
+        }
     }
 
     public void addStatements(List<GroovyExpression> exprs) {
-        body.addAll(exprs);
+        for(GroovyExpression expr : exprs) {
+            addStatement(expr);
+        }
     }
 
     @Override
     public void generateGroovy(GroovyGenerationContext context) {
 
-        /*
-         * the L:{} represents a groovy code block; the label is needed
-         * to distinguish it from a groovy closure.
-         */
-        context.append("L:{");
-        Iterator<GroovyExpression> stmtIt = body.iterator();
+        Iterator<GroovyExpression> stmtIt = stmts.iterator();
         while(stmtIt.hasNext()) {
             GroovyExpression stmt = stmtIt.next();
             stmt.generateGroovy(context);
@@ -55,7 +66,33 @@ public class CodeBlockExpression extends AbstractGroovyExpression {
                 context.append(";");
             }
         }
-        context.append("}");
+    }
+
+
+    public List<GroovyExpression> getStatements() {
+        return stmts;
+    }
+
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.unmodifiableList(stmts);
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        return new StatementListExpression(newChildren);
+    }
+
+    @Override
+    public TraversalStepType getType() {
+        return TraversalStepType.NONE;
+    }
 
+    /**
+     * @param oldExpr
+     * @param newExpr
+     */
+    public void replaceStatement(int index, GroovyExpression newExpr) {
+        stmts.set(index, newExpr);
     }
 }

common/src/main/java/org/apache/atlas/groovy/TernaryOperatorExpression.java

@@ -18,6 +18,9 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * Groovy expression that represents the ternary operator (expr ? trueValue :
  * falseValue)
@@ -29,7 +32,7 @@ public class TernaryOperatorExpression extends AbstractGroovyExpression {
     private GroovyExpression falseValue;
 
     public TernaryOperatorExpression(GroovyExpression booleanExpr, GroovyExpression trueValue,
-            GroovyExpression falseValue) {
+                                     GroovyExpression falseValue) {
 
         this.booleanExpr = booleanExpr;
         this.trueValue = trueValue;
@@ -41,9 +44,9 @@ public class TernaryOperatorExpression extends AbstractGroovyExpression {
 
         context.append("((");
         booleanExpr.generateGroovy(context);
-        context.append(") ? (");
+        context.append(")?(");
         trueValue.generateGroovy(context);
-        context.append(") : (");
+        context.append("):(");
         falseValue.generateGroovy(context);
         context.append("))");
     }
@@ -53,4 +56,20 @@ public class TernaryOperatorExpression extends AbstractGroovyExpression {
         generateGroovy(context);
         return context.getQuery();
     }
+
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Arrays.asList(booleanExpr, trueValue, falseValue);
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 3;
+        return new TernaryOperatorExpression(newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+
+    @Override
+    public TraversalStepType getType() {
+        return trueValue.getType();
+    }
 }

common/src/main/java/org/apache/atlas/groovy/TraversalStepType.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.groovy;
+
+/**
+ * Types of graph traversal steps.  These are based on the traversal steps
+ * described in the TinkerPop documentation at
+ * http://tinkerpop.apache.org/docs/current/reference/#graph-traversal-steps.
+ */
+public enum TraversalStepType {
+    /**
+     * Indicates that the expression is not part of a graph traversal.
+     */
+    NONE,
+
+    /**
+     * Indicates that the expression is a
+     * {@link org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource}.
+     * This is not technically a graph traversal step.  This is the expression the traversal is started from ("g").
+     */
+    SOURCE,
+
+    /**
+     * A Start step adds vertices or edges to the traversal.  These include "V", "E", and "inject".
+     */
+    START,
+
+    /**
+     * An End step causes the traversal to be executed.  This includes steps such as "toList", "toSet", and "fill"
+     */
+    END,
+
+    /**
+     * Map steps map the current traverser value to exactly one new value.  These
+     * steps include "map" and "select".  Here, we make a further distinction
+     * based on the type of expression that things are being mapped to.
+     * <p>
+     * MAP_TO_ELEMENT indicates that the traverser value is being mapped
+     * to either a Vertex or an Edge.
+     */
+    MAP_TO_ELEMENT,
+    /**
+     * Map steps map the current traverser value to exactly one new value.  These
+     * steps include "map" and "select".  Here, we make a further distinction
+     * based on the type of expression that things are being mapped to.
+     * <p>
+     * MAP_TO_VALUE indicates that the traverser value is being mapped
+     * to something that is not a Vertex or an Edge.
+     */
+    MAP_TO_VALUE,
+
+    /**
+     * FlatMap steps map the current value of the traverser to an iterator of objects that
+     * are streamed to the next step.  These are steps like "in, "out", "inE", and
+     * so forth which map the current value of the traverser from some vertex or edge
+     * to some other set of vertices or edges that is derived from the original set based
+     * on the structure of the graph.  This also includes "values", which maps a vertex or
+     * edge to the set of values for a given property. Here, we make a further distinction
+     * based on the type of expression that things are being mapped to.
+     * <p>
+     *  FLAT_MAP_TO_ELEMENTS indicates that the traverser value is being mapped
+     * to something that is a Vertex or an Edge (in, out, outE fall in this category).
+     */
+    FLAT_MAP_TO_ELEMENTS,
+
+    /**
+     * FlatMap steps map the current value of the traverser to an iterator of objects that
+     * are streamed to the next step.  These are steps like "in, "out", "inE", and
+     * so forth which map the current value of the traverser from some vertex or edge
+     * to some other set of vertices or edges that is derived from the original set based
+     * on the structure of the graph.  This also includes "values", which maps a vertex or
+     * edge to the set of values for a given property. Here, we make a further distinction
+     * based on the type of expression that things are being mapped to.
+     * <p>
+     *  FLAT_MAP_TO_VALUES indicates that the traverser value is being mapped
+     * to something that not is a Vertex or an Edge (values falls in this category).
+     */
+    FLAT_MAP_TO_VALUES,
+
+    /**
+     * Filter steps filter things out of the traversal.  These include "has", "where",
+     * "and", "or", and "filter".
+     */
+    FILTER,
+
+    /**
+     * Side effect steps do not affect the traverser value, but do something
+     * that affects the state of the traverser.  These include things such as
+     * "enablePath()", "as", and "by".
+     */
+    SIDE_EFFECT,
+
+    /**
+     * Branch steps split the traverser, for example, "repeat", "branch", "choose", and "union".
+     */
+    BRANCH,
+
+    /**
+     * Barrier steps in Gremlin force everything before them to be executed
+     * before moving on to the steps after them.  We also use this to indicate
+     * steps that need to do some aggregation or processing that requires the
+     * full query result to be present in order for the step to work correctly.
+     * This includes "range", "group", and "order", and "cap"
+     */
+    BARRIER,
+}

common/src/main/java/org/apache/atlas/groovy/TypeCoersionExpression.java

@@ -18,6 +18,9 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.Collections;
+import java.util.List;
+
 /**
  * Groovy expression that represents a type coersion (e.g obj as Set).
  */
@@ -28,17 +31,29 @@ public class TypeCoersionExpression extends AbstractGroovyExpression {
 
     public TypeCoersionExpression(GroovyExpression expr, String className) {
         this.expr = expr;
-        this.className  =className;
+        this.className  = className;
     }
 
     @Override
     public void generateGroovy(GroovyGenerationContext context) {
 
-        context.append("(");
+        context.append("((");
         expr.generateGroovy(context);
         context.append(")");
         context.append(" as ");
         context.append(className);
+        context.append(")");
+    }
+
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.singletonList(expr);
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 1;
+        return new TypeCoersionExpression(newChildren.get(0), className);
     }
 
 }

common/src/main/java/org/apache/atlas/groovy/VariableAssignmentExpression.java

@@ -18,6 +18,9 @@
 
 package org.apache.atlas.groovy;
 
+import java.util.Collections;
+import java.util.List;
+
 /**
  * Groovy statement that assigns a value to a variable.
  */
@@ -50,9 +53,20 @@ public class VariableAssignmentExpression extends AbstractGroovyExpression {
             context.append(" ");
         }
         context.append(name);
-        context.append(" = ");
+        context.append("=");
         value.generateGroovy(context);
 
     }
 
+    @Override
+    public List<GroovyExpression> getChildren() {
+        return Collections.singletonList(value);
+    }
+
+    @Override
+    public GroovyExpression copy(List<GroovyExpression> newChildren) {
+        assert newChildren.size() == 1;
+        return new VariableAssignmentExpression(name, newChildren.get(0));
+    }
+
 }

distro/src/conf/atlas-application.properties

@@ -29,6 +29,13 @@ atlas.graph.storage.hbase.table=apache_atlas_titan
 
 ${titan.storage.properties}
 
+# Gremlin Query Optimizer
+#
+# Enables rewriting gremlin queries to maximize performance. This flag is provided as
+# a possible way to work around any defects that are found in the optimizer until they
+# are resolved.
+#atlas.query.gremlinOptimizerEnabled=true
+
 # Delete handler
 #
 # This allows the default behavior of doing "soft" deletes to be changed.

graphdb/titan0/pom.xml

@@ -34,6 +34,7 @@
     <properties>
         <tinkerpop.version>2.6.0</tinkerpop.version>
         <titan.version>0.5.4</titan.version>
+	<guava.version>14.0</guava.version>
     </properties>
 
     <dependencies>
@@ -53,6 +54,13 @@
         </dependency>
 
         <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+        </dependency>
+
+
+        <dependency>
             <groupId>com.google.inject</groupId>
             <artifactId>guice</artifactId>
             <scope>provided</scope>

intg/pom.xml

@@ -60,7 +60,6 @@
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
-            <version>${guava.version}</version>
         </dependency>
 
         <dependency>

pom.xml

@@ -459,7 +459,7 @@
         <spring.security.version>3.1.3.RELEASE</spring.security.version>
         <spring-ldap-core.version>1.3.1.RELEASE</spring-ldap-core.version>
         <javax.servlet.version>3.1.0</javax.servlet.version>
-        <guava.version>18.0</guava.version>
+        <guava.version>19.0</guava.version>
 
         <!-- Needed for hooks -->
         <aopalliance.version>1.0</aopalliance.version>
@@ -633,6 +633,12 @@
 
     <dependencyManagement>
         <dependencies>
+
+            <dependency>
+                <groupId>com.google.guava</groupId>
+                <artifactId>guava</artifactId>
+                <version>${guava.version}</version>
+            </dependency>
             <!-- AOP dependencies. -->
             <dependency>
                 <groupId>org.aspectj</groupId>

release-log.txt

@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb)
 ATLAS-1517: updated hive_model to include schema related attributes (sarath.kum4r@gmail.com via mneethiraj)
 ATLAS-1514 Remove duplicates from class array attribute when target is deleted (dkantor)
 ATLAS-1509: fixed issues with deletion during updates (sumasai via mneethiraj)

repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java

@@ -18,6 +18,12 @@
 
 package org.apache.atlas.discovery;
 
+import java.util.Arrays;
+import java.util.Iterator;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasConfiguration;
@@ -25,6 +31,7 @@ import org.apache.atlas.AtlasException;
 import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
 import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.query.GremlinQueryResult;
 import org.apache.atlas.query.InputLineageClosureQuery;
 import org.apache.atlas.query.OutputLineageClosureQuery;
 import org.apache.atlas.query.QueryParams;
@@ -42,16 +49,12 @@ import org.apache.atlas.utils.ParamChecker;
 import org.apache.commons.configuration.Configuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import scala.Option;
 import scala.Some;
 import scala.collection.JavaConversions;
 import scala.collection.immutable.List;
 
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import java.util.Arrays;
-import java.util.Iterator;
-
 /**
  * Hive implementation of Lineage service interface.
  */
@@ -139,7 +142,8 @@ public class DataSetLineageService implements LineageService {
                 guid, HIVE_PROCESS_TYPE_NAME,
                 HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
                 SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
-        return inputsQuery.graph(null).toInstanceJson();
+        GremlinQueryResult result = inputsQuery.evaluate();
+        return inputsQuery.graph(result).toInstanceJson();
     }
 
     @Override
@@ -156,7 +160,8 @@ public class DataSetLineageService implements LineageService {
                 new OutputLineageClosureQuery(AtlasClient.DATA_SET_SUPER_TYPE, SELECT_INSTANCE_GUID, guid, HIVE_PROCESS_TYPE_NAME,
                         HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
                         SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
-        return outputsQuery.graph(null).toInstanceJson();
+        GremlinQueryResult result = outputsQuery.evaluate();
+        return outputsQuery.graph(result).toInstanceJson();
     }
 
     /**

repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java

@@ -18,7 +18,12 @@
 
 package org.apache.atlas.gremlin;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
 import org.apache.atlas.AtlasException;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
 import org.apache.atlas.groovy.CastExpression;
 import org.apache.atlas.groovy.ClosureExpression;
 import org.apache.atlas.groovy.ComparisonExpression;
@@ -34,13 +39,11 @@ import org.apache.atlas.groovy.LogicalExpression;
 import org.apache.atlas.groovy.LogicalExpression.LogicalOperator;
 import org.apache.atlas.groovy.RangeExpression;
 import org.apache.atlas.groovy.TernaryOperatorExpression;
+import org.apache.atlas.groovy.TraversalStepType;
 import org.apache.atlas.query.GraphPersistenceStrategies;
 import org.apache.atlas.query.TypeUtils.FieldInfo;
 import org.apache.atlas.typesystem.types.IDataType;
 
-import java.util.ArrayList;
-import java.util.List;
-
 
 /**
  * Generates gremlin query expressions using Gremlin 2 syntax.
@@ -54,12 +57,11 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
     private static final String PATH_FIELD = "path";
     private static final String ENABLE_PATH_METHOD = "enablePath";
     private static final String BACK_METHOD = "back";
-    private static final String VERTEX_LIST_CLASS = "List<Vertex>";
-    private static final String VERTEX_ARRAY_CLASS = "Vertex[]";
     private static final String LAST_METHOD = "last";
+
     @Override
     public GroovyExpression generateLogicalExpression(GroovyExpression parent, String operator, List<GroovyExpression> operands) {
-        return new FunctionCallExpression(parent, operator, operands);
+        return new FunctionCallExpression(TraversalStepType.FILTER, parent, operator, operands);
     }
 
 
@@ -72,7 +74,7 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
             return parent;
         }
         else {
-            return new FunctionCallExpression(parent, BACK_METHOD, new LiteralExpression(alias));
+            return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, BACK_METHOD, new LiteralExpression(alias));
         }
     }
 
@@ -100,23 +102,23 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
             whileFunction = new ClosureExpression(new TernaryOperatorExpression(pathContainsExpr, LiteralExpression.FALSE, LiteralExpression.TRUE));
         }
         GroovyExpression emitFunction = new ClosureExpression(emitExpr);
-        GroovyExpression loopCall = new FunctionCallExpression(loopExpr, LOOP_METHOD, new LiteralExpression(alias), whileFunction, emitFunction);
+        GroovyExpression loopCall = new FunctionCallExpression(TraversalStepType.BRANCH, loopExpr, LOOP_METHOD, new LiteralExpression(alias), whileFunction, emitFunction);
 
-        return new FunctionCallExpression(loopCall, ENABLE_PATH_METHOD);
+        return new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, loopCall, ENABLE_PATH_METHOD);
     }
 
     @Override
     public GroovyExpression typeTestExpression(GraphPersistenceStrategies s, String typeName, GroovyExpression itRef) {
 
-        GroovyExpression typeAttrExpr = new FieldExpression(itRef, s.typeAttributeName());
         GroovyExpression superTypeAttrExpr = new FieldExpression(itRef, s.superTypeAttributeName());
         GroovyExpression typeNameExpr = new LiteralExpression(typeName);
-
-        GroovyExpression typeMatchesExpr = new ComparisonExpression(typeAttrExpr, ComparisonOperator.EQUALS, typeNameExpr);
         GroovyExpression isSuperTypeExpr = new FunctionCallExpression(superTypeAttrExpr, CONTAINS, typeNameExpr);
         GroovyExpression superTypeMatchesExpr = new TernaryOperatorExpression(superTypeAttrExpr, isSuperTypeExpr, LiteralExpression.FALSE);
 
+        GroovyExpression typeAttrExpr = new FieldExpression(itRef, s.typeAttributeName());
+        GroovyExpression typeMatchesExpr = new ComparisonExpression(typeAttrExpr, ComparisonOperator.EQUALS, typeNameExpr);
         return new LogicalExpression(typeMatchesExpr, LogicalOperator.OR, superTypeMatchesExpr);
+
     }
 
     @Override
@@ -129,7 +131,7 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
         for(GroovyExpression expr : srcExprs) {
             selectArgs.add(new ClosureExpression(expr));
         }
-        return new FunctionCallExpression(parent, SELECT_METHOD, selectArgs);
+        return new FunctionCallExpression(TraversalStepType.MAP_TO_VALUE, parent, SELECT_METHOD, selectArgs);
     }
 
     @Override
@@ -142,7 +144,7 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
             GroovyExpression requiredValue, FieldInfo fInfo) throws AtlasException {
         GroovyExpression op = gremlin2CompOp(symbol);
         GroovyExpression propertyNameExpr = new LiteralExpression(propertyName);
-        return new FunctionCallExpression(parent, HAS_METHOD, propertyNameExpr, op, requiredValue);
+        return new FunctionCallExpression(TraversalStepType.FILTER, parent, HAS_METHOD, propertyNameExpr, op, requiredValue);
     }
 
     private GroovyExpression gremlin2CompOp(String op) throws AtlasException {
@@ -173,13 +175,52 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
     }
 
     @Override
-    protected GroovyExpression initialExpression(GraphPersistenceStrategies s, GroovyExpression varExpr) {
-        return new FunctionCallExpression(varExpr, "_");
+    protected GroovyExpression initialExpression(GroovyExpression varExpr, GraphPersistenceStrategies s) {
+        return generateSeededTraversalExpresssion(false, varExpr);
+    }
+
+    @Override
+    public GroovyExpression generateSeededTraversalExpresssion(boolean isMap, GroovyExpression varExpr) {
+        return new FunctionCallExpression(TraversalStepType.START, varExpr, "_");
     }
 
     @Override
-    public GroovyExpression generateLimitExpression(GroovyExpression parent, int offset, int totalRows) {
-        return new RangeExpression(parent, offset, totalRows);
+    public GroovyExpression generateRangeExpression(GroovyExpression parent, int startIndex, int endIndex) {
+        //treat as barrier step, since limits need to be applied globally (even though it
+        //is technically a filter step)
+        return new RangeExpression(TraversalStepType.BARRIER, parent, startIndex, endIndex);
+    }
+
+    @Override
+    public boolean isRangeExpression(GroovyExpression expr) {
+
+        return (expr instanceof RangeExpression);
+    }
+
+    @Override
+    public int[] getRangeParameters(AbstractFunctionExpression expr) {
+
+        if (isRangeExpression(expr)) {
+            RangeExpression rangeExpression = (RangeExpression) expr;
+            return new int[] {rangeExpression.getStartIndex(), rangeExpression.getEndIndex()};
+        }
+        else {
+            return null;
+        }
+    }
+
+    @Override
+    public void setRangeParameters(GroovyExpression expr, int startIndex, int endIndex) {
+
+        if (isRangeExpression(expr)) {
+            RangeExpression rangeExpression = (RangeExpression) expr;
+            rangeExpression.setStartIndex(startIndex);
+            rangeExpression.setEndIndex(endIndex);
+        }
+        else {
+            throw new IllegalArgumentException(expr.getClass().getName() + " is not a valid range expression - must be an instance of " + RangeExpression.class.getName());
+        }
+
     }
 
     @Override
@@ -195,7 +236,7 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
 
     @Override
     public GroovyExpression generateOrderByExpression(GroovyExpression parent, List<GroovyExpression> translatedOrderBy, boolean isAscending) {
-        GroovyExpression itExpr = getItVariable();
+
         GroovyExpression aPropertyExpr = translatedOrderBy.get(0);
         GroovyExpression bPropertyExpr = translatedOrderBy.get(1);
 
@@ -212,27 +253,28 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
         else {
             comparisonFunction = new ComparisonOperatorExpression(bCondition,  aCondition);
         }
-        return new FunctionCallExpression(parent, ORDER_METHOD, new ClosureExpression(comparisonFunction));
+        return new FunctionCallExpression(TraversalStepType.BARRIER, parent, ORDER_METHOD, new ClosureExpression(comparisonFunction));
     }
 
+
     @Override
     public GroovyExpression getAnonymousTraversalExpression() {
-        return new FunctionCallExpression("_");
+        return new FunctionCallExpression(TraversalStepType.START, "_");
     }
 
+
+
     @Override
     public GroovyExpression generateGroupByExpression(GroovyExpression parent, GroovyExpression groupByExpression,
-            GroovyExpression aggregationFunction) {
-
+                                                      GroovyExpression aggregationFunction) {
             GroovyExpression groupByClosureExpr = new ClosureExpression(groupByExpression);
             GroovyExpression itClosure = new ClosureExpression(getItVariable());
-            GroovyExpression result = new FunctionCallExpression(parent, "groupBy", groupByClosureExpr, itClosure);
-            result = new FunctionCallExpression(result, "cap");
-            result = new FunctionCallExpression(result, "next");
+            GroovyExpression result = new FunctionCallExpression(TraversalStepType.BARRIER, parent, "groupBy", groupByClosureExpr, itClosure);
+            result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, "cap");
+            result = new FunctionCallExpression(TraversalStepType.END, result, "next");
             result = new FunctionCallExpression(result, "values");
             result = new FunctionCallExpression(result, "toList");
 
-            GroovyExpression mapValuesClosure = new ClosureExpression(getItVariable());
             GroovyExpression aggregrationFunctionClosure = new ClosureExpression(aggregationFunction);
             result = new FunctionCallExpression(result, "collect", aggregrationFunctionClosure);
             return result;
@@ -251,8 +293,49 @@ public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
     //assumes cast already performed
     @Override
     public GroovyExpression generateCountExpression(GroovyExpression itExpr) {
-        GroovyExpression collectionExpr = new CastExpression(itExpr,"Collection");
         return new FunctionCallExpression(itExpr, "size");
     }
+
+    @Override
+    public String getTraversalExpressionClass() {
+        return "GremlinPipeline";
+    }
+
+
+    @Override
+    public boolean isSelectGeneratesMap(int aliasCount) {
+        //in Gremlin 2 select always generates a map
+        return true;
+    }
+
+    @Override
+    public GroovyExpression generateMapExpression(GroovyExpression parent, ClosureExpression closureExpression) {
+        return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, "transform", closureExpression);
+    }
+
+    @Override
+    public GroovyExpression generateGetSelectedValueExpression(LiteralExpression key,
+            GroovyExpression rowMap) {
+        rowMap = new CastExpression(rowMap, "Row");
+        GroovyExpression getExpr = new FunctionCallExpression(rowMap, "getColumn", key);
+        return getExpr;
+    }
+
+    @Override
+    public GroovyExpression getCurrentTraverserObject(GroovyExpression traverser) {
+        return traverser;
+    }
+
+    public List<String> getAliasesRequiredByExpression(GroovyExpression expr) {
+        if(!(expr instanceof FunctionCallExpression)) {
+            return Collections.emptyList();
+        }
+        FunctionCallExpression fc = (FunctionCallExpression)expr;
+        if(! fc.getFunctionName().equals(LOOP_METHOD)) {
+            return Collections.emptyList();
+        }
+       LiteralExpression aliasName =  (LiteralExpression)fc.getArguments().get(0);
+       return Collections.singletonList(aliasName.getValue().toString());
+    }
 }
 

repository/src/main/java/org/apache/atlas/gremlin/Gremlin3ExpressionFactory.java

@@ -19,21 +19,25 @@
 package org.apache.atlas.gremlin;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.atlas.AtlasException;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
 import org.apache.atlas.groovy.CastExpression;
 import org.apache.atlas.groovy.ClosureExpression;
 import org.apache.atlas.groovy.ComparisonExpression;
+import org.apache.atlas.groovy.ComparisonExpression.ComparisonOperator;
 import org.apache.atlas.groovy.ComparisonOperatorExpression;
 import org.apache.atlas.groovy.FunctionCallExpression;
 import org.apache.atlas.groovy.GroovyExpression;
 import org.apache.atlas.groovy.IdentifierExpression;
 import org.apache.atlas.groovy.LiteralExpression;
 import org.apache.atlas.groovy.LogicalExpression;
-import org.apache.atlas.groovy.TypeCoersionExpression;
-import org.apache.atlas.groovy.ComparisonExpression.ComparisonOperator;
 import org.apache.atlas.groovy.LogicalExpression.LogicalOperator;
+import org.apache.atlas.groovy.TernaryOperatorExpression;
+import org.apache.atlas.groovy.TraversalStepType;
+import org.apache.atlas.groovy.TypeCoersionExpression;
 import org.apache.atlas.query.GraphPersistenceStrategies;
 import org.apache.atlas.query.TypeUtils.FieldInfo;
 import org.apache.atlas.repository.graph.AtlasGraphProvider;
@@ -69,27 +73,14 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory {
     private static final String REPEAT_METHOD = "repeat";
     private static final String RANGE_METHOD = "range";
     private static final String LAST_METHOD = "last";
-    private static final String WHERE_METHOD = "where";
     private static final String TO_STRING_METHOD = "toString";
 
+    private static final GroovyExpression EMPTY_STRING_EXPRESSION = new LiteralExpression("");
+
     @Override
     public GroovyExpression generateLogicalExpression(GroovyExpression parent, String operator,
-            List<GroovyExpression> operands) {
-        if (operands.size() == 1) {
-            // gremlin 3 treats one element expressions as 'false'. Avoid
-            // creating a boolean expression in this case. Inline the expression
-            // note: we can't simply omit it, since it will cause us to traverse
-            // the edge!
-            // use 'where' instead
-            GroovyExpression expr = operands.get(0);
-            // if child is a back expression, that expression becomes an
-            // argument to where
-            return new FunctionCallExpression(parent, WHERE_METHOD, expr);
-        } else {
-            // Gremlin 3 does not support _() syntax
-            //
-            return new FunctionCallExpression(parent, operator, operands);
-        }
+                                                      List<GroovyExpression> operands) {
+        return new FunctionCallExpression(TraversalStepType.FILTER, parent, operator, operands);
     }
 
     @Override
@@ -97,20 +88,20 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory {
         if (inSelect) {
             return getFieldInSelect();
         } else {
-            return new FunctionCallExpression(parent, SELECT_METHOD, new LiteralExpression(alias));
+            return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, SELECT_METHOD, new LiteralExpression(alias));
         }
     }
 
     @Override
     public GroovyExpression typeTestExpression(GraphPersistenceStrategies s, String typeName, GroovyExpression itRef) {
-        LiteralExpression typeAttrExpr = new LiteralExpression(s.typeAttributeName());
         LiteralExpression superTypeAttrExpr = new LiteralExpression(s.superTypeAttributeName());
         LiteralExpression typeNameExpr = new LiteralExpression(typeName);
-
-        FunctionCallExpression result = new FunctionCallExpression(HAS_METHOD, typeAttrExpr, new FunctionCallExpression(EQ_METHOD, typeNameExpr));
-        result = new FunctionCallExpression(result, "or");
-        result = new FunctionCallExpression(HAS_METHOD, superTypeAttrExpr, new FunctionCallExpression(EQ_METHOD, typeNameExpr));
+        LiteralExpression typeAttrExpr = new LiteralExpression(s.typeAttributeName());
+        FunctionCallExpression result = new FunctionCallExpression(TraversalStepType.FILTER, HAS_METHOD, typeAttrExpr, new FunctionCallExpression(EQ_METHOD, typeNameExpr));
+        result = new FunctionCallExpression(TraversalStepType.FILTER, result, "or");
+        result = new FunctionCallExpression(TraversalStepType.FILTER, result, HAS_METHOD, superTypeAttrExpr, new FunctionCallExpression(EQ_METHOD, typeNameExpr));
         return result;
+
     }
 
     @Override
@@ -118,41 +109,48 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory {
 
         GroovyExpression emitExpr = generateLoopEmitExpression(s, dataType);
 
-        GroovyExpression result = new FunctionCallExpression(parent, REPEAT_METHOD, loopExpr);
+        GroovyExpression result = new FunctionCallExpression(TraversalStepType.BRANCH, parent, REPEAT_METHOD, loopExpr);
         if (times != null) {
             GroovyExpression timesExpr = new LiteralExpression(times);
-            result = new FunctionCallExpression(result, TIMES_METHOD, timesExpr);
+            result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, TIMES_METHOD, timesExpr);
         }
-        result = new FunctionCallExpression(result, EMIT_METHOD, emitExpr);
+        result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, EMIT_METHOD, emitExpr);
         return result;
 
     }
 
     @Override
     public GroovyExpression getLoopExpressionParent(GroovyExpression inputQry) {
-        return new IdentifierExpression("__");
+        GroovyExpression curTraversal = getAnonymousTraversalStartExpression();
+        return curTraversal;
+    }
+
+    private IdentifierExpression getAnonymousTraversalStartExpression() {
+        return new IdentifierExpression(TraversalStepType.START, "__");
     }
 
     @Override
     public GroovyExpression generateSelectExpression(GroovyExpression parent, List<LiteralExpression> sourceNames,
             List<GroovyExpression> srcExprs) {
-        FunctionCallExpression result = new FunctionCallExpression(parent, SELECT_METHOD, sourceNames);
+        FunctionCallExpression result = new FunctionCallExpression(TraversalStepType.MAP_TO_VALUE, parent, SELECT_METHOD, sourceNames);
 
         for (GroovyExpression expr : srcExprs) {
             GroovyExpression closure = new ClosureExpression(expr);
             GroovyExpression castClosure = new TypeCoersionExpression(closure, FUNCTION_CLASS);
-            result = new FunctionCallExpression(result, BY_METHOD, castClosure);
+            result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, BY_METHOD, castClosure);
         }
         return result;
     }
 
     @Override
     public GroovyExpression generateFieldExpression(GroovyExpression parent, FieldInfo fInfo,
-            String propertyName, boolean inSelect) {
+            String propertyName, boolean inSelect)  {
 
         AttributeInfo attrInfo = fInfo.attrInfo();
         IDataType attrType = attrInfo.dataType();
         GroovyExpression propertyNameExpr = new LiteralExpression(propertyName);
+        //Whether it is the user or shared graph does not matter here, since we're
+        //just getting the conversion expression.  Ideally that would be moved someplace else.
         AtlasGraph graph = AtlasGraphProvider.getGraphInstance();
         if (inSelect) {
 
@@ -161,13 +159,13 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory {
             return graph.generatePersisentToLogicalConversionExpression(expr, attrType);
         } else {
 
-            GroovyExpression unmapped = new FunctionCallExpression(parent, VALUES_METHOD, propertyNameExpr);
+            GroovyExpression unmapped = new FunctionCallExpression(TraversalStepType.FLAT_MAP_TO_VALUES, parent, VALUES_METHOD, propertyNameExpr);
             if (graph.isPropertyValueConversionNeeded(attrType)) {
                 GroovyExpression toConvert = new FunctionCallExpression(getItVariable(), GET_METHOD);
 
                 GroovyExpression conversionFunction = graph.generatePersisentToLogicalConversionExpression(toConvert,
                         attrType);
-                return new FunctionCallExpression(unmapped, MAP_METHOD, new ClosureExpression(conversionFunction));
+                return new FunctionCallExpression(TraversalStepType.MAP_TO_VALUE, unmapped, MAP_METHOD, new ClosureExpression(conversionFunction));
             } else {
                 return unmapped;
             }
@@ -238,15 +236,15 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory {
                     valueMatchesExpr);
 
             GroovyExpression filterFunction = new ClosureExpression(filterCondition);
-            return new FunctionCallExpression(parent, FILTER_METHOD, filterFunction);
+            return new FunctionCallExpression(TraversalStepType.FILTER, parent, FILTER_METHOD, filterFunction);
         } else {
             GroovyExpression valueMatches = new FunctionCallExpression(getComparisonFunction(symbol), requiredValue);
-            return new FunctionCallExpression(parent, HAS_METHOD, propertNameExpr, valueMatches);
+            return new FunctionCallExpression(TraversalStepType.FILTER, parent, HAS_METHOD, propertNameExpr, valueMatches);
         }
     }
 
     @Override
-    protected GroovyExpression initialExpression(GraphPersistenceStrategies s, GroovyExpression varExpr) {
+    protected GroovyExpression initialExpression(GroovyExpression varExpr, GraphPersistenceStrategies s) {
 
         // this bit of groovy magic converts the set of vertices in varName into
         // a String containing the ids of all the vertices. This becomes the
@@ -255,15 +253,61 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory {
         // _()
         // s"g.V(${varName}.collect{it.id()} as String[])"
 
-        GroovyExpression gExpr = getGraph();
+        GroovyExpression gExpr = getGraphExpression();
         GroovyExpression varRefExpr = new TypeCoersionExpression(varExpr, OBJECT_ARRAY_CLASS);
-        FunctionCallExpression expr = new FunctionCallExpression(gExpr, V_METHOD, varRefExpr);
+        GroovyExpression matchingVerticesExpr = new FunctionCallExpression(TraversalStepType.START, gExpr, V_METHOD, varRefExpr);
+        GroovyExpression isEmpty  = new FunctionCallExpression(varExpr, "isEmpty");
+        GroovyExpression emptyGraph = getEmptyTraversalExpression();
+
+        GroovyExpression expr = new TernaryOperatorExpression(isEmpty, emptyGraph, matchingVerticesExpr);
+
         return s.addInitialQueryCondition(expr);
     }
 
+    private GroovyExpression getEmptyTraversalExpression() {
+        GroovyExpression emptyGraph = new FunctionCallExpression(TraversalStepType.START, getGraphExpression(), V_METHOD, EMPTY_STRING_EXPRESSION);
+        return emptyGraph;
+    }
+
+    @Override
+    public GroovyExpression generateRangeExpression(GroovyExpression parent, int startIndex, int endIndex) {
+        //treat as barrier step, since limits need to be applied globally (even though it
+        //is technically a filter step)
+        return new FunctionCallExpression(TraversalStepType.BARRIER, parent, RANGE_METHOD, new LiteralExpression(startIndex), new LiteralExpression(endIndex));
+    }
+
+    @Override
+    public boolean isRangeExpression(GroovyExpression expr) {
+
+        return (expr instanceof FunctionCallExpression && ((FunctionCallExpression)expr).getFunctionName().equals(RANGE_METHOD));
+    }
+
+    @Override
+    public int[] getRangeParameters(AbstractFunctionExpression expr) {
+
+        if (isRangeExpression(expr)) {
+            FunctionCallExpression rangeExpression = (FunctionCallExpression) expr;
+            List<GroovyExpression> arguments = rangeExpression.getArguments();
+            int startIndex = (int)((LiteralExpression)arguments.get(0)).getValue();
+            int endIndex = (int)((LiteralExpression)arguments.get(1)).getValue();
+            return new int[]{startIndex, endIndex};
+        }
+        else {
+            return null;
+        }
+    }
+
     @Override
-    public GroovyExpression generateLimitExpression(GroovyExpression parent, int offset, int totalRows) {
-        return new FunctionCallExpression(parent, RANGE_METHOD, new LiteralExpression(offset), new LiteralExpression(totalRows));
+    public void setRangeParameters(GroovyExpression expr, int startIndex, int endIndex) {
+
+        if (isRangeExpression(expr)) {
+            FunctionCallExpression rangeExpression = (FunctionCallExpression) expr;
+            rangeExpression.setArgument(0, new LiteralExpression(Integer.valueOf(startIndex)));
+            rangeExpression.setArgument(1, new LiteralExpression(Integer.valueOf(endIndex)));
+        }
+        else {
+            throw new IllegalArgumentException(expr + " is not a valid range expression");
+        }
     }
 
     @Override
@@ -295,7 +339,8 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory {
             comparisonExpr = new ComparisonOperatorExpression(bCompExpr, aCompExpr);
         }
         ClosureExpression comparisonFunction = new ClosureExpression(comparisonExpr, "a", "b");
-        return new FunctionCallExpression(new FunctionCallExpression(parent, ORDER_METHOD), BY_METHOD, orderByClause, comparisonFunction);
+        FunctionCallExpression orderCall = new FunctionCallExpression(TraversalStepType.BARRIER, parent, ORDER_METHOD);
+        return new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, orderCall, BY_METHOD, orderByClause, comparisonFunction);
     }
 
     @Override
@@ -327,10 +372,10 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory {
     public GroovyExpression generateGroupByExpression(GroovyExpression parent, GroovyExpression groupByExpression,
             GroovyExpression aggregationFunction) {
 
-        GroovyExpression result = new FunctionCallExpression(parent, "group");
+        GroovyExpression result = new FunctionCallExpression(TraversalStepType.BARRIER, parent, "group");
         GroovyExpression groupByClosureExpr = new TypeCoersionExpression(new ClosureExpression(groupByExpression), "Function");
-        result = new FunctionCallExpression(result, "by", groupByClosureExpr);
-        result = new FunctionCallExpression(result, "toList");
+        result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, "by", groupByClosureExpr);
+        result = new FunctionCallExpression(TraversalStepType.END, result, "toList");
 
         GroovyExpression mapValuesClosure = new ClosureExpression(new FunctionCallExpression(new CastExpression(getItVariable(), "Map"), "values"));
 
@@ -347,8 +392,55 @@ public class Gremlin3ExpressionFactory extends GremlinExpressionFactory {
         result = new FunctionCallExpression(result, "collect", aggregrationFunctionClosure);
         return result;
     }
+
+    @Override
+    public GroovyExpression generateSeededTraversalExpresssion(boolean isMap, GroovyExpression valueCollection) {
+        GroovyExpression coersedExpression = new TypeCoersionExpression(valueCollection, isMap ? "Map[]" : "Vertex[]");
+        if(isMap) {
+
+            return new FunctionCallExpression(TraversalStepType.START, "__", coersedExpression);
+        }
+        else {
+            //We cannot always use an anonymous traversal because that breaks repeat steps
+            return new FunctionCallExpression(TraversalStepType.START, getEmptyTraversalExpression(), "inject", coersedExpression);
+        }
+    }
+
+    @Override
     public GroovyExpression getGroupBySelectFieldParent() {
         return null;
     }
-}
 
+    @Override
+    public String getTraversalExpressionClass() {
+        return "GraphTraversal";
+    }
+
+    @Override
+    public boolean isSelectGeneratesMap(int aliasCount) {
+        //in Gremlin 3, you only get a map if there is more than 1 alias.
+        return aliasCount > 1;
+    }
+
+    @Override
+    public GroovyExpression generateMapExpression(GroovyExpression parent, ClosureExpression closureExpression) {
+        return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, "map", closureExpression);
+    }
+
+    @Override
+    public GroovyExpression generateGetSelectedValueExpression(LiteralExpression key,
+            GroovyExpression rowMapExpr) {
+        rowMapExpr = new CastExpression(rowMapExpr, "Map");
+        GroovyExpression getExpr = new FunctionCallExpression(rowMapExpr, "get", key);
+        return getExpr;
+    }
+
+    @Override
+    public GroovyExpression getCurrentTraverserObject(GroovyExpression traverser) {
+        return new FunctionCallExpression(traverser, "get");
+    }
+
+    public List<String> getAliasesRequiredByExpression(GroovyExpression expr) {
+        return Collections.emptyList();
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/GremlinExpressionFactory.java

@@ -17,8 +17,14 @@
  */
 package org.apache.atlas.gremlin;
 
-import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.atlas.AtlasException;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
 import org.apache.atlas.groovy.ArithmeticExpression;
 import org.apache.atlas.groovy.ArithmeticExpression.ArithmeticOperator;
 import org.apache.atlas.groovy.CastExpression;
@@ -29,6 +35,7 @@ import org.apache.atlas.groovy.GroovyExpression;
 import org.apache.atlas.groovy.IdentifierExpression;
 import org.apache.atlas.groovy.ListExpression;
 import org.apache.atlas.groovy.LiteralExpression;
+import org.apache.atlas.groovy.TraversalStepType;
 import org.apache.atlas.groovy.TypeCoersionExpression;
 import org.apache.atlas.groovy.VariableAssignmentExpression;
 import org.apache.atlas.query.GraphPersistenceStrategies;
@@ -40,13 +47,9 @@ import org.apache.atlas.repository.graphdb.GremlinVersion;
 import org.apache.atlas.typesystem.types.IDataType;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.atlas.typesystem.types.cache.TypeCache.TYPE_FILTER;
+import org.apache.atlas.util.AtlasRepositoryConfiguration;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import com.google.common.collect.ImmutableList;
 
 /**
  * Factory to generate Groovy expressions representing Gremlin syntax that that
@@ -58,7 +61,8 @@ public abstract class GremlinExpressionFactory {
     private static final String G_VARIABLE = "g";
     private static final String IT_VARIABLE = "it";
 
-    private static final String SET_CLASS = "Set";
+    protected static final String SET_CLASS = "Set";
+
 
     private static final String OBJECT_FIELD = "object";
 
@@ -66,18 +70,25 @@ public abstract class GremlinExpressionFactory {
     protected static final String FILTER_METHOD = "filter";
     private static final String PATH_METHOD = "path";
     private static final String AS_METHOD = "as";
-    private static final String FILL_METHOD = "fill";
     private static final String IN_OPERATOR = "in";
     protected static final String HAS_METHOD = "has";
     protected static final String TO_LOWER_CASE_METHOD = "toLowerCase";
     protected static final String SELECT_METHOD = "select";
     protected static final String ORDER_METHOD = "order";
+    protected static final String FILL_METHOD = "fill";
 
     public static final GremlinExpressionFactory INSTANCE = AtlasGraphProvider.getGraphInstance()
             .getSupportedGremlinVersion() == GremlinVersion.THREE ? new Gremlin3ExpressionFactory()
                     : new Gremlin2ExpressionFactory();
 
     /**
+     * Returns the unqualified name of the class used in this version of gremlin to
+     * represent Gremlin queries as they are being generated.
+     * @return
+     */
+    public abstract String getTraversalExpressionClass();
+
+    /**
      * Gets the expression to use as the parent when translating the loop
      * expression in a loop
      *
@@ -172,14 +183,40 @@ public abstract class GremlinExpressionFactory {
             String propertyName, String symbol, GroovyExpression requiredValue, FieldInfo fInfo) throws AtlasException;
 
     /**
-     * Generates a limit expression
+     * Generates a range expression
      *
      * @param parent
-     * @param offset
-     * @param totalRows
+     * @param startIndex
+     * @param endIndex
      * @return
      */
-    public abstract GroovyExpression generateLimitExpression(GroovyExpression parent, int offset, int totalRows);
+    public abstract GroovyExpression generateRangeExpression(GroovyExpression parent, int startIndex, int endIndex);
+
+    /**
+     * Determines if the specified expression is a range method call.
+     *
+     * @param expr
+     * @return
+     */
+    public abstract boolean isRangeExpression(GroovyExpression expr);
+
+    /**
+     * Set the start index and end index of a range expression
+     *
+     * @param expr
+     * @param startIndex
+     * @param endIndex
+     */
+    public abstract void setRangeParameters(GroovyExpression expr, int startIndex, int endIndex);
+
+    /**
+     * If the specified function expression is a range expression, returns the start and end index parameters
+     * otherwise returns null.
+     *
+     * @param expr
+     * @return int array with two elements - element 0 is start index, element 1 is end index
+     */
+    public abstract int[] getRangeParameters(AbstractFunctionExpression expr);
 
     /**
      * Generates an order by expression
@@ -193,6 +230,22 @@ public abstract class GremlinExpressionFactory {
             List<GroovyExpression> translatedOrderBy, boolean isAscending);
 
     /**
+     * Determines if specified expression is an order method call
+     *
+     * @param expr
+     * @return
+     */
+    public boolean isOrderExpression(GroovyExpression expr) {
+        if (expr instanceof FunctionCallExpression) {
+            FunctionCallExpression functionCallExpression = (FunctionCallExpression) expr;
+            if (functionCallExpression.getFunctionName().equals(ORDER_METHOD)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
      * Returns the Groovy expressions that should be used as the parents when
      * translating an order by expression. This is needed because Gremlin 2 and
      * 3 handle order by expressions very differently.
@@ -207,6 +260,17 @@ public abstract class GremlinExpressionFactory {
      */
     public abstract GroovyExpression getAnonymousTraversalExpression();
 
+    public boolean isLeafAnonymousTraversalExpression(GroovyExpression expr) {
+        if(!(expr instanceof FunctionCallExpression)) {
+            return false;
+        }
+        FunctionCallExpression functionCallExpr = (FunctionCallExpression)expr;
+        if(functionCallExpr.getCaller() != null) {
+            return false;
+        }
+        return functionCallExpr.getFunctionName().equals("_") & functionCallExpr.getArguments().size() == 0;
+    }
+
     /**
      * Returns an expression representing
      *
@@ -216,11 +280,11 @@ public abstract class GremlinExpressionFactory {
 
     /**
      * Generates the expression the serves as the root of the Gremlin query.
-     * @param s
      * @param varExpr variable containing the vertices to traverse
      * @return
      */
-    protected abstract GroovyExpression initialExpression(GraphPersistenceStrategies s, GroovyExpression varExpr);
+    protected abstract GroovyExpression initialExpression(GroovyExpression varExpr, GraphPersistenceStrategies s);
+
 
     /**
      * Generates an expression that tests whether the vertex represented by the 'toTest'
@@ -236,19 +300,37 @@ public abstract class GremlinExpressionFactory {
             GroovyExpression vertexExpr);
 
     /**
+    /**
      * Generates a sequence of groovy expressions that filter the vertices to only
      * those that match the specified type.  If GraphPersistenceStrategies.collectTypeInstancesIntoVar()
-     * is set, the vertices are put into a variable whose name is geneated from the specified IntSequence.
-     * The last item in the result will be a graph traversal restricted to only the matching vertices.
+     * is set and the gremlin optimizer is disabled, the vertices are put into a variable whose name is generated
+     * from the specified IntSequence.  The last item in the result will be a graph traversal restricted to only
+     * the matching vertices.
      */
     public List<GroovyExpression> generateTypeTestExpression(GraphPersistenceStrategies s, GroovyExpression parent,
                                                              String typeName, IntSequence intSeq) throws AtlasException {
-        if (s.filterBySubTypes()) {
-            return typeTestExpressionUsingInFilter(s, parent, typeName);
-        } else if (s.collectTypeInstancesIntoVar()) {
-            return typeTestExpressionMultiStep(s, typeName, intSeq);
-        } else {
-            return typeTestExpressionUsingFilter(s, parent, typeName);
+
+        if(AtlasRepositoryConfiguration.isGremlinOptimizerEnabled()) {
+            GroovyExpression superTypeAttributeNameExpr = new LiteralExpression(s.superTypeAttributeName());
+            GroovyExpression typeNameExpr = new LiteralExpression(typeName);
+            GroovyExpression superTypeMatchesExpr = new FunctionCallExpression(TraversalStepType.FILTER, HAS_METHOD, superTypeAttributeNameExpr,
+                    typeNameExpr);
+
+            GroovyExpression typeAttributeNameExpr = new LiteralExpression(s.typeAttributeName());
+
+            GroovyExpression typeMatchesExpr = new FunctionCallExpression(TraversalStepType.FILTER, HAS_METHOD, typeAttributeNameExpr,
+                    typeNameExpr);
+            GroovyExpression result = new FunctionCallExpression(TraversalStepType.FILTER, parent, "or", typeMatchesExpr, superTypeMatchesExpr);
+            return Collections.singletonList(result);
+        }
+        else {
+            if (s.filterBySubTypes()) {
+                return typeTestExpressionUsingInFilter(s, parent, typeName);
+            } else if (s.collectTypeInstancesIntoVar()) {
+                return typeTestExpressionMultiStep(s, typeName, intSeq);
+            } else {
+                return typeTestExpressionUsingFilter(s, parent, typeName);
+            }
         }
     }
 
@@ -285,7 +367,7 @@ public abstract class GremlinExpressionFactory {
         result.add(newSetVar(varName));
         result.add(fillVarWithTypeInstances(s, typeName, varName));
         result.add(fillVarWithSubTypeInstances(s, typeName, varName));
-        result.add(initialExpression(s, varExpr));
+        result.add(initialExpression(varExpr, s));
 
         return result;
     }
@@ -324,7 +406,6 @@ public abstract class GremlinExpressionFactory {
         return Collections.singletonList(filterExpr);
     }
 
-
     /**
      * Generates an expression which checks whether the vertices in the query have
      * a field with the given name.
@@ -334,7 +415,7 @@ public abstract class GremlinExpressionFactory {
      * @return
      */
     public GroovyExpression generateUnaryHasExpression(GroovyExpression parent, String fieldName) {
-        return new FunctionCallExpression(parent, HAS_METHOD, new LiteralExpression(fieldName));
+        return new FunctionCallExpression(TraversalStepType.FILTER, parent, HAS_METHOD, new LiteralExpression(fieldName));
     }
 
     /**
@@ -344,7 +425,7 @@ public abstract class GremlinExpressionFactory {
      * @return
      */
     public GroovyExpression generatePathExpression(GroovyExpression parent) {
-        return new FunctionCallExpression(parent, PATH_METHOD);
+        return new FunctionCallExpression(TraversalStepType.MAP_TO_VALUE, parent, PATH_METHOD);
     }
 
     /**
@@ -365,7 +446,7 @@ public abstract class GremlinExpressionFactory {
      * @return
      */
     public GroovyExpression generateAliasExpression(GroovyExpression parent, String alias) {
-        return new FunctionCallExpression(parent, AS_METHOD, new LiteralExpression(alias));
+        return new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, parent, AS_METHOD, new LiteralExpression(alias));
     }
 
     /**
@@ -377,19 +458,19 @@ public abstract class GremlinExpressionFactory {
      * @return
      */
     public GroovyExpression generateAdjacentVerticesExpression(GroovyExpression parent, AtlasEdgeDirection dir) {
-        return new FunctionCallExpression(parent, getGremlinFunctionName(dir));
+        return new FunctionCallExpression(TraversalStepType.FLAT_MAP_TO_ELEMENTS, parent, getGremlinFunctionName(dir));
     }
 
     private String getGremlinFunctionName(AtlasEdgeDirection dir) {
         switch(dir) {
-            case IN:
-              return "in";
-            case OUT:
-                return "out";
-            case BOTH:
-                return "both";
-            default:
-                throw new RuntimeException("Unknown Atlas Edge Direction: " + dir);
+        case IN:
+            return "in";
+        case OUT:
+            return "out";
+        case BOTH:
+            return "both";
+        default:
+            throw new RuntimeException("Unknown Atlas Edge Direction: " + dir);
         }
     }
 
@@ -403,7 +484,7 @@ public abstract class GremlinExpressionFactory {
      */
     public GroovyExpression generateAdjacentVerticesExpression(GroovyExpression parent, AtlasEdgeDirection dir,
             String label) {
-        return new FunctionCallExpression(parent, getGremlinFunctionName(dir), new LiteralExpression(label));
+        return new FunctionCallExpression(TraversalStepType.FLAT_MAP_TO_ELEMENTS, parent, getGremlinFunctionName(dir), new LiteralExpression(label));
     }
 
     /**
@@ -423,17 +504,19 @@ public abstract class GremlinExpressionFactory {
     }
 
     protected GroovyExpression getAllVerticesExpr() {
-        GroovyExpression gExpr = getGraph();
-        return new FunctionCallExpression(gExpr, V_METHOD);
+        GroovyExpression gExpr = getGraphExpression();
+        return new FunctionCallExpression(TraversalStepType.START, gExpr, V_METHOD);
     }
 
-    protected IdentifierExpression getGraph() {
-        return new IdentifierExpression(G_VARIABLE);
+    protected IdentifierExpression getGraphExpression() {
+        return new IdentifierExpression(TraversalStepType.SOURCE, G_VARIABLE);
     }
 
+
     protected GroovyExpression getCurrentObjectExpression() {
         return new FieldExpression(getItVariable(), OBJECT_FIELD);
     }
+
     //assumes cast already performed
     public GroovyExpression generateCountExpression(GroovyExpression itExpr) {
         GroovyExpression collectionExpr = new CastExpression(itExpr,"Collection");
@@ -454,11 +537,9 @@ public abstract class GremlinExpressionFactory {
 
     private GroovyExpression getAggregrationExpression(GroovyExpression itExpr,
             GroovyExpression mapFunction, String functionName) {
-        GroovyExpression collectionExpr = new CastExpression(itExpr,
-                "Collection");
+        GroovyExpression collectionExpr = new CastExpression(itExpr,"Collection");
         ClosureExpression collectFunction = new ClosureExpression(mapFunction);
-        GroovyExpression transformedList = new FunctionCallExpression(
-                collectionExpr, "collect", collectFunction);
+        GroovyExpression transformedList = new FunctionCallExpression(collectionExpr, "collect", collectFunction);
         return new FunctionCallExpression(transformedList, functionName);
     }
 
@@ -466,5 +547,106 @@ public abstract class GremlinExpressionFactory {
         return getItVariable();
     }
 
+    /**
+     * Specifies the parent to use when translating the select list in
+     * a group by statement.
+     *
+     * @return
+     */
     public abstract GroovyExpression getGroupBySelectFieldParent();
+
+    public GroovyExpression generateFillExpression(GroovyExpression parent, GroovyExpression variable) {
+        return new FunctionCallExpression(TraversalStepType.END,parent , "fill", variable);
+    }
+
+    /**
+     * Generates an anonymous graph traversal initialized with the specified value.  In Gremlin 3, we need
+     * to use  a different syntax for this when the object is a map, so that information needs to be provided
+     * to this method so that the correct syntax is used.
+     *
+     * @param isMap true if the value contains Map instances, false if it contains Vertex instances
+     * @param valueCollection the source objects to start the traversal from.
+     */
+    public abstract GroovyExpression generateSeededTraversalExpresssion(boolean isMap, GroovyExpression valueCollection);
+
+    /**
+     * Returns the current value of the traverser.  This is used when generating closure expressions that
+     * need to operate on the current value in the graph graversal.
+     *
+     * @param traverser
+     * @return
+     */
+    public abstract GroovyExpression getCurrentTraverserObject(GroovyExpression traverser);
+
+    /**
+     * Generates an expression that transforms the current value of the traverser by
+     * applying the function specified
+     *
+     * @param parent
+     * @param closureExpression
+     * @return
+     */
+    public abstract GroovyExpression generateMapExpression(GroovyExpression parent, ClosureExpression closureExpression);
+
+    /**
+     * Returns whether a select statement generates a map (or Gremlin 2 "Row") when it contains the specified
+     * number of aliases.
+     *
+     */
+    public abstract boolean isSelectGeneratesMap(int aliasCount);
+
+    /**
+     * Generates an expression to get the value of the value from the row map
+     * generated by select() with the specified key.
+     *
+     */
+    public abstract GroovyExpression generateGetSelectedValueExpression(LiteralExpression key,
+            GroovyExpression rowMapExpr);
+
+    public GroovyExpression removeExtraMapFromPathInResult(GroovyExpression parent) {
+        GroovyExpression listItem = getItVariable();
+        GroovyExpression tailExpr = new FunctionCallExpression(listItem, "tail");
+        return new FunctionCallExpression(parent, "collect", new ClosureExpression(tailExpr));
+
+    }
+
+    /**
+     * Generates a toList expression to execute the gremlin query and
+     * store the result in a new list.
+     *
+     * @param expr
+     * @return
+     */
+    public GroovyExpression generateToListExpression(GroovyExpression expr) {
+        return new FunctionCallExpression(TraversalStepType.END, expr, "toList");
+    }
+
+    /**
+     * Finds aliases that absolutely must be brought along with this expression into
+     * the output expression and cannot just be recreated there.  For example, in the
+     * Gremlin 2 loop expression, the loop semantics break of the alias is simply recreated
+     * in the output expression.
+     * @param expr
+     * @return
+     */
+    public abstract List<String> getAliasesRequiredByExpression(GroovyExpression expr);
+
+
+    /**
+     * Checks if the given expression is an alias expression, and if so
+     * returns the alias from the expression.  Otherwise, null is
+     * returned.
+     */
+    public String getAliasNameIfRelevant(GroovyExpression expr) {
+        if(!(expr instanceof FunctionCallExpression)) {
+            return null;
+        }
+        FunctionCallExpression fc = (FunctionCallExpression)expr;
+        if(! fc.getFunctionName().equals(AS_METHOD)) {
+            return null;
+        }
+       LiteralExpression aliasName =  (LiteralExpression)fc.getArguments().get(0);
+       return aliasName.getValue().toString();
+
+    }
 }
\ No newline at end of file

repository/src/main/java/org/apache/atlas/gremlin/optimizer/AliasFinder.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.FunctionCallExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.LiteralExpression;
+import org.apache.atlas.groovy.TraversalStepType;
+
+/**
+ * Finds all aliases in the expression.
+ */
+public class AliasFinder implements CallHierarchyVisitor {
+
+    private List<LiteralExpression> foundAliases = new ArrayList<>();
+
+    //Whether a final alias is needed.  A final alias is needed
+    //if there are transformation steps after the last alias in
+    //the expression.  We initialize this to false since a final
+    //alias is not needed if there are no aliases.
+    private boolean finalAliasNeeded = false;
+
+    @Override
+    public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
+        return true;
+    }
+
+    @Override
+    public void visitNonFunctionCaller(GroovyExpression expr) {
+
+    }
+
+    @Override
+    public void visitNullCaller() {
+
+    }
+
+    private static final Set<TraversalStepType> TRANSFORMATION_STEP_TYPES = new HashSet<>(Arrays.asList(
+            TraversalStepType.MAP_TO_ELEMENT,
+            TraversalStepType.MAP_TO_VALUE,
+            TraversalStepType.FLAT_MAP_TO_ELEMENTS,
+            TraversalStepType.FLAT_MAP_TO_VALUES,
+            TraversalStepType.BARRIER,
+            TraversalStepType.NONE));
+
+
+    @Override
+    public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
+
+        if (functionCall instanceof FunctionCallExpression) {
+            FunctionCallExpression expr = (FunctionCallExpression)functionCall;
+            if (expr.getType() == TraversalStepType.SIDE_EFFECT && expr.getFunctionName().equals("as")) {
+                //We found an alias.  This is currently the last expression we've seen
+                //in our traversal back up the expression tree, so at this point a final
+                //alias is not needed.
+                LiteralExpression aliasNameExpr = (LiteralExpression)expr.getArguments().get(0);
+                foundAliases.add(aliasNameExpr);
+                finalAliasNeeded=false;
+            }
+        }
+
+        if(TRANSFORMATION_STEP_TYPES.contains(functionCall.getType())) {
+            //This step changes the value of the traverser.  Now, a final alias
+            //needs to be added.
+            if(!foundAliases.isEmpty()) {
+                finalAliasNeeded = true;
+            }
+        }
+
+        return true;
+    }
+
+    public List<LiteralExpression> getAliases() {
+        return foundAliases;
+    }
+
+    public boolean isFinalAliasNeeded() {
+
+        return finalAliasNeeded;
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/CallHierarchyVisitor.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+
+/**
+ * Call back interface for visiting the call hierarchy of a function call.
+ */
+public interface CallHierarchyVisitor {
+
+    /**
+     * Visits a function expression before the visit to its caller.
+     *
+     * @param expr
+     *
+     * @return false to terminate the recursion
+     */
+    boolean preVisitFunctionCaller(AbstractFunctionExpression expr);
+
+    /**
+     * Called when a caller that is not an instance of
+     * AbstractFunctionExpression is found. This indicates that the deepest
+     * point in the call hierarchy has been reached.
+     *
+     *
+     */
+    void visitNonFunctionCaller(GroovyExpression expr);
+
+    /**
+     * Called when a null caller is found (this happens for static/user-defined
+     * functions). This indicates that the deepest point in the call hierarchy
+     * has been reached.
+     *
+     */
+    void visitNullCaller();
+
+    /**
+     * Visits a function expression after the visit to its caller.
+     *
+     * @param expr
+     *
+     * @return false to terminate the recursion
+     */
+    boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall);
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/ExpandAndsOptimization.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.FunctionCallExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Optimizer that pulls has expressions out of an 'and' expression.
+ *
+ * For example:
+ *
+ * g.V().and(has('x'),has('y')
+ *
+ * is optimized to:
+ *
+ * g.V().has('x').has('y')
+ *
+ * There are certain cases where it is not safe to move an expression out
+ * of the 'and'.  For example, in the expression
+ *
+ * g.V().and(has('x').out('y'),has('z'))
+ *
+ * has('x').out('y') cannot be moved out of the 'and', since it changes the value of the traverser.
+ *
+ * At this time, the ExpandAndsOptimizer is not able to handle this scenario, so we don't extract
+ * that expression.  In this case, the result is:
+ *
+ * g.V().has('z').and(has('x').out('y'))
+ *
+ * The optimizer will call ExpandAndsOptimization recursively on the children, so
+ * there is no need to recursively update the children here.
+ *
+ * @param expr
+ * @param context
+ * @return the expressions that should be unioned together to get the query result
+ */
+public class ExpandAndsOptimization implements GremlinOptimization {
+
+    private static final Logger logger_ = LoggerFactory.getLogger(ExpandAndsOptimization.class);
+
+
+    private final GremlinExpressionFactory factory;
+
+    public ExpandAndsOptimization(GremlinExpressionFactory factory) {
+        this.factory = factory;
+    }
+
+    @Override
+    public boolean appliesTo(GroovyExpression expr, OptimizationContext contxt) {
+        return expr instanceof FunctionCallExpression && ((FunctionCallExpression)expr).getFunctionName().equals("and");
+    }
+
+    /**
+     * Expands the given and expression.  There is no need to recursively
+     * expand the children here.  This method is called recursively by
+     * GremlinQueryOptimier on the children.
+     *
+     */
+    @Override
+    public GroovyExpression apply(GroovyExpression expr, OptimizationContext context) {
+
+        FunctionCallExpression exprAsFunction = (FunctionCallExpression)expr;
+        GroovyExpression result = exprAsFunction.getCaller();
+
+        List<GroovyExpression> nonExtractableArguments = new ArrayList<>();
+        for(GroovyExpression argument : exprAsFunction.getArguments()) {
+
+            if (GremlinQueryOptimizer.isExtractable(argument)) {
+                //Set the caller of the deepest expression in the call hierarchy
+                //of the argument to point to the current result.
+                //For example, if result is "g.V()" and the updatedArgument is "has('x').has('y')",
+                //updatedArgument would be a tree like this:
+                //
+                //                  has('y')
+                //                 /
+                //                / caller
+                //              |/_
+                //           has('x')
+                //             /
+                //            / caller
+                //          |/_
+                //        (null)
+                //
+                //We would set the caller of has('x') to be g.V(), so result would become g.V().has('x').has('y').
+                //
+                // Note: This operation is currently done by making a copy of the argument tree.  That should
+                // be changed.
+                result = GremlinQueryOptimizer.copyWithNewLeafNode(
+                        (AbstractFunctionExpression) argument, result);
+            } else {
+                logger_.warn("Found non-extractable argument '{}' in the 'and' expression '{}'",argument.toString(), expr.toString());
+                nonExtractableArguments.add(argument);
+            }
+        }
+
+        if (!nonExtractableArguments.isEmpty()) {
+            //add a final 'and' call with the arguments that could not be extracted
+            result =  factory.generateLogicalExpression(result, "and", nonExtractableArguments);
+        }
+        return result;
+    }
+
+    @Override
+    public boolean isApplyRecursively() {
+        return true;
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/ExpandOrsOptimization.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.ClosureExpression;
+import org.apache.atlas.groovy.FunctionCallExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.LiteralExpression;
+import org.apache.atlas.groovy.StatementListExpression;
+import org.apache.atlas.groovy.TraversalStepType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+
+
+/**
+ * Optimization that removes 'or' expressions from a graph traversal when possible
+ * and replaces them with separate calls that are combined using a logical union operation.
+ * Unfortunately, Titan does not use indices when executing the child graph traversals associated
+ * with an 'or' call.  In order to make the index be used, we split queries with
+ * or expressions into multiple queries.  These queries are executed individually,
+ * using indices, and then the results are combined back together.  Here is a
+ * simple example to illustrate this:
+ *
+ * <h4>Original Query</h4>
+ *
+ * <pre>
+ * g.V().or(has('name','Fred'),has('age','17'))
+ * </pre>
+ *
+ *<h4>Optimized Query</h4>
+ *
+ * <pre>
+ * def r = [] as Set;
+ * g.V().has('name','Fred').fill(r);
+ * g.V().has('age','17').fill(r);
+ * r;
+ * </pre>
+ *
+ * Here, we introduce an intermediate variable "r" which is declared as a Set.  The Set is performing
+ * the union for us.  If there are vertices that happen to both have "Fred" as the name and "17" as the age,
+ * the Set will prevent the second query execution from adding a duplicate vertex to the result.  Recall that
+ * in Groovy scripts, the last expression is the one that will be returned back to the caller.  We refer to
+ * that expression is the "result expression".  For this example, the result expression is simply "r", which
+ * contains the vertices that matched the query.
+ * <p/>
+ * If the query does any kind of transformation of the vertices to produce the query result, that needs
+ * to be done in the result expression.  To understand why that is, let's take a look at another example:
+ *
+ * <h4>Original Query</h4>
+ *
+ * <pre>
+ * g.V().or(has('name','Fred'),has('age','17')).as('person').select('person').by('gender')
+ * </pre>
+ *
+ * <h4>Incorrect Optimized Query</h4>
+ *
+ * <pre>
+ * def r = [] as Set;
+ * g.V().has('name','Fred').as('person').select('person').by('gender').fill(r)
+ * g.V().has('age','17').as('person').select('person').by('gender').fill(r)
+ * r;
+ * </pre>
+ *
+ * The problem with this query is that now 'r' contains Strings (the gender of the person).  Suppose
+ * that there is one person named Fred and there are 3 people whose age is 17 (let's say Fred's age is 16).
+ * The original query would have produced 4 rows, one corresponding to each of those people.  The new
+ * query would produce at most 2 rows - one for 'male' and one for 'female'.  This is happening because
+ * we are now performing the union on the Strings, not on the vertices.  To fix this, we need to split
+ * the original query and put the end portion into the result expression:
+ *
+ * <h4>Correct Optimized Query</h4>
+ *
+ * <pre>
+ * def r = [] as Set;
+ * g.V().has('name','Fred').fill(r)
+ * g.V().has('age','17').fill(r)
+ * __.inject(r as Object[]).as('person').select('person').by('gender')
+ * </pre>
+ *
+ * The logic for doing this splitting is described in more detail in
+ * {@link #moveTransformationsToResultExpression(GroovyExpression, OptimizationContext)}.
+ * <p/>
+ * There is one more problematic case that this optimizer is able to handle.  Let's look at the following example:
+ *
+ * <h4>Original Query</h4>
+ *
+ * <pre>
+ * g.V().or(has('type','Person'),has('superType','Person')).as('x').has('qualifiedName','Fred').as('y').select('x','y').by('name').by('name')
+ * </pre>
+ *
+ * Queries of this form appear often when translating DSL queries.
+ *
+ * If we were to optimize this query using the logic described above, we would get something like this:
+ *
+ * <h4>Incorrect Optimized Query</h4>
+ *
+ * <pre>
+ * def r = [] as Set;
+ * g.V().has('type','Person').fill(r);
+ * g.V().has('superType','Person').fill(r);
+ * __.inject(r as Object[]).as('x').has('qualifiedName','Fred').as('y').select('x','y');
+ * </pre>
+ *
+ * While not strictly incorrect, this query will not perform well since the index on qualifiedName will
+ * not be used.  In order for that index to be used, the 'has' expression needs to be part of the original
+ * query.  However, if we do that alone, the query will be broken, since the select
+ * will now refer to an undefined label:
+ *
+ * <h4>Incorrect Optimized Query</h4>
+ *
+ * <pre>
+ * def r = [] as Set;
+ * g.V().has('type','Person').as('x').has('qualifiedName','Fred').fill(r);
+ * g.V().has('superType','Person').as('x').has('qualifiedName','Fred').fill(r);
+ * __.inject(r as Object[]).as('y').select('x','y')
+ * </pre>
+ *
+ * To fix this, we need to save the values of the aliased vertices in the original
+ * query, and create labels in the result expression that refer to them.  We do this
+ * as follows:
+ *
+ * <h4>Correct Optimized Query</h4>
+ *
+ * <pre>
+ * def r = [] as Set;
+ * g.V().has('type','Person').as('x').has('qualifiedName','Fred').as('y').select('x','y').fill(r);
+ * g.V().has('superType','Person').as('x').has('qualifiedName','Fred').select('x','y').fill(r);
+ * __.inject(r as Object[]).as('__tmp').map({((Map)it.get()).get('x')}).as('x').select('__tmp').map({((Map)it.get()).get('x')}).as('y').select('x','y').by('name').by('name')
+ * </pre>
+ *
+ * This is not pretty, but is the best solution we've found so far for supporting expressions that contain aliases in this optimization.
+ * What ends up happening is that r gets populated with alias->Vertex maps.  In the result expression, we make 'x' point
+ * to a step where the value in the traverser is the vertex for 'x', and we do the same thing for y.  The <code>select('_tmp')</code> step in the middle restores the value of
+ * the traverser back to the map.
+ * <p/>
+ * The one known issue with the alias rearrangement is that it breaks loop expressions.  As a result, expressions containing loops are currently excluded
+ * from this optimization.
+ *
+ * ExpandOrsOptimization expands the entire expression tree recursively, so it is not invoked
+ * recursively by GremlinQueryOptimizer.
+ *
+ */
+public class ExpandOrsOptimization implements GremlinOptimization {
+
+    private static final Logger logger_ = LoggerFactory.getLogger(ExpandOrsOptimization.class);
+
+    private final GremlinExpressionFactory factory;
+
+    public ExpandOrsOptimization(GremlinExpressionFactory factory) {
+        this.factory = factory;
+    }
+
+    @Override
+    public boolean appliesTo(GroovyExpression expr, OptimizationContext contxt) {
+
+        ExpressionFinder finder = new ExpressionFinder(IsOr.INSTANCE);
+        GremlinQueryOptimizer.visitCallHierarchy(expr, finder);
+        return finder.isExpressionFound();
+    }
+
+    @Override
+    public GroovyExpression apply(GroovyExpression expr, OptimizationContext context) {
+
+        setupRangeOptimization(expr, context);
+        GroovyExpression traveralExpression = moveTransformationsToResultExpression(expr, context);
+
+        FunctionGenerator functionGenerator = new FunctionGenerator(factory, context);
+        GremlinQueryOptimizer.visitCallHierarchy(traveralExpression, functionGenerator);
+        traveralExpression = functionGenerator.getNewRootExpression();
+        List<GroovyExpression> bodyExpressions = expandOrs(traveralExpression, context);
+
+
+        //Adds a statement to define the result variable 'v' in the
+        //groovy script.  The variable is declared as a Set.  The type
+        //of the objects in the Set depend on the number of aliases in the Groovy
+        // expression:
+        //   - 0 or 1 alias : Vertex
+        //   - multiple aliases: Map<String,Vertex>
+        StatementListExpression result = new StatementListExpression();
+        context.prependStatement(context.getDefineResultVariableStmt());
+
+
+        for (GroovyExpression bodyExpression : bodyExpressions) {
+            result.addStatement(bodyExpression);
+        }
+        result.addStatement(context.getResultExpression());
+        return result;
+    }
+
+    private void setupRangeOptimization(GroovyExpression expr, OptimizationContext context) {
+
+        // Find any range expressions in the expression tree.
+        RangeFinder rangeFinder = new RangeFinder(factory);
+        GremlinQueryOptimizer.visitCallHierarchy(expr, rangeFinder);
+        List<AbstractFunctionExpression> rangeExpressions = rangeFinder.getRangeExpressions();
+        if (rangeExpressions.size() == 1) {
+            OrderFinder orderFinder = new OrderFinder(factory);
+            GremlinQueryOptimizer.visitCallHierarchy(expr, orderFinder);
+            if (!orderFinder.hasOrderExpression()) {
+                // If there is one range expression and no order expression in the unoptimized gremlin,
+                // save the range parameters to use for adding a range expression to
+                // each expanded "or" expression result, such that it will only contain the specified range of vertices.
+                // For now, apply this optimization only if the range start index is zero.
+                AbstractFunctionExpression rangeExpression = rangeExpressions.get(0);
+                int[] rangeParameters = factory.getRangeParameters(rangeExpression);
+                if (rangeParameters[0] == 0) {
+                    context.setRangeExpression(rangeExpression);
+                }
+            }
+        }
+    }
+
+    private GroovyExpression moveTransformationsToResultExpression(GroovyExpression expr, OptimizationContext context) {
+        GroovyExpression traveralExpression = expr;
+
+        // Determine the 'split point'.  This is the expression that will become
+        // the deepest function call in the result expression.  If a split
+        // point is found, its caller is changed.  The new caller is
+        // set to the graph traversal expression in the result expression.
+        // The original caller becomes the new traversal expression that
+        // will be carried through the rest of the 'or' expansion processing.
+        //
+        // Example: g.V().has('x').as('x').select('x')
+        // Here, select('x') is the split expression
+        // so :
+        // 1) the result expression in OptimizationContext becomes [base result expression].select('x')
+        // 2) we return g.V().has('x').as('x')
+
+        SplitPointFinder finder = new SplitPointFinder(factory);
+        GremlinQueryOptimizer.visitCallHierarchy(traveralExpression, finder);
+        AbstractFunctionExpression splitPoint = finder.getSplitPoint();
+
+
+        List<LiteralExpression> aliases = new ArrayList<>();
+
+        //If we're not splitting the query, there is no need to save/restore
+        //the aliases.
+        if(splitPoint != null) {
+
+            traveralExpression = splitPoint.getCaller();
+
+            AliasFinder aliasFinder = new AliasFinder();
+            GremlinQueryOptimizer.visitCallHierarchy(traveralExpression, aliasFinder);
+            aliases.addAll(aliasFinder.getAliases());
+            if(aliasFinder.isFinalAliasNeeded()) {
+                //The last alias in the expression does not capture the final vertex in the traverser,
+                //so we need to create an alias to record that.
+                traveralExpression = factory.generateAliasExpression(traveralExpression, context.getFinalAliasName());
+                aliases.add(new LiteralExpression(context.getFinalAliasName()));
+            }
+
+            GroovyExpression resultExpr = getBaseResultExpression(context, aliases);
+            splitPoint.setCaller(resultExpr);
+            expr = removeMapFromPathsIfNeeded(expr, aliases);
+            context.setResultExpression(expr);
+        }
+
+        //Add expression(s) to the end of the traversal expression to add the vertices
+        //that were found into the intermediate variable ('r')
+        traveralExpression = addCallToUpdateResultVariable(traveralExpression, aliases, context);
+        return traveralExpression;
+    }
+
+    private GroovyExpression removeMapFromPathsIfNeeded(GroovyExpression expr, List<LiteralExpression> aliases) {
+        if(aliases.size() > 0 && factory.isSelectGeneratesMap(aliases.size())) {
+            PathExpressionFinder pathExprFinder = new PathExpressionFinder();
+            GremlinQueryOptimizer.visitCallHierarchy(expr, pathExprFinder);
+            boolean hasPath = pathExprFinder.isPathExpressionFound();
+            if(hasPath) {
+                //the path will now start with the map that we added.  That is an artifact
+                //of the optimization process and must be removed.
+                if(expr.getType() != TraversalStepType.END && expr.getType() != TraversalStepType.NONE) {
+                    //we're still in the pipeline, need to execute the query before we can
+                    //modify the result
+                    expr = factory.generateToListExpression(expr);
+                }
+                expr = factory.removeExtraMapFromPathInResult(expr);
+            }
+
+        }
+        return expr;
+    }
+
+    /**
+     * This method adds steps to the end of the initial traversal to add the vertices
+     * that were found into an intermediate variable (defined as a Set).  If there is one alias,
+     * this set will contain the vertices associated with that Alias.  If there are multiple
+     * aliases, the values in the set will be alias->vertex maps that have the vertex
+     * associated with the alias for each result.
+
+     * @param expr
+     * @param aliasNames
+     * @param context
+     * @return
+     */
+    private GroovyExpression addCallToUpdateResultVariable(GroovyExpression expr,List<LiteralExpression> aliasNames, OptimizationContext context) {
+
+        GroovyExpression result = expr;
+        // If there is one range expression in the unoptimized gremlin,
+        // add a range expression here so that the intermediate variable will only contain
+        // the specified range of vertices.
+        AbstractFunctionExpression rangeExpression = context.getRangeExpression();
+        if (rangeExpression != null) {
+            int[] rangeParameters = factory.getRangeParameters(rangeExpression);
+            result = factory.generateRangeExpression(result, rangeParameters[0], rangeParameters[1]);
+        }
+        if( ! aliasNames.isEmpty()) {
+            result = factory.generateSelectExpression(result, aliasNames, Collections.<GroovyExpression>emptyList());
+        }
+        return factory.generateFillExpression(result, context.getResultVariable());
+    }
+
+    /**
+     * Recursively traverses the given expression, expanding or expressions
+     * wherever they are found.
+     *
+     * @param expr
+     * @param context
+     * @return expressions that should be unioned together to get the query result
+     */
+    private List<GroovyExpression> expandOrs(GroovyExpression expr, OptimizationContext context) {
+
+        if (GremlinQueryOptimizer.isOrExpression(expr)) {
+            return expandOrFunction(expr, context);
+        }
+        return processOtherExpression(expr, context);
+    }
+
+    /**
+     * This method takes an 'or' expression and expands it into multiple expressions.
+     *
+     * For example:
+     *
+     * g.V().or(has('x'),has('y')
+     *
+     * is expanded to:
+     *
+     * g.V().has('x')
+     * g.V().has('y')
+     *
+     * There are certain cases where it is not safe to move an expression out
+     * of the 'or'.  For example, in the expression
+     *
+     * g.V().or(has('x').out('y'),has('z'))
+     *
+     * has('x').out('y') cannot be moved out of the 'or', since it changes the value of the traverser.
+     *
+     * At this time, the ExpandOrsOptimizer is not able to handle this scenario, so we don't remove
+     * that expression.  In cases like this, a final expression is created that ors together
+     * all of the expressions that could not be extracted.  In this case that would be:
+     *
+     * g.V().has('z')
+     * g.V().or(has('y').out('z'))
+     *
+     * This processing is done recursively.
+     *
+     *
+     * @param expr
+     * @param context
+     * @return the expressions that should be unioned together to get the query result
+     */
+    private List<GroovyExpression> expandOrFunction(GroovyExpression expr, OptimizationContext context) {
+        FunctionCallExpression functionCall = (FunctionCallExpression) expr;
+        GroovyExpression caller = functionCall.getCaller();
+        List<GroovyExpression> updatedCallers = null;
+        if (caller != null) {
+            updatedCallers = expandOrs(caller, context);
+        } else {
+            updatedCallers = Collections.singletonList(null);
+        }
+        UpdatedExpressions newArguments = getUpdatedChildren(functionCall.getArguments(), context);
+        List<GroovyExpression> allUpdatedArguments = new ArrayList<>();
+        for (List<GroovyExpression> exprs : newArguments.getUpdatedChildren()) {
+            allUpdatedArguments.addAll(exprs);
+        }
+        List<AbstractFunctionExpression> extractableArguments = new ArrayList<>();
+        List<GroovyExpression> nonExtractableArguments = new ArrayList<>();
+        for (GroovyExpression argument : allUpdatedArguments) {
+
+            if (GremlinQueryOptimizer.isExtractable(argument)) {
+                extractableArguments.add((AbstractFunctionExpression) argument);
+            } else {
+                logger_.warn("Found non-extractable argument '{}; in the 'or' expression '{}'",argument.toString(), expr.toString());
+                nonExtractableArguments.add(argument);
+            }
+        }
+
+        List<GroovyExpression> result = new ArrayList<>();
+        for (GroovyExpression updatedCaller : updatedCallers) {
+
+            for (AbstractFunctionExpression arg : extractableArguments) {
+                GroovyExpression updated = GremlinQueryOptimizer.copyWithNewLeafNode(arg, updatedCaller);
+                result.add(updated);
+            }
+            if (!nonExtractableArguments.isEmpty()) {
+                result.add(factory.generateLogicalExpression(updatedCaller, "or", nonExtractableArguments));
+            }
+
+        }
+        return result;
+    }
+
+    private UpdatedExpressions getUpdatedChildren(List<GroovyExpression> children, OptimizationContext context) {
+        List<List<GroovyExpression>> updatedChildren = new ArrayList<>();
+        boolean changed = false;
+        for (GroovyExpression child : children) {
+            List<GroovyExpression> childChoices = expandOrs(child, context);
+            if (childChoices.size() != 1 || childChoices.iterator().next() != child) {
+                changed = true;
+            }
+            updatedChildren.add(childChoices);
+        }
+        return new UpdatedExpressions(changed, updatedChildren);
+    }
+
+    private UpdatedExpressions getUpdatedChildren(GroovyExpression expr, OptimizationContext context) {
+        return getUpdatedChildren(expr.getChildren(), context);
+    }
+
+    /**
+     * This is called when we encounter an expression that is not an "or", for example an "and" expressio.  For these
+     * expressions, we process the children and create copies with the cartesian product of the updated
+     * arguments.
+     *
+     * Example:
+     *
+     * g.V().and(or(has('x),has('y'), or(has('a'),has('b')))
+     *
+     * Here, we have an "and" expression with two children:
+     *
+     * 1) or(has('x),has('y')
+     * 2) or(has('a'),has('b'))
+     *
+     * We first process these children.  They each yield 2 expressions:
+     *
+     * 1 -> [ has('x'), has('y') ]
+     * 2 -> [ has('a'), has('b') ]
+     *
+     * The cartesian product of these gives this:
+     *
+     *    [ has('x'), has('a') ]
+     *    [ has('x'), has('b') ]
+     *    [ has('y'), has('a') ]
+     *    [ has('y'), has('b') ]
+     *
+     * So the overall result is:
+     *
+     * g.V().and(has('x'), has('a'))
+     * g.V().and(has('x'), has('b'))
+     * g.V().and(has('y'), has('a'))
+     * g.V().and(has('y'), has('b'))
+     *
+     *
+     * @param source
+     * @param context
+     * @return expressions that should be unioned together to get the query result
+     */
+    private List<GroovyExpression> processOtherExpression(GroovyExpression source, OptimizationContext context) {
+        UpdatedExpressions updatedChildren = getUpdatedChildren(source, context);
+        if (!updatedChildren.hasChanges()) {
+            return Collections.singletonList(source);
+        }
+        List<GroovyExpression> result = new ArrayList<GroovyExpression>();
+
+        //The updated children list we get back has the possible values for each child
+        //in the expression.  We compute a cartesian product to get all possible
+        //combinations of child values.
+        List<List<GroovyExpression>> updateChildLists = Lists.cartesianProduct(updatedChildren.getUpdatedChildren());
+
+        for (List<GroovyExpression> updatedChildList : updateChildLists) {
+            result.add(source.copy(updatedChildList));
+        }
+        return result;
+    }
+
+    @Override
+    public boolean isApplyRecursively() {
+        return false;
+    }
+
+    /**
+     *
+     * This method creates a base result expression that recreates the state of the
+     * graph traverser at start of the result expression to what it would have been
+     * if we had been executing one Gremlin query (instead of many and doing a union).
+     *
+     * To do this, we start with an anonymous graph traversal that will iterate
+     * through the values in the intermediate Set that was created.  We then need
+     * to set things up so that the aliases that were in the original gremlin query
+     * refer to steps with the correct traverser value.
+     *
+     * The way we do this depends on the number of aliases.  If there are 0 or 1 alias,
+     * the intermediate variable already contains Vertices, so we just create the alias.
+     *
+     * If there are multiple aliases, the intermediate variable contains a String->Vertex
+     * map.  We first create a temporary alias that refers to that map.  For each alias,
+     * we use a MapStep to map the map to the Vertex for that alias.  We then add back
+     * the alias, making it refer to the MapStep.  Between the alias restorations, we restore the
+     * traverser object back to the map.
+     *
+     * @param context
+     * @param aliases
+     * @return
+     */
+    private GroovyExpression getBaseResultExpression(OptimizationContext context,
+                                                            List<LiteralExpression> aliases) {
+
+        //Start with an anonymous traversal that gets its objects from the intermediate result variable.
+        GroovyExpression parent = factory.generateSeededTraversalExpresssion(aliases.size() > 1, context.getResultVariable());
+
+        if(aliases.isEmpty()) {
+            return parent;
+        }
+
+        //The expression we will return.
+        GroovyExpression result = parent;
+
+        //We use a temporary alias to save/restore the original value of the traverser
+        //at the start of the query.  We do this so we can set the value of the traverser
+        //back to being the map after we retrieve each alias.  If there is only one
+        //alias, the save/restore is not needed, so there is no need to create this alias.
+        if(aliases.size() > 1) {
+
+            result = factory.generateAliasExpression(result, context.getTempAliasName());
+        }
+
+        Iterator<LiteralExpression> it = aliases.iterator();
+        while(it.hasNext()) {
+            LiteralExpression curAlias = it.next();
+            //A map is only generated by Gremlin when there is more than one alias.  When there is only one
+            //alias, the intermediate variable will directly contain the vertices.`
+            if(factory.isSelectGeneratesMap(aliases.size())) {
+                //Since there is more than one alias, the current traverser object is an alias->vertex
+                //map.  We use a MapStep to map that map to the Vertex for the current alias.  This sets
+                //the current traverser object to that Vertex.  We do this by defining the closure we
+                //pass to the MapStep call [map].get(aliasName) where [map] is the expression
+                //that refers to the map.
+
+                GroovyExpression rowMapExpr = factory.getCurrentTraverserObject(factory.getClosureArgumentValue());
+                GroovyExpression getExpr = factory.generateGetSelectedValueExpression(curAlias, rowMapExpr);
+                result = factory.generateMapExpression(result, new ClosureExpression(getExpr));
+            }
+
+            //Create alias that points to the previous step.  The traverser value at that step
+            //is the Vertex associated with this alias.
+            result = factory.generateAliasExpression(result, curAlias.getValue().toString());
+            if(it.hasNext()) {
+                //Restore the current value of the traverser back to the current alias->vertex map
+                result = factory.generateBackReferenceExpression(result, false, context.getTempAliasName());
+            }
+        }
+        return result;
+    }
+
+
+
+
+}
+

repository/src/main/java/org/apache/atlas/gremlin/optimizer/ExpressionFinder.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import com.google.common.base.Function;
+
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+
+/**
+ * Call hierarchy visitor that checks if an expression
+ * matching the specified criteria is present
+ * in the call hierarch.
+ */
+public class ExpressionFinder implements CallHierarchyVisitor {
+
+    private final Function<GroovyExpression, Boolean> predicate;
+    private boolean expressionFound = false;
+
+    public ExpressionFinder(Function<GroovyExpression, Boolean> predicate) {
+        this.predicate = predicate;
+    }
+    @Override
+    public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
+        if (predicate.apply(expr)) {
+            expressionFound = true;
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void visitNonFunctionCaller(GroovyExpression expr) {
+        if (predicate.apply(expr)) {
+            expressionFound = true;
+        }
+    }
+
+    @Override
+    public void visitNullCaller() {
+        //nothing to do
+    }
+
+    @Override
+    public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
+        //nothing to do
+        return true;
+    }
+
+    public boolean isExpressionFound() {
+        return expressionFound;
+    }
+
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/FunctionGenerator.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.List;
+
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.ClosureExpression;
+import org.apache.atlas.groovy.ClosureExpression.VariableDeclaration;
+import org.apache.atlas.groovy.FunctionCallExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.IdentifierExpression;
+
+/**
+ * Extracts common expressions from an or-containing expression
+ * into functions.  These expressions would otherwise be duplicated
+ * as part of expanding the "or".  Doing this shortens the overall length
+ * of the Gremlin script so we can maximize query performance.
+ *
+ */
+public class FunctionGenerator implements CallHierarchyVisitor {
+
+    //Function length constants.
+    //These assume we won't reach more than 9 function definition.  Even if we do, this is still
+    //a reasonable approximation.
+    private static final int INITIAL_FUNCTION_DEF_LENGTH = "def f1={};".length();
+    private final int functionDefLength;
+    private static final int FUNCTION_CALL_OVERHEAD = "f1()".length();
+
+    /**
+     * The expression that should be the first (deepest) expression
+     * in the body of the next generated function.  As we go up the
+     * expression tree in the post visit, this is updated based on the
+     * expressions we see.  During the post visits, if it is null,
+     * the body expression is set to the expression we're visiting.
+     * As we go up the tree, it is nulled out if we create a function
+     * or encounter an or expression.  This guarantees that the
+     * next function body will not contain any or expressions
+     * and that it will not have expressions that are already
+     * part of some other function.
+     */
+    private GroovyExpression nextFunctionBodyStart;
+
+    /**
+     * The number of times expressions will be duplicated.
+     */
+    private int scaleFactor = 1;
+
+    private final OptimizationContext context;
+
+    /**
+     * The current depth in the expression tree.
+     */
+    private int depth = 0;
+
+    /**
+     * The name of the last function that was generated.  If set,
+     * we can safely update this function instead of creating a new one.
+     */
+    private String currentFunctionName;
+
+    /**
+     * The updated expression we will pass back to the caller.
+     */
+    private GroovyExpression newRootExpression;
+
+    private final GremlinExpressionFactory factory;
+
+    public FunctionGenerator(GremlinExpressionFactory factory, OptimizationContext context) {
+        this.context = context;
+        this.factory = factory;
+        functionDefLength = ("def f1={" + factory.getTraversalExpressionClass() + " x->};").length();
+    }
+
+    @Override
+    public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
+        depth++;
+        if (IsOr.INSTANCE.apply(expr)) {
+            FunctionCallExpression functionCall = (FunctionCallExpression) expr;
+            scaleFactor *= functionCall.getArguments().size();
+        }
+        if (newRootExpression == null) {
+            newRootExpression = expr;
+        }
+
+        return true;
+    }
+
+    @Override
+    public void visitNonFunctionCaller(GroovyExpression expr) {
+        if (nextFunctionBodyStart == null) {
+            nextFunctionBodyStart = expr;
+        }
+
+    }
+
+    @Override
+    public void visitNullCaller() {
+        //nothing to do
+    }
+
+    @Override
+    public boolean postVisitFunctionCaller(AbstractFunctionExpression expr) {
+        boolean isRootExpr = depth == 1;
+        visitParentExpression(expr);
+
+        //The root expression has no parent.  To simplify the logic, we create
+        //a dummy expression so it does have a parent, then call visitParentExpression again
+        //to examine the root expression.
+        if (isRootExpr) {
+            FunctionCallExpression dummyParent = new FunctionCallExpression(expr, "dummy");
+            visitParentExpression(dummyParent);
+            newRootExpression = dummyParent.getCaller();
+        }
+
+        depth--;
+        return true;
+    }
+
+    /**
+     * Checks to see if the *caller* of this expression should become part
+     * of a function.  If so, either a new function is created, or the
+     * expression becomes part of the last function we created.
+     *
+     * @param parentExpr
+     */
+    private void visitParentExpression(AbstractFunctionExpression parentExpr) {
+
+        if (nextFunctionBodyStart == null) {
+            nextFunctionBodyStart = parentExpr;
+        }
+
+        if (currentFunctionName != null) {
+            updateCurrentFunction(parentExpr);
+        } else {
+            createFunctionIfNeeded(parentExpr);
+        }
+
+        if (GremlinQueryOptimizer.isOrExpression(parentExpr)) {
+            //reset
+            currentFunctionName = null;
+            //don't include 'or' in generated functions
+            nextFunctionBodyStart = null;
+        }
+
+    }
+
+    /**
+     * Creates a function whose body goes from the child of parentExpr
+     * up to (and including) the functionBodyEndExpr.
+     * @param parentExpr
+     */
+    private void createFunctionIfNeeded(AbstractFunctionExpression parentExpr) {
+        GroovyExpression potentialFunctionBody = parentExpr.getCaller();
+
+        if (creatingFunctionShortensGremlin(potentialFunctionBody)) {
+            GroovyExpression functionCall = null;
+
+            if (nextFunctionBodyStart instanceof AbstractFunctionExpression) {
+                //The function body start is a a function call.  In this
+                //case, we generate a function that takes one argument, which
+                //is a graph traversal.  We have an expression tree that
+                //looks kind of like the following:
+                //
+                //                     parentExpr
+                //                       /
+                //                      / caller
+                //                    |/_
+                //           potentialFunctionBody
+                //                   /
+                //                  / caller
+                //                |/_
+                //               ...
+                //               /
+                //              / caller
+                //            |/_
+                //    nextFunctionBodyStart
+                //           /
+                //          / caller
+                //        |/_
+                //    oldCaller
+                //
+                //
+                // Note that potentialFunctionBody and nextFunctionBodyStart
+                // could be the same expression.  Let's say that the next
+                // function name is f1
+                //
+                // We reshuffle these expressions to the following:
+                //
+                //                     parentExpr
+                //                       /
+                //                      / caller
+                //                    |/_
+                //                f1(oldCaller)
+                //
+                //
+                //           potentialFunctionBody   <- body of new function "f1(GraphTraversal x)"
+                //                   /
+                //                  / caller
+                //                |/_
+                //               ...
+                //               /
+                //              / caller
+                //            |/_
+                //    nextFunctionBodyStart
+                //           /
+                //          / caller
+                //        |/_
+                //        x
+                //
+                // As an example, suppose parentExpr is g.V().or(x,y).has(a).has(b).has(c)
+                // where has(a) is nextFunctionBodyStart.
+                //
+                // We generate a function f1 = { GraphTraversal x -> x.has(a).has(b) }
+                // parentExpr would become : f1(g.V().or(x,y)).has(c)
+
+                AbstractFunctionExpression nextFunctionBodyStartFunction=
+                        (AbstractFunctionExpression) nextFunctionBodyStart;
+                String variableName = "x";
+                IdentifierExpression var = new IdentifierExpression(variableName);
+                GroovyExpression oldCaller = nextFunctionBodyStartFunction.getCaller();
+                nextFunctionBodyStartFunction.setCaller(var);
+
+                currentFunctionName = context.addFunctionDefinition(new VariableDeclaration(factory.getTraversalExpressionClass(), "x"),
+                        potentialFunctionBody);
+                functionCall = new FunctionCallExpression(potentialFunctionBody.getType(),
+                        currentFunctionName, oldCaller);
+
+            } else {
+                //The function body start is a not a function call.  In this
+                //case, we generate a function that takes no arguments.
+
+                // As an example, suppose parentExpr is g.V().has(a).has(b).has(c)
+                // where g is nextFunctionBodyStart.
+                //
+                // We generate a function f1 = { g.V().has(a).has(b) }
+                // parentExpr would become : f1().has(c)
+
+                currentFunctionName = context.addFunctionDefinition(null, potentialFunctionBody);
+                functionCall = new FunctionCallExpression(potentialFunctionBody.getType(), currentFunctionName);
+            }
+
+            //functionBodyEnd is now part of a function definition, don't propagate it
+            nextFunctionBodyStart = null;
+            parentExpr.setCaller(functionCall);
+        }
+    }
+
+    /**
+     * Adds the caller of parentExpr to the current body of the last
+     * function that was created.
+     *
+     * @param parentExpr
+     */
+    private void updateCurrentFunction(AbstractFunctionExpression parentExpr) {
+        GroovyExpression expr = parentExpr.getCaller();
+        if (expr instanceof AbstractFunctionExpression) {
+            AbstractFunctionExpression exprAsFunction = (AbstractFunctionExpression) expr;
+            GroovyExpression exprCaller = exprAsFunction.getCaller();
+            parentExpr.setCaller(exprCaller);
+            updateCurrentFunctionDefintion(exprAsFunction);
+        }
+    }
+
+    private void updateCurrentFunctionDefintion(AbstractFunctionExpression exprToAdd) {
+        ClosureExpression functionBodyClosure = context.getUserDefinedFunctionBody(currentFunctionName);
+        if (functionBodyClosure == null) {
+            throw new IllegalStateException("User-defined function " + currentFunctionName + " not found!");
+        }
+        List<GroovyExpression> exprs = functionBodyClosure.getStatements();
+        GroovyExpression currentFunctionBody = exprs.get(exprs.size() - 1);
+        //Update the expression so it is called by the current return
+        //value of the function.
+        exprToAdd.setCaller(currentFunctionBody);
+        functionBodyClosure.replaceStatement(exprs.size() - 1, exprToAdd);
+    }
+
+    //Determines if extracting this expression into a function will shorten
+    //the overall length of the Groovy script.
+    private boolean creatingFunctionShortensGremlin(GroovyExpression headExpr) {
+        int tailLength = getTailLength();
+        int length = headExpr.toString().length() - tailLength;
+
+        int overhead = 0;
+        if (nextFunctionBodyStart instanceof AbstractFunctionExpression) {
+            overhead = functionDefLength;
+        } else {
+            overhead = INITIAL_FUNCTION_DEF_LENGTH;
+        }
+        overhead += FUNCTION_CALL_OVERHEAD * scaleFactor;
+        //length * scaleFactor = space taken by having the expression be inlined [scaleFactor] times
+        //overhead + length = space taken by the function definition and its calls
+        return length * scaleFactor > overhead + length;
+    }
+
+    private int getTailLength() {
+        if (nextFunctionBodyStart == null) {
+            return 0;
+        }
+        if (!(nextFunctionBodyStart instanceof AbstractFunctionExpression)) {
+            return 0;
+        }
+        AbstractFunctionExpression bodyEndAsFunction = (AbstractFunctionExpression) nextFunctionBodyStart;
+        return bodyEndAsFunction.getCaller().toString().length();
+    }
+
+    public GroovyExpression getNewRootExpression() {
+        return newRootExpression;
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/GremlinOptimization.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import org.apache.atlas.groovy.GroovyExpression;
+
+/**
+ * An optimization that can be applied to a gremlin query.
+ */
+public interface GremlinOptimization {
+
+    /**
+     * Whether or not this optimization should be applied to the given expression
+     * @param expr
+     * @param contxt
+     * @return
+     */
+    boolean appliesTo(GroovyExpression expr, OptimizationContext contxt);
+    /**
+     * Whether or not GremlinQueryOptimizer should call this optimization recursively
+     * on the updated children.
+     */
+    boolean isApplyRecursively();
+
+    /**
+     * Applies the optimization.
+     *
+     * @param expr
+     * @param context
+     * @return the optimized expression
+     */
+    GroovyExpression apply(GroovyExpression expr, OptimizationContext context);
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/GremlinQueryOptimizer.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.StatementListExpression;
+import org.apache.atlas.groovy.TraversalStepType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+
+
+/**
+ * Optimizer for gremlin queries.  This class provides a framework for applying optimizations
+ * to gremlin queries.  Each optimization is implemented as a class that implements {@link GremlinOptimization}.
+ *
+ * The GremlinQueryOptimizer is the entry point for applying these optimizations.
+ *
+ *
+ */
+public final class GremlinQueryOptimizer {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(GremlinQueryOptimizer.class);
+
+
+    private final List<GremlinOptimization> optimizations = new ArrayList<>();
+
+    //Allows expression factory to be substituted in unit tests.
+    private static volatile GremlinExpressionFactory FACTORY = GremlinExpressionFactory.INSTANCE;
+
+    private static volatile GremlinQueryOptimizer INSTANCE = null;
+
+    private GremlinQueryOptimizer() {
+
+    }
+
+    private void addOptimization(GremlinOptimization opt) {
+        optimizations.add(opt);
+    }
+
+    public static GremlinQueryOptimizer getInstance() {
+        if(INSTANCE == null) {
+            synchronized(GremlinQueryOptimizer.class) {
+                if(INSTANCE == null) {
+                    GremlinQueryOptimizer createdInstance = new GremlinQueryOptimizer();
+                    //The order here is important.  If there is an "or" nested within an "and",
+                    //that will not be found if ExpandOrsOptimization runs before ExpandAndsOptimization.
+                    createdInstance.addOptimization(new ExpandAndsOptimization(FACTORY));
+                    createdInstance.addOptimization(new ExpandOrsOptimization(FACTORY));
+                    INSTANCE = createdInstance;
+                }
+            }
+        }
+        return INSTANCE;
+    }
+
+    /**
+     * For testing only
+     */
+    @VisibleForTesting
+    public static void setExpressionFactory(GremlinExpressionFactory factory) {
+        GremlinQueryOptimizer.FACTORY = factory;
+    }
+
+    /**
+     * For testing only
+     */
+    @VisibleForTesting
+    public static void reset() {
+        INSTANCE = null;
+    }
+
+    /**
+     * Optimizes the provided groovy expression.  Note that the optimization
+     * is a <i>destructive</i> process.  The source GroovyExpression will be
+     * modified as part of the optimization process.  This is done to avoid
+     * expensive copying operations where possible.
+     *
+     * @param source what to optimize
+     * @return the optimized query
+     */
+    public GroovyExpression optimize(GroovyExpression source) {
+        LOGGER.debug("Optimizing gremlin query: " + source);
+        OptimizationContext context = new OptimizationContext();
+        GroovyExpression updatedExpression = source;
+        for (GremlinOptimization opt : optimizations) {
+            updatedExpression = optimize(updatedExpression, opt, context);
+            LOGGER.debug("After "+ opt.getClass().getSimpleName() + ", query = " + updatedExpression);
+        }
+
+        StatementListExpression result = new StatementListExpression();
+        result.addStatements(context.getInitialStatements());
+        result.addStatement(updatedExpression);
+        LOGGER.debug("Final optimized query:  " + result.toString());
+        return result;
+    }
+
+    /**
+     * Optimizes the expression using the given optimization
+     * @param source
+     * @param optimization
+     * @param context
+     * @return
+     */
+    private GroovyExpression optimize(GroovyExpression source, GremlinOptimization optimization,
+                                          OptimizationContext context) {
+        GroovyExpression result = source;
+        if (optimization.appliesTo(source, context)) {
+            //Apply the optimization to the expression.
+            result = optimization.apply(source, context);
+        }
+        if (optimization.isApplyRecursively()) {
+            //Visit the children, update result with the optimized
+            //children.
+            List<GroovyExpression> updatedChildren = new ArrayList<>();
+            boolean changed = false;
+            for (GroovyExpression child : result.getChildren()) {
+                //Recursively optimize this child.
+                GroovyExpression updatedChild = optimize(child, optimization, context);
+                changed |= updatedChild != child;
+                updatedChildren.add(updatedChild);
+            }
+            if (changed) {
+                //TBD - Can we update in place rather than making a copy?
+                result = result.copy(updatedChildren);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Visits all expressions in the call hierarchy of an expression.  For example,
+     * in the expression g.V().has('x','y'), the order would be
+     * <ol>
+     *    <li>pre-visit has('x','y')</li>
+     *    <li>pre-visit V()</li>
+     *    <li>visit g (non-function caller)</li>
+     *    <li>post-visit V()</li>
+     *    <li>post-visit has('x','y')</li>
+     * </ol>
+     * @param expr
+     * @param visitor
+     */
+    public static void visitCallHierarchy(GroovyExpression expr, CallHierarchyVisitor visitor) {
+
+        if (expr == null) {
+            visitor.visitNullCaller();
+            return;
+        }
+        if (expr instanceof AbstractFunctionExpression) {
+            AbstractFunctionExpression functionCall = (AbstractFunctionExpression)expr;
+            if (!visitor.preVisitFunctionCaller(functionCall)) {
+                return;
+            }
+            GroovyExpression caller = functionCall.getCaller();
+            visitCallHierarchy(caller, visitor);
+            if (!visitor.postVisitFunctionCaller(functionCall)) {
+                return;
+            }
+        } else {
+            visitor.visitNonFunctionCaller(expr);
+        }
+    }
+
+    /**
+     * Determines if the given expression is an "or" expression.
+     * @param expr
+     * @return
+     */
+    public static boolean isOrExpression(GroovyExpression expr) {
+        return IsOr.INSTANCE.apply(expr);
+    }
+
+    /**
+     * Determines whether the given expression can safely
+     * be pulled out of an and/or expression.
+     *
+     * @param expr an argument to an and or or function
+     * @return
+     */
+    public static boolean isExtractable(GroovyExpression expr) {
+
+        HasForbiddenType hasForbiddenTypePredicate = new HasForbiddenType(FACTORY);
+
+        //alias could conflict with alias in parent traversal
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.SIDE_EFFECT);
+
+        //inlining out(), in() steps will change the result of calls after the and/or()
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.FLAT_MAP_TO_ELEMENTS);
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.FLAT_MAP_TO_VALUES);
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.BARRIER);
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.MAP_TO_ELEMENT);
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.MAP_TO_VALUE);
+
+        //caller expects to be able to continue the traversal.  We can't end it
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.END);
+
+
+        //we can't inline child traversals
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.SOURCE);
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.START);
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.SIDE_EFFECT);
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.NONE);
+        hasForbiddenTypePredicate.addForbiddenType(TraversalStepType.BRANCH);
+
+        ExpressionFinder forbiddenExpressionFinder = new ExpressionFinder(hasForbiddenTypePredicate);
+        GremlinQueryOptimizer.visitCallHierarchy(expr, forbiddenExpressionFinder);
+        return ! forbiddenExpressionFinder.isExpressionFound();
+    }
+
+    /**
+     * Recursively copies and follows the caller hierarchy of the expression until we come
+     * to a function call with a null caller.  The caller of that expression is set
+     * to newLeaf.
+     *
+     * @param expr
+     * @param newLeaf
+     * @return the updated (/copied) expression
+     */
+    public static GroovyExpression copyWithNewLeafNode(AbstractFunctionExpression expr, GroovyExpression newLeaf) {
+
+
+        AbstractFunctionExpression result = (AbstractFunctionExpression)expr.copy();
+
+        //remove leading anonymous traversal expression, if there is one
+        if(FACTORY.isLeafAnonymousTraversalExpression(expr)) {
+            result = (AbstractFunctionExpression)newLeaf;
+        } else {
+            GroovyExpression newCaller = null;
+            if (expr.getCaller() == null) {
+                newCaller = newLeaf;
+            } else {
+                newCaller = copyWithNewLeafNode((AbstractFunctionExpression)result.getCaller(), newLeaf);
+            }
+            result.setCaller(newCaller);
+        }
+        return result;
+    }
+
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/HasForbiddenType.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.HashSet;
+import java.util.Set;
+import com.google.common.base.Function;
+
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.TraversalStepType;
+
+/**
+ * Function that tests whether the expression is an 'or'
+ * graph traversal function.
+ */
+public final class HasForbiddenType implements Function<GroovyExpression, Boolean> {
+
+    private Set<TraversalStepType> forbiddenTypes =  new HashSet<>();
+    private final GremlinExpressionFactory factory;
+
+    public HasForbiddenType(GremlinExpressionFactory factory) {
+        this.factory = factory;
+    }
+
+    public void addForbiddenType(TraversalStepType type) {
+        forbiddenTypes.add(type);
+    }
+
+    @Override
+    public Boolean apply(GroovyExpression expr) {
+        if(factory.isLeafAnonymousTraversalExpression(expr)) {
+            return false;
+        }
+        return forbiddenTypes.contains(expr.getType());
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/IsOr.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import com.google.common.base.Function;
+
+import org.apache.atlas.groovy.FunctionCallExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.TraversalStepType;
+
+/**
+ * Function that tests whether the expression is an 'or'
+ * graph traversal function.
+ */
+public final class IsOr implements Function<GroovyExpression, Boolean> {
+
+    public static final IsOr INSTANCE = new IsOr();
+
+    private IsOr() {
+    }
+
+    @Override
+    public Boolean apply(GroovyExpression expr) {
+        if (!(expr instanceof FunctionCallExpression)) {
+            return false;
+        }
+        if (expr.getType() != TraversalStepType.FILTER) {
+            return false;
+        }
+        FunctionCallExpression functionCall = (FunctionCallExpression)expr;
+        return functionCall.getFunctionName().equals("or");
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/IsOrParent.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import com.google.common.base.Function;
+
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.FunctionCallExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.TraversalStepType;
+
+/**
+ * Matches an expression that gets called after calling or().  For example,
+ * in g.V().or(x,y).toList(), "toList()" is the "or parent", so calling
+ * "apply()" on this expression would return true and calling it on all
+ * the other ones would return false.
+ */
+public final class IsOrParent implements Function<GroovyExpression, Boolean> {
+
+    public static final IsOrParent INSTANCE = new IsOrParent();
+
+    private IsOrParent() {
+
+    }
+
+    @Override
+    public Boolean apply(GroovyExpression expr) {
+        if (!(expr instanceof AbstractFunctionExpression)) {
+            return false;
+        }
+        AbstractFunctionExpression functionCall = (AbstractFunctionExpression)expr;
+        GroovyExpression target = functionCall.getCaller();
+
+        if (!(target instanceof FunctionCallExpression)) {
+            return false;
+        }
+
+        if (target.getType() != TraversalStepType.FILTER) {
+            return false;
+        }
+
+        FunctionCallExpression targetFunction = (FunctionCallExpression)target;
+        return targetFunction.getFunctionName().equals("or");
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/OptimizationContext.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.ClosureExpression;
+import org.apache.atlas.groovy.ClosureExpression.VariableDeclaration;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.IdentifierExpression;
+import org.apache.atlas.groovy.ListExpression;
+import org.apache.atlas.groovy.TypeCoersionExpression;
+import org.apache.atlas.groovy.VariableAssignmentExpression;
+
+/**
+ * Maintains state information during gremlin optimization.
+ */
+public class OptimizationContext {
+
+    private static final String TMP_ALIAS_NAME = "__tmp";
+    private static final String FINAL_ALIAS_NAME = "__res";
+    private static final String RESULT_VARIABLE = "r";
+    private final List<GroovyExpression> initialStatements = new ArrayList<>();
+    private GroovyExpression resultExpression = getResultVariable();
+    private int counter = 1;
+    private final Map<String, ClosureExpression> functionBodies = new HashMap<>();
+    private AbstractFunctionExpression rangeExpression;
+
+    public OptimizationContext() {
+
+    }
+
+    /**
+     * @return
+     */
+    public List<GroovyExpression> getInitialStatements() {
+        return initialStatements;
+    }
+
+    public void prependStatement(GroovyExpression expr) {
+        initialStatements.add(0, expr);
+    }
+
+    public String getUniqueFunctionName() {
+        return "f" + (counter++);
+    }
+
+
+    public GroovyExpression getDefineResultVariableStmt() {
+        GroovyExpression castExpression = new TypeCoersionExpression(new ListExpression(), "Set");
+        GroovyExpression resultVarDef =  new VariableAssignmentExpression(RESULT_VARIABLE, castExpression);
+        return resultVarDef;
+
+    }
+    public void setResultExpression(GroovyExpression expr) {
+        resultExpression = expr;
+    }
+
+    public GroovyExpression getResultExpression() {
+        return resultExpression;
+    }
+
+    public GroovyExpression getResultVariable() {
+        return new IdentifierExpression(RESULT_VARIABLE);
+    }
+
+    public ClosureExpression getUserDefinedFunctionBody(String functionName) {
+        return functionBodies.get(functionName);
+    }
+
+    public String addFunctionDefinition(VariableDeclaration decl, GroovyExpression body) {
+        String functionName = getUniqueFunctionName();
+        List<VariableDeclaration> decls = (decl == null) ? Collections.<VariableDeclaration>emptyList() : Collections.singletonList(decl);
+        ClosureExpression bodyClosure = new ClosureExpression(body, decls);
+        VariableAssignmentExpression expr = new VariableAssignmentExpression(functionName, bodyClosure);
+        initialStatements.add(expr);
+        functionBodies.put(functionName, bodyClosure);
+        return functionName;
+    }
+
+    public String getFinalAliasName() {
+        return FINAL_ALIAS_NAME;
+    }
+
+    public String getTempAliasName() {
+        return TMP_ALIAS_NAME;
+    }
+
+    public void setRangeExpression(AbstractFunctionExpression rangeExpression) {
+        this.rangeExpression = rangeExpression;
+    }
+
+    public AbstractFunctionExpression getRangeExpression() {
+        return rangeExpression;
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/OrderFinder.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+
+
+/**
+ * Finds order expression in the call hierarchy.
+ *
+ */
+public class OrderFinder implements CallHierarchyVisitor {
+
+    private boolean hasOrderExpression;
+    private GremlinExpressionFactory gremlinFactory;
+
+    public OrderFinder(GremlinExpressionFactory gremlinFactory) {
+        this.gremlinFactory = gremlinFactory;
+    }
+
+    @Override
+    public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
+
+        return true;
+    }
+
+    @Override
+    public void visitNonFunctionCaller(GroovyExpression expr) {
+    }
+
+    @Override
+    public void visitNullCaller() {
+    }
+
+    @Override
+    public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
+
+        if (gremlinFactory.isOrderExpression(functionCall)) {
+            hasOrderExpression = true;
+            return false;
+        }
+        return true;
+    }
+
+
+    public boolean hasOrderExpression() {
+
+        return hasOrderExpression;
+    }
+
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/PathExpressionFinder.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.FunctionCallExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+
+/**
+ * Determines whether an expression contains a path() function.
+ */
+public class PathExpressionFinder implements CallHierarchyVisitor {
+
+    private boolean found = false;
+
+    @Override
+    public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
+        if(expr instanceof FunctionCallExpression) {
+            found = ((FunctionCallExpression)expr).getFunctionName().equals("path");
+            if(found) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public void visitNonFunctionCaller(GroovyExpression expr) {
+
+    }
+
+    @Override
+    public void visitNullCaller() {
+
+    }
+
+    public boolean isPathExpressionFound() {
+        return found;
+    }
+
+    @Override
+    public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
+
+        return false;
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/RangeFinder.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+
+
+/**
+ * Finds all range expressions in the call hierarchy.
+ *
+ */
+public class RangeFinder implements CallHierarchyVisitor {
+
+    private List<AbstractFunctionExpression> rangeExpressions = new ArrayList<>();
+    private GremlinExpressionFactory factory;
+
+    public RangeFinder(GremlinExpressionFactory factory) {
+        this.factory = factory;
+    }
+
+    @Override
+    public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
+
+        return true;
+    }
+
+    @Override
+    public void visitNonFunctionCaller(GroovyExpression expr) {
+    }
+
+    @Override
+    public void visitNullCaller() {
+    }
+
+    @Override
+    public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
+
+        if (factory.isRangeExpression(functionCall)) {
+            rangeExpressions.add(functionCall);
+        }
+        return true;
+    }
+
+    public List<AbstractFunctionExpression> getRangeExpressions() {
+        return rangeExpressions;
+    }
+
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/SplitPointFinder.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.FunctionCallExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.TraversalStepType;
+
+
+/**
+ * This class finds the first place in the expression where the value of the
+ * traverser is changed from being a vertex to being something else.  This is
+ * important in the "or" optimization logic, since the union operation must be
+ * done on *vertices* in order to preserve the semantics of the query.  In addition,
+ * expressions that have side effects must be moved as well, so that those
+ * side effects will be available to the steps that need them.
+ */
+public class SplitPointFinder implements CallHierarchyVisitor {
+
+    //Any steps that change the traverser value to something that is not a vertex or edge
+    //must be included here, so that the union created by ExpandOrsOptimization
+    //is done over vertices/edges.
+    private static final Set<TraversalStepType> TYPES_REQUIRED_IN_RESULT_EXPRESSION = new HashSet<>(
+            Arrays.asList(
+                    TraversalStepType.BARRIER,
+                    TraversalStepType.BRANCH,
+                    TraversalStepType.SIDE_EFFECT,
+                    TraversalStepType.MAP_TO_VALUE,
+                    TraversalStepType.FLAT_MAP_TO_VALUES,
+                    TraversalStepType.END,
+                    TraversalStepType.NONE));
+
+    private final Set<String> requiredAliases = new HashSet<>();
+
+    //Exceptions to the requirement that all expressions with a type
+    //in the above list must be in the result expression.  If the
+    //function name is in this list, it is ok for that expression
+    //to not be in the result expression.  This mechanism allows
+    //aliases to remain outside the result expression.  Other
+    //exceptions may be found in the future.
+    private static final Map<TraversalStepType, WhiteList> WHITE_LISTS = new HashMap<>();
+    static {
+        WHITE_LISTS.put(TraversalStepType.SIDE_EFFECT, new WhiteList("as"));
+    }
+
+    private final GremlinExpressionFactory factory;
+
+    public SplitPointFinder(GremlinExpressionFactory factory) {
+        this.factory = factory;
+    }
+
+    /**
+     * Represents a set of function names.
+     */
+    private static final class WhiteList {
+        private Set<String> allowedFunctionNames = new HashSet<>();
+        public WhiteList(String... names) {
+            for(String name : names) {
+                allowedFunctionNames.add(name);
+            }
+        }
+        public boolean contains(String name) {
+            return allowedFunctionNames.contains(name);
+        }
+    }
+
+    private AbstractFunctionExpression splitPoint;
+
+    @Override
+    public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
+        requiredAliases.addAll(factory.getAliasesRequiredByExpression(expr));
+        return true;
+    }
+
+    @Override
+    public void visitNonFunctionCaller(GroovyExpression expr) {
+
+    }
+
+    @Override
+    public void visitNullCaller() {
+
+    }
+
+    public AbstractFunctionExpression getSplitPoint() {
+        return splitPoint;
+    }
+
+    @Override
+    public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
+        String aliasName = factory.getAliasNameIfRelevant(functionCall);
+        if (splitPoint == null) {
+
+            boolean required = isRequiredAlias(aliasName) ||
+                    isRequiredInResultExpression(functionCall);
+            if (required) {
+                splitPoint = functionCall;
+            }
+        }
+        removeSeenAlias(aliasName);
+
+        return true;
+    }
+
+    private void removeSeenAlias(String aliasName) {
+        if(aliasName != null) {
+            requiredAliases.remove(aliasName);
+        }
+    }
+
+    private boolean isRequiredAlias(String aliasName) {
+        if(aliasName != null) {
+            return requiredAliases.contains(aliasName);
+        }
+        return false;
+    }
+
+    private boolean isRequiredInResultExpression(AbstractFunctionExpression expr) {
+
+        TraversalStepType type = expr.getType();
+        if (!TYPES_REQUIRED_IN_RESULT_EXPRESSION.contains(type)) {
+            return false;
+        }
+
+        if(expr instanceof FunctionCallExpression) {
+            FunctionCallExpression functionCall = (FunctionCallExpression)expr;
+            //check if the white list permits this function call.  If there is
+            //no white list, all expressions with the current step type must go in the
+            //result expression.
+            WhiteList whiteList = WHITE_LISTS.get(type);
+            if(whiteList != null && whiteList.contains(functionCall.getFunctionName())) {
+                return false;
+            }
+        }
+        return true;
+
+    }
+}

repository/src/main/java/org/apache/atlas/gremlin/optimizer/UpdatedExpressions.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.gremlin.optimizer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.atlas.groovy.GroovyExpression;
+
+/**
+ * Represents a list of updated expressions.
+ */
+public class UpdatedExpressions {
+
+    private List<List<GroovyExpression>> updatedChildren = new ArrayList<>();
+    private boolean changed = false;
+
+    public UpdatedExpressions(boolean changed, List<List<GroovyExpression>> updatedChildren) {
+        this.changed = changed;
+        this.updatedChildren = updatedChildren;
+    }
+
+    public List<List<GroovyExpression>> getUpdatedChildren() {
+        return updatedChildren;
+    }
+
+    public boolean hasChanges() {
+        return changed;
+    }
+}

repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java

@@ -67,19 +67,26 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
 
     private static final GraphHelper graphHelper = GraphHelper.getInstance();
 
-    private final AtlasGraph graph;
-
     private DeleteHandler deleteHandler;
 
-    private GraphToTypedInstanceMapper graphToInstanceMapper;
+    private final IAtlasGraphProvider graphProvider;
+    private final GraphToTypedInstanceMapper graphToInstanceMapper;
 
     @Inject
     public GraphBackedMetadataRepository(DeleteHandler deleteHandler) {
-        this.graph = AtlasGraphProvider.getGraphInstance();
-        graphToInstanceMapper = new GraphToTypedInstanceMapper(graph);
+        this.graphProvider = new AtlasGraphProvider();
+        this.graphToInstanceMapper = new GraphToTypedInstanceMapper(graphProvider);
         this.deleteHandler = deleteHandler;
     }
 
+    //for testing only
+    public GraphBackedMetadataRepository(IAtlasGraphProvider graphProvider, DeleteHandler deleteHandler) {
+        this.graphProvider = graphProvider;
+        this.graphToInstanceMapper = new GraphToTypedInstanceMapper(graphProvider);
+        this.deleteHandler = deleteHandler;
+    }
+
+
     public GraphToTypedInstanceMapper getGraphToInstanceMapper() {
         return graphToInstanceMapper;
     }
@@ -194,7 +201,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             LOG.debug("Retrieving entity list for type={}", entityType);
         }
 
-        AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
+        AtlasGraphQuery query = getGraph().query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
         Iterator<AtlasVertex> results = query.vertices().iterator();
         if (!results.hasNext()) {
             return Collections.emptyList();
@@ -429,7 +436,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                 requestContext.getDeletedEntityIds());
     }
 
-    public AtlasGraph getGraph() {
-        return AtlasGraphProvider.getGraphInstance();
+    public AtlasGraph getGraph() throws RepositoryException {
+        return graphProvider.get();
     }
 }

repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java

@@ -19,6 +19,7 @@ package org.apache.atlas.repository.graph;
 
 import com.google.inject.Singleton;
 import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.RepositoryException;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.AtlasEdge;
 import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
@@ -59,10 +60,10 @@ public final class GraphToTypedInstanceMapper {
     private static TypeSystem typeSystem = TypeSystem.getInstance();
     private static final GraphHelper graphHelper = GraphHelper.getInstance();
 
-    private AtlasGraph graph;
+    private final IAtlasGraphProvider graphProvider;
 
-    public GraphToTypedInstanceMapper(AtlasGraph graph) {
-        this.graph = graph;
+    public GraphToTypedInstanceMapper(IAtlasGraphProvider graphProvider) {
+        this.graphProvider = graphProvider;
     }
 
     public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, AtlasVertex instanceVertex)
@@ -407,7 +408,7 @@ public final class GraphToTypedInstanceMapper {
 
 
     public ITypedInstance getReferredEntity(String edgeId, IDataType<?> referredType) throws AtlasException {
-        final AtlasEdge edge = graph.getEdge(edgeId);
+        final AtlasEdge edge = getGraph().getEdge(edgeId);
         if (edge != null) {
             final AtlasVertex referredVertex = edge.getInVertex();
             if (referredVertex != null) {
@@ -433,5 +434,9 @@ public final class GraphToTypedInstanceMapper {
         }
         return null;
     }
+
+    private AtlasGraph getGraph() throws RepositoryException {
+        return graphProvider.get();
+    }
 }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java

@@ -17,8 +17,14 @@
  */
 package org.apache.atlas.repository.store.graph.v1;
 
-import atlas.shaded.hbase.guava.common.annotations.VisibleForTesting;
-import com.google.inject.Provider;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.TypeCategory;
@@ -26,8 +32,8 @@ import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasObjectId;
 import org.apache.atlas.model.instance.AtlasStruct;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
-import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
 import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
+import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
 import org.apache.atlas.repository.store.graph.EntityResolver;
 import org.apache.atlas.type.AtlasArrayType;
 import org.apache.atlas.type.AtlasEntityType;
@@ -36,14 +42,9 @@ import org.apache.atlas.type.AtlasStructType;
 import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 
-import javax.inject.Inject;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
 
 
 public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java

@@ -18,8 +18,9 @@
 package org.apache.atlas.repository.store.graph.v1;
 
 
-import atlas.shaded.hbase.guava.common.annotations.VisibleForTesting;
-import com.google.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.RequestContextV1;
@@ -34,15 +35,17 @@ import org.apache.atlas.model.instance.EntityMutationResponse;
 import org.apache.atlas.model.instance.EntityMutations;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.AtlasEntityStore;
-import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
 import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
+import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.List;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.Inject;
+
+
 
 public class AtlasEntityStoreV1 implements AtlasEntityStore {
 

repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java

@@ -24,9 +24,11 @@ import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.repository.audit.EntityAuditRepository;
 import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
 import org.apache.atlas.repository.graph.DeleteHandler;
 import org.apache.atlas.repository.graph.SoftDeleteHandler;
 import org.apache.atlas.repository.graphdb.GraphDatabase;
+import org.apache.atlas.repository.graphdb.GremlinVersion;
 import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
 import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1;
 import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
@@ -137,7 +139,6 @@ public class AtlasRepositoryConfiguration {
         }
     }
 
-
     private static final String GRAPH_DATABASE_IMPLEMENTATION_PROPERTY = "atlas.graphdb.backend";
     private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = "org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase";
 
@@ -153,6 +154,22 @@ public class AtlasRepositoryConfiguration {
     }
 
     /**
+     * This optimization is configurable as a fail-safe in case issues are found
+     * with the optimizer in production systems.
+     */
+    public static final String GREMLIN_OPTIMIZER_ENABLED_PROPERTY = "atlas.query.gremlinOptimizerEnabled";
+    private static final boolean DEFAULT_GREMLIN_OPTIMZER_ENABLED = true;
+
+    public static boolean isGremlinOptimizerEnabled() {
+        try {
+            return ApplicationProperties.get().getBoolean(GREMLIN_OPTIMIZER_ENABLED_PROPERTY, DEFAULT_GREMLIN_OPTIMZER_ENABLED);
+        } catch (AtlasException e) {
+            LOG.error("Could not determine value of " + GREMLIN_OPTIMIZER_ENABLED_PROPERTY + ".  Defaulting to " + DEFAULT_GREMLIN_OPTIMZER_ENABLED, e);
+            return DEFAULT_GREMLIN_OPTIMZER_ENABLED;
+        }
+    }
+
+    /**
      * Get the list of operations which are configured to be skipped from auditing
      * Valid format is HttpMethod:URL eg: GET:Version
      * @return list of string

repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala

@@ -32,15 +32,19 @@ import scala.collection.JavaConversions.bufferAsJavaList
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 
+
 import org.apache.atlas.gremlin.GremlinExpressionFactory
+import org.apache.atlas.gremlin.optimizer.GremlinQueryOptimizer
 import org.apache.atlas.groovy.CastExpression
-import org.apache.atlas.groovy.CodeBlockExpression
+import org.apache.atlas.groovy.ClosureExpression
+import org.apache.atlas.groovy.LabeledExpression
 import org.apache.atlas.groovy.FunctionCallExpression
 import org.apache.atlas.groovy.GroovyExpression
 import org.apache.atlas.groovy.GroovyGenerationContext
 import org.apache.atlas.groovy.IdentifierExpression
 import org.apache.atlas.groovy.ListExpression
 import org.apache.atlas.groovy.LiteralExpression
+import org.apache.atlas.groovy.TraversalStepType
 import org.apache.atlas.query.Expressions.AliasExpression
 import org.apache.atlas.query.Expressions.ArithmeticExpression
 import org.apache.atlas.query.Expressions.BackReference
@@ -78,7 +82,10 @@ import org.apache.atlas.query.Expressions.MaxExpression
 import org.apache.atlas.query.Expressions.MinExpression
 import org.apache.atlas.query.Expressions.SumExpression
 import org.apache.atlas.query.Expressions.CountExpression
+
+import org.apache.atlas.util.AtlasRepositoryConfiguration
 import java.util.HashSet
+
 trait IntSequence {
     def next: Int
 }
@@ -120,6 +127,69 @@ trait SelectExpressionHandling {
         }
     }
 
+    // Removes back references in comparison expressions that are
+    // right after an alias expression.
+    //
+    //For example:
+    // .as('x').and(select('x').has(y),...) is changed to
+    // .as('x').and(has(y),...)
+    //
+    //This allows the "has" to be extracted out of the and/or by
+    //the GremlinQueryOptimizer so the index can be used to evaluate
+    //the predicate.
+
+     val RemoveUnneededBackReferences : PartialFunction[Expression, Expression] = {
+
+         case filterExpr@FilterExpression(aliasExpr@AliasExpression(_,aliasName), filterChild) => {
+             val updatedChild = removeUnneededBackReferences(filterChild, aliasName)
+             val changed  = !(updatedChild eq filterChild)
+             if(changed) {
+                 FilterExpression(aliasExpr, updatedChild)
+             }
+             else {
+                 filterExpr
+             }
+
+         }
+         case x => x
+     }
+     def removeUnneededBackReferences(expr: Expression, outerAlias: String) : Expression = expr match {
+         case logicalExpr@LogicalExpression(logicalOp,children) => {
+            var changed : Boolean = false;
+            val updatedChildren : List[Expression] = children.map { child =>
+                val updatedChild = removeUnneededBackReferences(child, outerAlias);
+                changed |= ! (updatedChild eq child);
+                updatedChild
+            }
+           if(changed) {
+               LogicalExpression(logicalOp,updatedChildren)
+           }
+           else {
+              logicalExpr
+           }
+        }
+        case comparisonExpr@ComparisonExpression(_,_,_) => {
+            var changed = false
+             val updatedLeft = removeUnneededBackReferences(comparisonExpr.left, outerAlias);
+             changed |= !( updatedLeft eq comparisonExpr.left);
+
+             val updatedRight = removeUnneededBackReferences(comparisonExpr.right, outerAlias);
+             changed |= !(updatedRight eq comparisonExpr.right);
+
+             if (changed) {
+                 ComparisonExpression(comparisonExpr.symbol, updatedLeft, updatedRight)
+             } else {
+                 comparisonExpr
+             }
+        }
+        case FieldExpression(fieldName, fieldInfo, Some(br @ BackReference(brAlias, _, _))) if outerAlias.equals(brAlias) => {
+           //Remove the back reference, since the thing it references is right in front
+           //of the comparison expression we're in
+           FieldExpression(fieldName, fieldInfo, None)
+       }
+       case x => x
+    }
+
     //in groupby, convert alias expressions defined in the group by child to BackReferences
     //in the groupby list and selectList.
     val AddBackReferencesToGroupBy : PartialFunction[Expression, Expression] = {
@@ -456,7 +526,10 @@ class GremlinTranslator(expr: Expression,
             return translateLiteralValue(l.dataType, l);
         }
         case list: ListLiteral[_] =>  {
-            val  values : java.util.List[GroovyExpression] = translateList(list.rawValue, true); //why hard coded
+            //Here, we are creating a Groovy list literal expression ([value1, value2, value3]).  Because
+            //of this, any gremlin query expressions within the list must start with an anonymous traversal.
+            //We set 'inClosure' to true in this case to make that happen.
+            val  values : java.util.List[GroovyExpression] = translateList(list.rawValue, true);
             return new ListExpression(values);
         }
         case in@TraitInstanceExpression(child) => {
@@ -493,7 +566,7 @@ class GremlinTranslator(expr: Expression,
         case limitOffset@LimitExpression(child, limit, offset) => {
             val childExpr = genQuery(parent, child, inClosure);
             val totalResultRows = limit.value + offset.value;
-            return GremlinExpressionFactory.INSTANCE.generateLimitExpression(childExpr, offset.value, totalResultRows);
+            return GremlinExpressionFactory.INSTANCE.generateRangeExpression(childExpr, offset.value, totalResultRows);
         }
         case count@CountExpression() => {
           val listExpr = GremlinExpressionFactory.INSTANCE.getClosureArgumentValue();
@@ -621,8 +694,7 @@ class GremlinTranslator(expr: Expression,
 
     def genFullQuery(expr: Expression, hasSelect: Boolean): String = {
 
-        var q : GroovyExpression = new FunctionCallExpression(new IdentifierExpression("g"),"V");
-
+        var q : GroovyExpression = new FunctionCallExpression(TraversalStepType.START, new IdentifierExpression(TraversalStepType.SOURCE, "g"),"V");
 
         val debug:Boolean = false
         if(gPersistenceBehavior.addGraphVertexPrefix(preStatements)) {
@@ -631,15 +703,23 @@ class GremlinTranslator(expr: Expression,
 
         q = genQuery(q, expr, false)
 
-        q = new FunctionCallExpression(q, "toList");
+        q = GremlinExpressionFactory.INSTANCE.generateToListExpression(q);
         q = gPersistenceBehavior.getGraph().addOutputTransformationPredicate(q, hasSelect, expr.isInstanceOf[PathExpression]);
 
-        var overallExpression = new CodeBlockExpression();
-        overallExpression.addStatements(preStatements);
-        overallExpression.addStatement(q)
-        overallExpression.addStatements(postStatements);
 
-        var qryStr = generateGremlin(overallExpression);
+        if(AtlasRepositoryConfiguration.isGremlinOptimizerEnabled()) {
+            q = GremlinQueryOptimizer.getInstance().optimize(q);
+        }
+
+	    val closureExpression = new ClosureExpression();
+
+        closureExpression.addStatements(preStatements);
+        closureExpression.addStatement(q)
+        closureExpression.addStatements(postStatements);
+
+	    val overallExpression = new LabeledExpression("L", closureExpression);
+
+        val qryStr = generateGremlin(overallExpression);
 
         if(debug) {
           println(" query " + qryStr)
@@ -666,6 +746,7 @@ class GremlinTranslator(expr: Expression,
         e1 = e1.transformUp(addAliasToLoopInput())
         e1 = e1.transformUp(instanceClauseToTop(e1))
         e1 = e1.transformUp(traitClauseWithInstanceForTop(e1))
+        e1 = e1.transformUp(RemoveUnneededBackReferences)
 
         //Following code extracts the select expressions from expression tree.
 

repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java

@@ -508,6 +508,9 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
                 {"from hive_db limit 3 offset 1", 2},
                 {"hive_db", 3},
                 {"hive_db where hive_db.name=\"Reporting\"", 1},
+                {"hive_db where hive_db.name=\"Reporting\" or hive_db.name=\"Sales\" or hive_db.name=\"Logging\" limit 1 offset 1", 1},
+                {"hive_db where hive_db.name=\"Reporting\" or hive_db.name=\"Sales\" or hive_db.name=\"Logging\" limit 1 offset 2", 1},
+                {"hive_db where hive_db.name=\"Reporting\" or hive_db.name=\"Sales\" or hive_db.name=\"Logging\" limit 2 offset 1", 2},
                 {"hive_db where hive_db.name=\"Reporting\" limit 10 ", 1},
                 {"hive_db hive_db.name = \"Reporting\"", 1},
                 {"hive_db where hive_db.name=\"Reporting\" select name, owner", 1},

repository/src/test/java/org/apache/atlas/repository/graph/AbstractGremlinQueryOptimizerTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.apache.atlas.gremlin.optimizer.GremlinQueryOptimizer;
+import org.apache.atlas.gremlin.optimizer.RangeFinder;
+import org.apache.atlas.groovy.AbstractFunctionExpression;
+import org.apache.atlas.groovy.FunctionCallExpression;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.groovy.IdentifierExpression;
+import org.apache.atlas.groovy.LiteralExpression;
+import org.apache.atlas.groovy.TraversalStepType;
+import org.apache.atlas.query.GraphPersistenceStrategies;
+import org.apache.atlas.query.TypeUtils.FieldInfo;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.GremlinVersion;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public abstract class AbstractGremlinQueryOptimizerTest implements IAtlasGraphProvider {
+
+    protected abstract GremlinExpressionFactory getFactory();
+
+    private MetadataRepository repo = new GraphBackedMetadataRepository(this, new HardDeleteHandler(TypeSystem.getInstance()));
+    private final GraphPersistenceStrategies STRATEGY = mock(GraphPersistenceStrategies.class);
+    @BeforeClass
+    public void setUp() {
+        GremlinQueryOptimizer.reset();
+        GremlinQueryOptimizer.setExpressionFactory(getFactory());
+        when(STRATEGY.typeAttributeName()).thenReturn(Constants.ENTITY_TYPE_PROPERTY_KEY);
+        when(STRATEGY.superTypeAttributeName()).thenReturn(Constants.SUPER_TYPES_PROPERTY_KEY);
+    }
+
+    private FieldInfo getTestFieldInfo() throws AtlasException {
+        AttributeDefinition def = new AttributeDefinition("foo", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null);
+        AttributeInfo attrInfo = new AttributeInfo(TypeSystem.getInstance(), def, null);
+        return new FieldInfo(DataTypes.STRING_TYPE, attrInfo, null, null);
+    }
+
+    private GroovyExpression getVerticesExpression() {
+        IdentifierExpression g = new IdentifierExpression("g");
+        return new FunctionCallExpression(TraversalStepType.START, g, "V");
+    }
+
+
+    @Test
+    public void testPullHasExpressionsOutOfAnd() throws AtlasException {
+
+        GroovyExpression expr1 = makeOutExpression(null, "out1");
+        GroovyExpression expr2 = makeOutExpression(null, "out2");
+        GroovyExpression expr3 = makeHasExpression("prop1","Fred");
+        GroovyExpression expr4 = makeHasExpression("prop2","George");
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "and", Arrays.asList(expr1, expr2, expr3, expr4));
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestPullHasExpressionsOutOfHas());
+    }
+
+    protected abstract String getExpectedGremlinForTestPullHasExpressionsOutOfHas();
+
+
+    @Test
+    public void testOrGrouping() throws AtlasException {
+        GroovyExpression expr1 = makeOutExpression(null, "out1");
+        GroovyExpression expr2 = makeOutExpression(null, "out2");
+        GroovyExpression expr3 = makeHasExpression("prop1","Fred");
+        GroovyExpression expr4 = makeHasExpression("prop2","George");
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "or", Arrays.asList(expr1, expr2, expr3, expr4));
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestOrGrouping());
+    }
+
+    protected abstract String getExpectedGremlinForTestOrGrouping();
+
+
+    @Test
+    public void testAndOfOrs() throws AtlasException {
+
+        GroovyExpression or1Cond1 = makeHasExpression("p1","e1");
+        GroovyExpression or1Cond2 = makeHasExpression("p2","e2");
+        GroovyExpression or2Cond1 = makeHasExpression("p3","e3");
+        GroovyExpression or2Cond2 = makeHasExpression("p4","e4");
+
+        GroovyExpression or1 = getFactory().generateLogicalExpression(null, "or", Arrays.asList(or1Cond1, or1Cond2));
+        GroovyExpression or2 = getFactory().generateLogicalExpression(null, "or", Arrays.asList(or2Cond1, or2Cond2));
+        GroovyExpression toOptimize  = getFactory().generateLogicalExpression(getVerticesExpression(), "and", Arrays.asList(or1, or2));
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAndOfOrs());
+
+    }
+
+    protected abstract String getExpectedGremlinForTestAndOfOrs();
+
+    @Test
+    public void testAndWithMultiCallArguments() throws AtlasException {
+
+        GroovyExpression cond1 = makeHasExpression("p1","e1");
+        GroovyExpression cond2 = makeHasExpression(cond1, "p2","e2");
+        GroovyExpression cond3 = makeHasExpression("p3","e3");
+        GroovyExpression cond4 = makeHasExpression(cond3, "p4","e4");
+
+        GroovyExpression toOptimize  = getFactory().generateLogicalExpression(getVerticesExpression(), "and", Arrays.asList(cond2, cond4));
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAndWithMultiCallArguments());
+    }
+
+
+    protected abstract String getExpectedGremlinForTestAndWithMultiCallArguments();
+
+    @Test
+    public void testOrOfAnds() throws AtlasException {
+
+        GroovyExpression or1Cond1 = makeHasExpression("p1","e1");
+        GroovyExpression or1Cond2 = makeHasExpression("p2","e2");
+        GroovyExpression or2Cond1 = makeHasExpression("p3","e3");
+        GroovyExpression or2Cond2 = makeHasExpression("p4","e4");
+
+        GroovyExpression or1 = getFactory().generateLogicalExpression(null, "and", Arrays.asList(or1Cond1, or1Cond2));
+        GroovyExpression or2 = getFactory().generateLogicalExpression(null, "and", Arrays.asList(or2Cond1, or2Cond2));
+        GroovyExpression toOptimize  = getFactory().generateLogicalExpression(getVerticesExpression(), "or", Arrays.asList(or1, or2));
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestOrOfAnds());
+    }
+
+    protected abstract String getExpectedGremlinForTestOrOfAnds();
+
+    @Test
+    public void testHasNotMovedToResult() throws AtlasException {
+        GroovyExpression toOptimize = getVerticesExpression();
+        GroovyExpression or1Cond1 = makeHasExpression("p1","e1");
+        GroovyExpression or1Cond2 = makeHasExpression("p2","e2");
+
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(or1Cond1, or1Cond2));
+        toOptimize = makeHasExpression(toOptimize, "p3","e3");
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "_src");
+        toOptimize = getFactory().generateSelectExpression(toOptimize, Collections.singletonList(new LiteralExpression("src1")), Collections.<GroovyExpression>singletonList(new IdentifierExpression("it")));
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(),
+                getExpectedGremlinForTestHasNotMovedToResult());
+    }
+
+    protected abstract String getExpectedGremlinForTestHasNotMovedToResult();
+
+    @Test
+    public void testOptimizeLoopExpression() throws AtlasException {
+
+
+        GroovyExpression input = getVerticesExpression();
+        input = getFactory().generateTypeTestExpression(STRATEGY,  input, "DataSet", TestIntSequence.INSTANCE).get(0);
+        input = makeHasExpression(input, "name","Fred");
+        input = getFactory().generateAliasExpression(input, "label");
+
+
+        GroovyExpression loopExpr = getFactory().getLoopExpressionParent(input);
+        loopExpr = getFactory().generateAdjacentVerticesExpression(loopExpr, AtlasEdgeDirection.IN, "inputTables");
+        loopExpr = getFactory().generateAdjacentVerticesExpression(loopExpr, AtlasEdgeDirection.OUT, "outputTables");
+        GroovyExpression result = getFactory().generateLoopExpression(input, STRATEGY, DataTypes.STRING_TYPE, loopExpr, "label", null);
+        result = getFactory().generateToListExpression(result);
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(result);
+
+        assertEquals(optimized.toString(), getExpectedGremlinForOptimizeLoopExpression());
+    }
+
+    protected abstract String getExpectedGremlinForOptimizeLoopExpression();
+
+    @Test
+    public void testLongStringEndingWithOr() throws AtlasException {
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = makeHasExpression(toOptimize, "name","Fred");
+        toOptimize = makeHasExpression(toOptimize, "age","13");
+        toOptimize = makeOutExpression(toOptimize, "livesIn");
+        toOptimize = makeHasExpression(toOptimize, "state","Massachusetts");
+
+        GroovyExpression or1cond1 = makeHasExpression("p1", "e1");
+        GroovyExpression or1cond2 = makeHasExpression("p2", "e2");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(or1cond1, or1cond2));
+
+        GroovyExpression or2cond1 = makeHasExpression("p3", "e3");
+        GroovyExpression or2cond2 = makeHasExpression("p4", "e4");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(or2cond1, or2cond2));
+        toOptimize = makeHasExpression(toOptimize, "p5","e5");
+        toOptimize = makeHasExpression(toOptimize, "p6","e6");
+        GroovyExpression or3cond1 = makeHasExpression("p7", "e7");
+        GroovyExpression or3cond2 = makeHasExpression("p8", "e8");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(or3cond1, or3cond2));
+
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestLongStringEndingWithOr());
+    }
+
+    protected abstract String getExpectedGremlinForTestLongStringEndingWithOr();
+
+    @Test
+    public void testLongStringNotEndingWithOr() throws AtlasException {
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = makeHasExpression(toOptimize, "name","Fred");
+        toOptimize = makeHasExpression(toOptimize, "age","13");
+        toOptimize = makeOutExpression(toOptimize, "livesIn");
+        toOptimize = makeHasExpression(toOptimize, "state","Massachusetts");
+
+        GroovyExpression or1cond1 = makeHasExpression("p1", "e1");
+        GroovyExpression or1cond2 = makeHasExpression("p2", "e2");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(or1cond1, or1cond2));
+
+        GroovyExpression or2cond1 = makeHasExpression("p3", "e3");
+        GroovyExpression or2cond2 = makeHasExpression("p4", "e4");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(or2cond1, or2cond2));
+        toOptimize = makeHasExpression(toOptimize, "p5","e5");
+        toOptimize = makeHasExpression(toOptimize, "p6","e6");
+        GroovyExpression or3cond1 = makeHasExpression("p7", "e7");
+        GroovyExpression or3cond2 = makeHasExpression("p8", "e8");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(or3cond1, or3cond2));
+        toOptimize = makeHasExpression(toOptimize, "p9","e9");
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestLongStringNotEndingWithOr());
+    }
+
+    protected abstract String getExpectedGremlinForTestLongStringNotEndingWithOr();
+
+    @Test
+    public void testToListConversion() throws AtlasException {
+
+        GroovyExpression expr1 = makeHasExpression("prop1","Fred");
+        GroovyExpression expr2 = makeHasExpression("prop2","George");
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "or", Arrays.asList(expr1, expr2));
+        toOptimize = new FunctionCallExpression(TraversalStepType.END, toOptimize,"toList");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestToListConversion());
+    }
+
+    protected abstract String getExpectedGremlinForTestToListConversion();
+
+    @Test
+    public void testToListWithExtraStuff() throws AtlasException {
+
+        GroovyExpression expr1 = makeHasExpression("prop1","Fred");
+        GroovyExpression expr2 = makeHasExpression("prop2","George");
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "or", Arrays.asList(expr1, expr2));
+        toOptimize = new FunctionCallExpression(TraversalStepType.END, toOptimize,"toList");
+        toOptimize = new FunctionCallExpression(toOptimize,"size");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestToListWithExtraStuff());
+
+    }
+
+    protected abstract String getExpectedGremlinForTestToListWithExtraStuff();
+
+    public void testAddClosureWithExitExpressionDifferentFromExpr() throws AtlasException {
+
+        GroovyExpression expr1 = makeHasExpression("prop1","Fred");
+        GroovyExpression expr2 = makeHasExpression("prop2","George");
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "or", Arrays.asList(expr1, expr2));
+        toOptimize = makeOutExpression(toOptimize, "knows");
+        toOptimize = makeOutExpression(toOptimize, "livesIn");
+        toOptimize = new FunctionCallExpression(TraversalStepType.END, toOptimize,"toList");
+        toOptimize = new FunctionCallExpression(toOptimize,"size");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAddClosureWithExitExpressionDifferentFromExpr());
+
+    }
+
+    protected abstract String getExpectedGremlinForTestAddClosureWithExitExpressionDifferentFromExpr();
+
+    @Test
+    public void testAddClosureNoExitExpression() throws AtlasException {
+
+        GroovyExpression expr1 = makeHasExpression("prop1","Fred");
+        GroovyExpression expr2 = makeHasExpression("prop2","George");
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "or", Arrays.asList(expr1, expr2));
+        toOptimize = makeOutExpression(toOptimize, "knows");
+        toOptimize = makeOutExpression(toOptimize, "livesIn");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAddClosureNoExitExpression());
+    }
+
+    protected abstract String getExpectedGremlinForTestAddClosureNoExitExpression();
+
+
+    private GroovyExpression makeOutExpression(GroovyExpression parent, String label) {
+        return getFactory().generateAdjacentVerticesExpression(parent, AtlasEdgeDirection.OUT, label);
+    }
+
+    @Test
+    public void testAddClosureWithExitExpressionEqualToExpr() throws AtlasException {
+
+        GroovyExpression expr1 = makeHasExpression("prop1","Fred");
+        GroovyExpression expr2 = makeHasExpression("prop2","George");
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "or", Arrays.asList(expr1, expr2));
+
+        toOptimize = makeOutExpression(toOptimize, "knows");
+        toOptimize = makeOutExpression(toOptimize, "livesIn");
+        toOptimize = new FunctionCallExpression(TraversalStepType.END, toOptimize,"toList");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAddClosureWithExitExpressionEqualToExpr());
+    }
+
+    protected abstract String getExpectedGremlinForTestAddClosureWithExitExpressionEqualToExpr();
+
+
+    @Test
+    public void testClosureNotCreatedWhenNoOrs() throws AtlasException {
+
+        GroovyExpression expr1 = makeHasExpression("prop1","Fred");
+        GroovyExpression expr2 = makeHasExpression("prop2","George");
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "and", Arrays.asList(expr1, expr2));
+        toOptimize = makeOutExpression(toOptimize, "knows");
+        toOptimize = makeOutExpression(toOptimize, "livesIn");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestClosureNotCreatedWhenNoOrs());
+    }
+
+    protected abstract String getExpectedGremlinForTestClosureNotCreatedWhenNoOrs();
+
+
+    private GroovyExpression makeHasExpression(String name, String value) throws AtlasException {
+        return makeHasExpression(null, name, value);
+    }
+    private GroovyExpression makeHasExpression(GroovyExpression parent, String name, String value) throws AtlasException {
+        return getFactory().generateHasExpression(STRATEGY, parent, name, "=", new LiteralExpression(value), getTestFieldInfo());
+    }
+    private GroovyExpression makeFieldExpression(GroovyExpression parent, String fieldName) throws AtlasException {
+        return getFactory().generateFieldExpression(parent, getTestFieldInfo(), fieldName, false);
+    }
+
+    @Test
+    public void testOrFollowedByAnd() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression("name","George");
+        GroovyExpression expr3 = makeHasExpression("age","13");
+        GroovyExpression expr4 = makeHasExpression("age","14");
+
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "or", Arrays.asList(expr1,expr2));
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "and", Arrays.asList(expr3, expr4));
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestOrFollowedByAnd());
+    }
+
+    protected abstract String getExpectedGremlinForTestOrFollowedByAnd();
+
+    @Test
+    public void testOrFollowedByOr() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression("name","George");
+        GroovyExpression expr3 = makeHasExpression("age","13");
+        GroovyExpression expr4 = makeHasExpression("age","14");
+
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "or", Arrays.asList(expr1,expr2));
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr3, expr4));
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestOrFollowedByOr());
+    }
+
+    protected abstract String getExpectedGremlinForTestOrFollowedByOr();
+
+    @Test
+    public void testMassiveOrExpansion() throws AtlasException {
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = makeHasExpression(toOptimize, "h1","h2");
+        toOptimize = makeHasExpression(toOptimize, "h3","h4");
+        for(int i = 0; i < 5; i++) {
+            GroovyExpression expr1 = makeHasExpression("p1" + i,"e1" + i);
+            GroovyExpression expr2 = makeHasExpression("p2" + i,"e2" + i);
+            toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr1,expr2));
+            toOptimize = makeHasExpression(toOptimize, "ha" + i,"hb" + i);
+            toOptimize = makeHasExpression(toOptimize, "hc" + i,"hd" + i);
+        }
+        toOptimize = makeHasExpression(toOptimize, "h5","h6");
+        toOptimize = makeHasExpression(toOptimize, "h7","h8");
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestMassiveOrExpansion());
+    }
+
+    protected abstract String getExpectedGremlinForTestMassiveOrExpansion();
+
+    @Test
+    public void testAndFollowedByAnd() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression("name","George");
+        GroovyExpression expr3 = makeHasExpression("age","13");
+        GroovyExpression expr4 = makeHasExpression("age","14");
+
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "and", Arrays.asList(expr1,expr2));
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "and", Arrays.asList(expr3, expr4));
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAndFollowedByAnd());
+
+
+    }
+
+    protected abstract String getExpectedGremlinForTestAndFollowedByAnd();
+
+    @Test
+    public void testAndFollowedByOr() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression("name","George");
+        GroovyExpression expr3 = makeHasExpression("age","13");
+        GroovyExpression expr4 = makeHasExpression("age","14");
+
+        GroovyExpression toOptimize = getFactory().generateLogicalExpression(getVerticesExpression(), "and", Arrays.asList(expr1,expr2));
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr3, expr4));
+
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAndFollowedByOr());
+    }
+
+    protected abstract String getExpectedGremlinForTestAndFollowedByOr();
+
+    @Test
+    public void testInitialAlias() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression("name","George");
+
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "x");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr1, expr2));
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestInitialAlias());
+    }
+
+    protected abstract String getExpectedGremlinForTestInitialAlias();
+
+    @Test
+    public void testFinalAlias() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression("name","George");
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr1, expr2));
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "x");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestFinalAlias());
+    }
+
+    protected abstract String getExpectedGremlinForTestFinalAlias();
+
+    @Test
+    public void testAliasInMiddle() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression("name","George");
+        GroovyExpression expr3 = makeHasExpression("age","13");
+        GroovyExpression expr4 = makeHasExpression("age","14");
+
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr1, expr2));
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "x");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr3, expr4));
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAliasInMiddle());
+    }
+
+    protected abstract String getExpectedGremlinForTestAliasInMiddle();
+
+    @Test
+    public void testMultipleAliases() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression("name","George");
+        GroovyExpression expr3 = makeHasExpression("age","13");
+        GroovyExpression expr4 = makeHasExpression("age","14");
+
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr1, expr2));
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "x");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr3, expr4));
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "y");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGreminForTestMultipleAliases());
+    }
+
+    protected abstract String getExpectedGreminForTestMultipleAliases();
+
+    @Test
+    public void testAliasInOrExpr() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = getFactory().generateAliasExpression(makeHasExpression("name","George"), "george");
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr1, expr2));
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAliasInOrExpr());
+    }
+
+    protected abstract String getExpectedGremlinForTestAliasInOrExpr();
+
+    @Test
+    public void testAliasInAndExpr() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = getFactory().generateAliasExpression(makeHasExpression("name","George"), "george");
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "and", Arrays.asList(expr1, expr2));
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        //expression with alias cannot currently be pulled out of the and
+        assertEquals(optimized.toString(), getExpectedGremlinForTestAliasInAndExpr());
+    }
+
+
+    protected abstract String getExpectedGremlinForTestAliasInAndExpr();
+    @Test
+    public void testFlatMapExprInAnd() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression(makeOutExpression(null,"knows"), "name","George");
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "and", Arrays.asList(expr1, expr2));
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestFlatMapExprInAnd());
+    }
+
+
+    protected abstract String getExpectedGremlinForTestFlatMapExprInAnd();
+    @Test
+    public void testFlatMapExprInOr() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression(makeOutExpression(null,"knows"), "name","George");
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr1, expr2));
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestFlatMapExprInOr());
+    }
+
+    protected abstract String getExpectedGremlinForTestFlatMapExprInOr();
+
+    @Test
+    public void testFieldExpressionPushedToResultExpression() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression("name","Fred");
+        GroovyExpression expr2 = makeHasExpression(makeOutExpression(null,"knows"), "name","George");
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr1, expr2));
+        toOptimize = makeFieldExpression(toOptimize, "name");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestFieldExpressionPushedToResultExpression());
+    }
+
+    protected abstract String getExpectedGremlinForTestFieldExpressionPushedToResultExpression();
+
+    @Test
+    public void testOrWithNoChildren() throws AtlasException {
+        GroovyExpression toOptimize = getVerticesExpression();
+        GroovyExpression expr1 = makeHasExpression(toOptimize, "name","Fred");
+
+        toOptimize = getFactory().generateLogicalExpression(expr1, "or", Collections.<GroovyExpression>emptyList());
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        //or with no children matches no vertices
+        assertEquals(optimized.toString(), getExpectedGremlinFortestOrWithNoChildren());
+    }
+
+    protected abstract String getExpectedGremlinFortestOrWithNoChildren();
+
+    @Test
+    public void testFinalAliasNeeded() throws AtlasException {
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = makeHasExpression(toOptimize, "name", "Fred");
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "person");
+        toOptimize = makeOutExpression(toOptimize, "livesIn");
+        GroovyExpression isChicago = makeHasExpression(null, "name", "Chicago");
+        GroovyExpression isBoston = makeHasExpression(null, "name", "Boston");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(isChicago, isBoston));
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "city");
+        toOptimize = makeOutExpression(toOptimize, "state");
+        toOptimize = makeHasExpression(toOptimize, "name", "Massachusetts");
+        toOptimize = getFactory().generatePathExpression(toOptimize);
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestFinalAliasNeeded());
+    }
+
+    protected abstract String getExpectedGremlinForTestFinalAliasNeeded();
+
+    @Test
+    public void testSimpleRangeExpression() throws AtlasException {
+        GroovyExpression expr1 = makeHasExpression(null, "name","Fred");
+        GroovyExpression expr2 = makeHasExpression(null, "name","George");
+        GroovyExpression expr3 = makeHasExpression(null, "age","34");
+        GroovyExpression expr4 = makeHasExpression(null, "size","small");
+
+        GroovyExpression toOptimize = getVerticesExpression();
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr1, expr2));
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "and", Collections.singletonList(expr3));
+        toOptimize = getFactory().generateAdjacentVerticesExpression(toOptimize, AtlasEdgeDirection.OUT, "eats");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "and", Collections.singletonList(expr4));
+        toOptimize = makeHasExpression(toOptimize, "color","blue");
+        toOptimize = getFactory().generateRangeExpression(toOptimize, 0, 10);
+        toOptimize = new FunctionCallExpression(TraversalStepType.END, toOptimize, "toList");
+        toOptimize = new FunctionCallExpression(toOptimize, "size");
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestSimpleRangeExpression());
+    }
+
+    protected abstract String getExpectedGremlinForTestSimpleRangeExpression();
+
+
+    @Test
+    public void testRangeWithNonZeroOffset() throws Exception {
+        // g.V().or(has('__typeName','OMAS_OMRSAsset'),has('__superTypeNames','OMAS_OMRSAsset')).range(5,10).as('inst').select('inst')
+        GroovyExpression toOptimize = getVerticesExpression();
+
+        GroovyExpression expr0 = makeHasExpression("__typeName", "OMAS_OMRSAsset");
+        GroovyExpression expr1 = makeHasExpression("__superTypeNames", "OMAS_OMRSAsset");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr0, expr1));
+        toOptimize = getFactory().generateRangeExpression(toOptimize, 5, 10);
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "inst");
+        toOptimize = getFactory().generateSelectExpression(toOptimize, Collections.singletonList(new LiteralExpression("inst")), Collections.<GroovyExpression>emptyList());
+        RangeFinder visitor = new RangeFinder(getFactory());
+        GremlinQueryOptimizer.visitCallHierarchy(toOptimize, visitor);
+        List<AbstractFunctionExpression> rangeExpressions = visitor.getRangeExpressions();
+        assertEquals(rangeExpressions.size(), 1);
+        int[] rangeParameters = getFactory().getRangeParameters(rangeExpressions.get(0));
+        assertNotNull(rangeParameters);
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        // The range optimization is not supported with a non-zero start index, so the optimizer should not add range expressions
+        // to the expanded or's.
+        assertEquals(optimized.toString(), getExpectedGremlinForTestRangeWithNonZeroOffset());
+    }
+
+    protected abstract String getExpectedGremlinForTestRangeWithNonZeroOffset();
+
+    @Test
+    public void testRangeWithOrderBy() throws Exception {
+        // The range optimization is not supported with order, so the optimizer should not add range expressions
+        // to the expanded or's.
+        GroovyExpression toOptimize = getVerticesExpression();
+
+        GroovyExpression expr0 = makeHasExpression("__typeName", "OMAS_OMRSAsset");
+        GroovyExpression expr1 = makeHasExpression("__superTypeNames", "OMAS_OMRSAsset");
+        toOptimize = getFactory().generateLogicalExpression(toOptimize, "or", Arrays.asList(expr0, expr1));
+        toOptimize = getFactory().generateRangeExpression(toOptimize, 5, 10);
+        toOptimize = getFactory().generateAliasExpression(toOptimize, "inst");
+        //toOptimize = getFactory().generateSelectExpression(toOptimize, Collections.singletonList(new LiteralExpression("inst")), Collections.<GroovyExpression>emptyList());
+        GroovyExpression orderFielda = makeFieldExpression(getFactory().getCurrentTraverserObject(getFactory().getClosureArgumentValue()), "name");
+        GroovyExpression orderFieldb = makeFieldExpression(getFactory().getCurrentTraverserObject(getFactory().getClosureArgumentValue()), "name");
+        toOptimize = getFactory().generateOrderByExpression(toOptimize,Arrays.asList(orderFielda, orderFieldb), true);
+        RangeFinder visitor = new RangeFinder(getFactory());
+        GremlinQueryOptimizer.visitCallHierarchy(toOptimize, visitor);
+        List<AbstractFunctionExpression> rangeExpressions = visitor.getRangeExpressions();
+        assertEquals(rangeExpressions.size(), 1);
+        int[] rangeParameters = getFactory().getRangeParameters(rangeExpressions.get(0));
+        assertNotNull(rangeParameters);
+        GroovyExpression optimized = GremlinQueryOptimizer.getInstance().optimize(toOptimize);
+        assertEquals(optimized.toString(), getExpectedGremlinForTestRangeWithOrderBy());
+    }
+
+
+
+    protected abstract String getExpectedGremlinForTestRangeWithOrderBy();
+    @Override
+    public AtlasGraph get() throws RepositoryException {
+        AtlasGraph graph = mock(AtlasGraph.class);
+        when(graph.getSupportedGremlinVersion()).thenReturn(GremlinVersion.THREE);
+        when(graph.isPropertyValueConversionNeeded(any(IDataType.class))).thenReturn(false);
+        return graph;
+    }
+}

repository/src/test/java/org/apache/atlas/repository/graph/Gremlin2QueryOptimizerTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import org.apache.atlas.gremlin.Gremlin2ExpressionFactory;
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.testng.annotations.Test;
+
+
+@Test
+public class Gremlin2QueryOptimizerTest extends AbstractGremlinQueryOptimizerTest {
+
+
+    private static final GremlinExpressionFactory FACTORY = new Gremlin2ExpressionFactory();
+
+    @Override
+    protected GremlinExpressionFactory getFactory() {
+        return FACTORY;
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestPullHasExpressionsOutOfHas() {
+        return "g.V().has('prop1',T.'eq','Fred').has('prop2',T.'eq','George').and(out('out1'),out('out2'))";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestOrGrouping() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',T.'eq','Fred').fill(r);"
+                + "g.V().has('prop2',T.'eq','George').fill(r);"
+                + "g.V().or(out('out1'),out('out2')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAndOfOrs() {
+        return  "def r=(([]) as Set);"
+                + "g.V().has('p1',T.'eq','e1').has('p3',T.'eq','e3').fill(r);"
+                + "g.V().has('p1',T.'eq','e1').has('p4',T.'eq','e4').fill(r);"
+                + "g.V().has('p2',T.'eq','e2').has('p3',T.'eq','e3').fill(r);"
+                + "g.V().has('p2',T.'eq','e2').has('p4',T.'eq','e4').fill(r);"
+                + "r";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestAndWithMultiCallArguments() {
+        return "g.V().has('p1',T.'eq','e1').has('p2',T.'eq','e2').has('p3',T.'eq','e3').has('p4',T.'eq','e4')";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestOrOfAnds() {
+
+        return "def r=(([]) as Set);"
+                + "g.V().has('p1',T.'eq','e1').has('p2',T.'eq','e2').fill(r);"
+                + "g.V().has('p3',T.'eq','e3').has('p4',T.'eq','e4').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestHasNotMovedToResult() {
+        return "def r=(([]) as Set);"
+                + "def f1={GremlinPipeline x->x.has('p3',T.'eq','e3').as('_src').select(['_src']).fill(r)};"
+                + "f1(g.V().has('p1',T.'eq','e1'));"
+                + "f1(g.V().has('p2',T.'eq','e2'));"
+                + "r._().transform({((Row)it).getColumn('_src')}).as('_src').select(['src1'],{it})";
+    }
+
+    @Override
+    protected String getExpectedGremlinForOptimizeLoopExpression() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('__typeName','DataSet').has('name',T.'eq','Fred').fill(r);"
+                + "g.V().has('__superTypeNames','DataSet').has('name',T.'eq','Fred').fill(r);"
+                + "r._().as('label').in('inputTables').out('outputTables').loop('label',{((it.'path'.contains(it.'object'))?(false):(true))},{it.'object'.'__typeName' == 'string' || ((it.'object'.'__superTypeNames')?(it.'object'.'__superTypeNames'.contains('string')):(false))}).enablePath().toList()";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestLongStringEndingWithOr() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('name',T.'eq','Fred').has('age',T.'eq','13').out('livesIn').has('state',T.'eq','Massachusetts')};"
+                + "def f2={GremlinPipeline x->x.has('p5',T.'eq','e5').has('p6',T.'eq','e6')};"
+                + "f2(f1().has('p1',T.'eq','e1').has('p3',T.'eq','e3')).has('p7',T.'eq','e7').fill(r);"
+                + "f2(f1().has('p1',T.'eq','e1').has('p3',T.'eq','e3')).has('p8',T.'eq','e8').fill(r);"
+                + "f2(f1().has('p1',T.'eq','e1').has('p4',T.'eq','e4')).has('p7',T.'eq','e7').fill(r);"
+                + "f2(f1().has('p1',T.'eq','e1').has('p4',T.'eq','e4')).has('p8',T.'eq','e8').fill(r);"
+                + "f2(f1().has('p2',T.'eq','e2').has('p3',T.'eq','e3')).has('p7',T.'eq','e7').fill(r);"
+                + "f2(f1().has('p2',T.'eq','e2').has('p3',T.'eq','e3')).has('p8',T.'eq','e8').fill(r);"
+                + "f2(f1().has('p2',T.'eq','e2').has('p4',T.'eq','e4')).has('p7',T.'eq','e7').fill(r);"
+                + "f2(f1().has('p2',T.'eq','e2').has('p4',T.'eq','e4')).has('p8',T.'eq','e8').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestLongStringNotEndingWithOr() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('name',T.'eq','Fred').has('age',T.'eq','13').out('livesIn').has('state',T.'eq','Massachusetts')};"
+                + "def f2={GremlinPipeline x->x.has('p5',T.'eq','e5').has('p6',T.'eq','e6')};"
+                + "def f3={GremlinPipeline x->x.has('p9',T.'eq','e9').fill(r)};"
+                + "f3(f2(f1().has('p1',T.'eq','e1').has('p3',T.'eq','e3')).has('p7',T.'eq','e7'));"
+                + "f3(f2(f1().has('p1',T.'eq','e1').has('p3',T.'eq','e3')).has('p8',T.'eq','e8'));"
+                + "f3(f2(f1().has('p1',T.'eq','e1').has('p4',T.'eq','e4')).has('p7',T.'eq','e7'));"
+                + "f3(f2(f1().has('p1',T.'eq','e1').has('p4',T.'eq','e4')).has('p8',T.'eq','e8'));"
+                + "f3(f2(f1().has('p2',T.'eq','e2').has('p3',T.'eq','e3')).has('p7',T.'eq','e7'));"
+                + "f3(f2(f1().has('p2',T.'eq','e2').has('p3',T.'eq','e3')).has('p8',T.'eq','e8'));"
+                + "f3(f2(f1().has('p2',T.'eq','e2').has('p4',T.'eq','e4')).has('p7',T.'eq','e7'));"
+                + "f3(f2(f1().has('p2',T.'eq','e2').has('p4',T.'eq','e4')).has('p8',T.'eq','e8'));"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestToListConversion() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',T.'eq','Fred').fill(r);"
+                + "g.V().has('prop2',T.'eq','George').fill(r);"
+                + "r._().toList()";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestToListWithExtraStuff() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',T.'eq','Fred').fill(r);"
+                + "g.V().has('prop2',T.'eq','George').fill(r);"
+                + "r._().toList().size()";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestAddClosureWithExitExpressionDifferentFromExpr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',T.'eq','Fred').out('knows').out('livesIn').fill(r);"
+                + "g.V().has('prop2',T.'eq','George').out('knows').out('livesIn').fill(r);"
+                + "r._().toList().size()";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAddClosureNoExitExpression() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',T.'eq','Fred').out('knows').out('livesIn').fill(r);"
+                + "g.V().has('prop2',T.'eq','George').out('knows').out('livesIn').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAddClosureWithExitExpressionEqualToExpr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',T.'eq','Fred').out('knows').out('livesIn').fill(r);"
+                + "g.V().has('prop2',T.'eq','George').out('knows').out('livesIn').fill(r);"
+                + "r._().toList()";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestClosureNotCreatedWhenNoOrs() {
+        return "g.V().has('prop1',T.'eq','Fred').has('prop2',T.'eq','George').out('knows').out('livesIn')";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestOrFollowedByAnd() {
+        return "def r=(([]) as Set);"
+                + "def f1={GremlinPipeline x->x.has('age',T.'eq','13').has('age',T.'eq','14').fill(r)};"
+                + "f1(g.V().has('name',T.'eq','Fred'));"
+                + "f1(g.V().has('name',T.'eq','George'));"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestOrFollowedByOr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',T.'eq','Fred').has('age',T.'eq','13').fill(r);"
+                + "g.V().has('name',T.'eq','Fred').has('age',T.'eq','14').fill(r);"
+                + "g.V().has('name',T.'eq','George').has('age',T.'eq','13').fill(r);"
+                + "g.V().has('name',T.'eq','George').has('age',T.'eq','14').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestMassiveOrExpansion() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('h1',T.'eq','h2').has('h3',T.'eq','h4')};"
+                + "def f2={GremlinPipeline x->x.has('ha0',T.'eq','hb0').has('hc0',T.'eq','hd0')};"
+                + "def f3={GremlinPipeline x->x.has('ha1',T.'eq','hb1').has('hc1',T.'eq','hd1')};"
+                + "def f4={GremlinPipeline x->x.has('ha2',T.'eq','hb2').has('hc2',T.'eq','hd2')};"
+                + "def f5={GremlinPipeline x->x.has('ha3',T.'eq','hb3').has('hc3',T.'eq','hd3')};"
+                + "def f6={GremlinPipeline x->x.has('ha4',T.'eq','hb4').has('hc4',T.'eq','hd4').has('h5',T.'eq','h6').has('h7',T.'eq','h8').fill(r)};"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p11',T.'eq','e11')).has('p12',T.'eq','e12')).has('p13',T.'eq','e13')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p11',T.'eq','e11')).has('p12',T.'eq','e12')).has('p13',T.'eq','e13')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p11',T.'eq','e11')).has('p12',T.'eq','e12')).has('p23',T.'eq','e23')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p11',T.'eq','e11')).has('p12',T.'eq','e12')).has('p23',T.'eq','e23')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p11',T.'eq','e11')).has('p22',T.'eq','e22')).has('p13',T.'eq','e13')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p11',T.'eq','e11')).has('p22',T.'eq','e22')).has('p13',T.'eq','e13')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p11',T.'eq','e11')).has('p22',T.'eq','e22')).has('p23',T.'eq','e23')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p11',T.'eq','e11')).has('p22',T.'eq','e22')).has('p23',T.'eq','e23')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p21',T.'eq','e21')).has('p12',T.'eq','e12')).has('p13',T.'eq','e13')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p21',T.'eq','e21')).has('p12',T.'eq','e12')).has('p13',T.'eq','e13')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p21',T.'eq','e21')).has('p12',T.'eq','e12')).has('p23',T.'eq','e23')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p21',T.'eq','e21')).has('p12',T.'eq','e12')).has('p23',T.'eq','e23')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p21',T.'eq','e21')).has('p22',T.'eq','e22')).has('p13',T.'eq','e13')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p21',T.'eq','e21')).has('p22',T.'eq','e22')).has('p13',T.'eq','e13')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p21',T.'eq','e21')).has('p22',T.'eq','e22')).has('p23',T.'eq','e23')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',T.'eq','e10')).has('p21',T.'eq','e21')).has('p22',T.'eq','e22')).has('p23',T.'eq','e23')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p11',T.'eq','e11')).has('p12',T.'eq','e12')).has('p13',T.'eq','e13')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p11',T.'eq','e11')).has('p12',T.'eq','e12')).has('p13',T.'eq','e13')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p11',T.'eq','e11')).has('p12',T.'eq','e12')).has('p23',T.'eq','e23')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p11',T.'eq','e11')).has('p12',T.'eq','e12')).has('p23',T.'eq','e23')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p11',T.'eq','e11')).has('p22',T.'eq','e22')).has('p13',T.'eq','e13')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p11',T.'eq','e11')).has('p22',T.'eq','e22')).has('p13',T.'eq','e13')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p11',T.'eq','e11')).has('p22',T.'eq','e22')).has('p23',T.'eq','e23')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p11',T.'eq','e11')).has('p22',T.'eq','e22')).has('p23',T.'eq','e23')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p21',T.'eq','e21')).has('p12',T.'eq','e12')).has('p13',T.'eq','e13')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p21',T.'eq','e21')).has('p12',T.'eq','e12')).has('p13',T.'eq','e13')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p21',T.'eq','e21')).has('p12',T.'eq','e12')).has('p23',T.'eq','e23')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p21',T.'eq','e21')).has('p12',T.'eq','e12')).has('p23',T.'eq','e23')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p21',T.'eq','e21')).has('p22',T.'eq','e22')).has('p13',T.'eq','e13')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p21',T.'eq','e21')).has('p22',T.'eq','e22')).has('p13',T.'eq','e13')).has('p24',T.'eq','e24'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p21',T.'eq','e21')).has('p22',T.'eq','e22')).has('p23',T.'eq','e23')).has('p14',T.'eq','e14'));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',T.'eq','e20')).has('p21',T.'eq','e21')).has('p22',T.'eq','e22')).has('p23',T.'eq','e23')).has('p24',T.'eq','e24'));"
+                + "r";
+
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAndFollowedByAnd() {
+        return "g.V().has('name',T.'eq','Fred').has('name',T.'eq','George').has('age',T.'eq','13').has('age',T.'eq','14')";
+
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAndFollowedByOr() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('name',T.'eq','Fred').has('name',T.'eq','George')};f1().has('age',T.'eq','13').fill(r);"
+                + "f1().has('age',T.'eq','14').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestInitialAlias() {
+        return "def r=(([]) as Set);"
+                + "g.V().as('x').has('name',T.'eq','Fred').fill(r);"
+                + "g.V().as('x').has('name',T.'eq','George').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestFinalAlias() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',T.'eq','Fred').as('x').fill(r);"
+                + "g.V().has('name',T.'eq','George').as('x').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAliasInMiddle() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',T.'eq','Fred').as('x').has('age',T.'eq','13').fill(r);"
+                + "g.V().has('name',T.'eq','Fred').as('x').has('age',T.'eq','14').fill(r);"
+                + "g.V().has('name',T.'eq','George').as('x').has('age',T.'eq','13').fill(r);"
+                + "g.V().has('name',T.'eq','George').as('x').has('age',T.'eq','14').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGreminForTestMultipleAliases() {
+        return "def r=(([]) as Set);"
+                + "def f1={GremlinPipeline x->x.as('y').fill(r)};"
+                + "f1(g.V().has('name',T.'eq','Fred').as('x').has('age',T.'eq','13'));"
+                + "f1(g.V().has('name',T.'eq','Fred').as('x').has('age',T.'eq','14'));"
+                + "f1(g.V().has('name',T.'eq','George').as('x').has('age',T.'eq','13'));"
+                + "f1(g.V().has('name',T.'eq','George').as('x').has('age',T.'eq','14'));"
+                + "r";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestAliasInOrExpr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',T.'eq','Fred').fill(r);"
+                + "g.V().or(has('name',T.'eq','George').as('george')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAliasInAndExpr() {
+        return "g.V().has('name',T.'eq','Fred').and(has('name',T.'eq','George').as('george'))";
+    }
+    @Override
+    protected String getExpectedGremlinForTestFlatMapExprInAnd() {
+        return "g.V().has('name',T.'eq','Fred').and(out('knows').has('name',T.'eq','George'))";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestFlatMapExprInOr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',T.'eq','Fred').fill(r);"
+                + "g.V().or(out('knows').has('name',T.'eq','George')).fill(r);"
+                + "r";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestFieldExpressionPushedToResultExpression() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',T.'eq','Fred').fill(r);"
+                + "g.V().or(out('knows').has('name',T.'eq','George')).fill(r);"
+                + "r._().'name'";
+    }
+
+    @Override
+    protected String getExpectedGremlinFortestOrWithNoChildren() {
+        return "def r=(([]) as Set);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestFinalAliasNeeded() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('name',T.'eq','Fred').as('person').out('livesIn')};"
+                + "def f2={GremlinPipeline x->x.as('city').out('state').has('name',T.'eq','Massachusetts').as('__res').select(['person', 'city', '__res']).fill(r)};"
+                + "f2(f1().has('name',T.'eq','Chicago'));"
+                + "f2(f1().has('name',T.'eq','Boston'));"
+                + "r._().as('__tmp').transform({((Row)it).getColumn('person')}).as('person').back('__tmp').transform({((Row)it).getColumn('city')}).as('city').back('__tmp').transform({((Row)it).getColumn('__res')}).as('__res').path().toList().collect({it.tail()})";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestSimpleRangeExpression() {
+        return "def r=(([]) as Set);"
+                + "def f1={GremlinPipeline x->x.has('age',T.'eq','34').out('eats').has('size',T.'eq','small').has('color',T.'eq','blue') [0..<10].fill(r)};"
+                + "f1(g.V().has('name',T.'eq','Fred'));"
+                + "f1(g.V().has('name',T.'eq','George'));"
+                + "r._() [0..<10].toList().size()";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestRangeWithNonZeroOffset() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('__typeName',T.'eq','OMAS_OMRSAsset').fill(r);"
+                + "g.V().has('__superTypeNames',T.'eq','OMAS_OMRSAsset').fill(r);"
+                + "r._() [5..<10].as('inst').select(['inst'])";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestRangeWithOrderBy() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('__typeName',T.'eq','OMAS_OMRSAsset').fill(r);"
+                + "g.V().has('__superTypeNames',T.'eq','OMAS_OMRSAsset').fill(r);"
+                + "r._() [5..<10].as('inst').order({((it.'name' != null)?(it.'name'.toLowerCase()):(it.'name')) <=> ((it.'name' != null)?(it.'name'.toLowerCase()):(it.'name'))})";
+    }
+
+}

repository/src/test/java/org/apache/atlas/repository/graph/Gremlin3QueryOptimizerTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import org.apache.atlas.gremlin.Gremlin3ExpressionFactory;
+import org.apache.atlas.gremlin.GremlinExpressionFactory;
+import org.testng.annotations.Test;
+
+
+@Test
+public class Gremlin3QueryOptimizerTest extends AbstractGremlinQueryOptimizerTest {
+
+    public static final GremlinExpressionFactory FACTORY = new Gremlin3ExpressionFactory();
+
+    @Override
+    protected GremlinExpressionFactory getFactory() {
+        return FACTORY;
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestPullHasExpressionsOutOfHas() {
+        return "g.V().has('prop1',eq('Fred')).has('prop2',eq('George')).and(out('out1'),out('out2'))";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestOrGrouping() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',eq('Fred')).fill(r);"
+                + "g.V().has('prop2',eq('George')).fill(r);"
+                + "g.V().or(out('out1'),out('out2')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAndOfOrs() {
+
+        return "def r=(([]) as Set);"
+                + "g.V().has('p1',eq('e1')).has('p3',eq('e3')).fill(r);"
+                + "g.V().has('p1',eq('e1')).has('p4',eq('e4')).fill(r);"
+                + "g.V().has('p2',eq('e2')).has('p3',eq('e3')).fill(r);"
+                + "g.V().has('p2',eq('e2')).has('p4',eq('e4')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAndWithMultiCallArguments() {
+
+        return "g.V().has('p1',eq('e1')).has('p2',eq('e2')).has('p3',eq('e3')).has('p4',eq('e4'))";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestOrOfAnds() {
+        return  "def r=(([]) as Set);"
+                + "g.V().has('p1',eq('e1')).has('p2',eq('e2')).fill(r);"
+                + "g.V().has('p3',eq('e3')).has('p4',eq('e4')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestHasNotMovedToResult() {
+        return "def r=(([]) as Set);"
+                + "def f1={GraphTraversal x->x.has('p3',eq('e3')).as('_src').select('_src').fill(r)};"
+                + "f1(g.V().has('p1',eq('e1')));f1(g.V().has('p2',eq('e2')));"
+                + "g.V('').inject(((r) as Vertex[])).as('_src').select('src1').by((({it}) as Function))";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestLongStringEndingWithOr() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('name',eq('Fred')).has('age',eq('13')).out('livesIn').has('state',eq('Massachusetts'))};"
+                + "def f2={GraphTraversal x->x.has('p5',eq('e5')).has('p6',eq('e6'))};"
+                + "f2(f1().has('p1',eq('e1')).has('p3',eq('e3'))).has('p7',eq('e7')).fill(r);"
+                + "f2(f1().has('p1',eq('e1')).has('p3',eq('e3'))).has('p8',eq('e8')).fill(r);"
+                + "f2(f1().has('p1',eq('e1')).has('p4',eq('e4'))).has('p7',eq('e7')).fill(r);"
+                + "f2(f1().has('p1',eq('e1')).has('p4',eq('e4'))).has('p8',eq('e8')).fill(r);"
+                + "f2(f1().has('p2',eq('e2')).has('p3',eq('e3'))).has('p7',eq('e7')).fill(r);"
+                + "f2(f1().has('p2',eq('e2')).has('p3',eq('e3'))).has('p8',eq('e8')).fill(r);"
+                + "f2(f1().has('p2',eq('e2')).has('p4',eq('e4'))).has('p7',eq('e7')).fill(r);"
+                + "f2(f1().has('p2',eq('e2')).has('p4',eq('e4'))).has('p8',eq('e8')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestLongStringNotEndingWithOr() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('name',eq('Fred')).has('age',eq('13')).out('livesIn').has('state',eq('Massachusetts'))};"
+                + "def f2={GraphTraversal x->x.has('p5',eq('e5')).has('p6',eq('e6'))};"
+                + "def f3={GraphTraversal x->x.has('p9',eq('e9')).fill(r)};"
+                + "f3(f2(f1().has('p1',eq('e1')).has('p3',eq('e3'))).has('p7',eq('e7')));"
+                + "f3(f2(f1().has('p1',eq('e1')).has('p3',eq('e3'))).has('p8',eq('e8')));"
+                + "f3(f2(f1().has('p1',eq('e1')).has('p4',eq('e4'))).has('p7',eq('e7')));"
+                + "f3(f2(f1().has('p1',eq('e1')).has('p4',eq('e4'))).has('p8',eq('e8')));"
+                + "f3(f2(f1().has('p2',eq('e2')).has('p3',eq('e3'))).has('p7',eq('e7')));"
+                + "f3(f2(f1().has('p2',eq('e2')).has('p3',eq('e3'))).has('p8',eq('e8')));"
+                + "f3(f2(f1().has('p2',eq('e2')).has('p4',eq('e4'))).has('p7',eq('e7')));"
+                + "f3(f2(f1().has('p2',eq('e2')).has('p4',eq('e4'))).has('p8',eq('e8')));"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestToListConversion() {
+
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',eq('Fred')).fill(r);"
+                + "g.V().has('prop2',eq('George')).fill(r);"
+                + "g.V('').inject(((r) as Vertex[])).toList()";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestToListWithExtraStuff() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',eq('Fred')).fill(r);"
+                + "g.V().has('prop2',eq('George')).fill(r);"
+                + "g.V('').inject(((r) as Vertex[])).toList().size()";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestAddClosureWithExitExpressionDifferentFromExpr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',eq('Fred')).out('knows').out('livesIn').fill(r);"
+                + "g.V().has('prop2',eq('George')).out('knows').out('livesIn').fill(r);"
+                + "g.V('').inject(((r) as Vertex[])).toList().size()";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAddClosureNoExitExpression() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',eq('Fred')).out('knows').out('livesIn').fill(r);"
+                + "g.V().has('prop2',eq('George')).out('knows').out('livesIn').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAddClosureWithExitExpressionEqualToExpr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('prop1',eq('Fred')).out('knows').out('livesIn').fill(r);"
+                + "g.V().has('prop2',eq('George')).out('knows').out('livesIn').fill(r);"
+                + "g.V('').inject(((r) as Vertex[])).toList()";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestClosureNotCreatedWhenNoOrs() {
+        return "g.V().has('prop1',eq('Fred')).has('prop2',eq('George')).out('knows').out('livesIn')";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestOrFollowedByAnd() {
+        return "def r=(([]) as Set);"
+                + "def f1={GraphTraversal x->x.has('age',eq('13')).has('age',eq('14')).fill(r)};"
+                + "f1(g.V().has('name',eq('Fred')));"
+                + "f1(g.V().has('name',eq('George')));"
+                + "r";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestOrFollowedByOr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',eq('Fred')).has('age',eq('13')).fill(r);"
+                + "g.V().has('name',eq('Fred')).has('age',eq('14')).fill(r);"
+                + "g.V().has('name',eq('George')).has('age',eq('13')).fill(r);"
+                + "g.V().has('name',eq('George')).has('age',eq('14')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestMassiveOrExpansion() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('h1',eq('h2')).has('h3',eq('h4'))};"
+                + "def f2={GraphTraversal x->x.has('ha0',eq('hb0')).has('hc0',eq('hd0'))};"
+                + "def f3={GraphTraversal x->x.has('ha1',eq('hb1')).has('hc1',eq('hd1'))};"
+                + "def f4={GraphTraversal x->x.has('ha2',eq('hb2')).has('hc2',eq('hd2'))};"
+                + "def f5={GraphTraversal x->x.has('ha3',eq('hb3')).has('hc3',eq('hd3'))};"
+                + "def f6={GraphTraversal x->x.has('ha4',eq('hb4')).has('hc4',eq('hd4')).has('h5',eq('h6')).has('h7',eq('h8')).fill(r)};"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p11',eq('e11'))).has('p12',eq('e12'))).has('p13',eq('e13'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p11',eq('e11'))).has('p12',eq('e12'))).has('p13',eq('e13'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p11',eq('e11'))).has('p12',eq('e12'))).has('p23',eq('e23'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p11',eq('e11'))).has('p12',eq('e12'))).has('p23',eq('e23'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p11',eq('e11'))).has('p22',eq('e22'))).has('p13',eq('e13'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p11',eq('e11'))).has('p22',eq('e22'))).has('p13',eq('e13'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p11',eq('e11'))).has('p22',eq('e22'))).has('p23',eq('e23'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p11',eq('e11'))).has('p22',eq('e22'))).has('p23',eq('e23'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p21',eq('e21'))).has('p12',eq('e12'))).has('p13',eq('e13'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p21',eq('e21'))).has('p12',eq('e12'))).has('p13',eq('e13'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p21',eq('e21'))).has('p12',eq('e12'))).has('p23',eq('e23'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p21',eq('e21'))).has('p12',eq('e12'))).has('p23',eq('e23'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p21',eq('e21'))).has('p22',eq('e22'))).has('p13',eq('e13'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p21',eq('e21'))).has('p22',eq('e22'))).has('p13',eq('e13'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p21',eq('e21'))).has('p22',eq('e22'))).has('p23',eq('e23'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p10',eq('e10'))).has('p21',eq('e21'))).has('p22',eq('e22'))).has('p23',eq('e23'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p11',eq('e11'))).has('p12',eq('e12'))).has('p13',eq('e13'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p11',eq('e11'))).has('p12',eq('e12'))).has('p13',eq('e13'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p11',eq('e11'))).has('p12',eq('e12'))).has('p23',eq('e23'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p11',eq('e11'))).has('p12',eq('e12'))).has('p23',eq('e23'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p11',eq('e11'))).has('p22',eq('e22'))).has('p13',eq('e13'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p11',eq('e11'))).has('p22',eq('e22'))).has('p13',eq('e13'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p11',eq('e11'))).has('p22',eq('e22'))).has('p23',eq('e23'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p11',eq('e11'))).has('p22',eq('e22'))).has('p23',eq('e23'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p21',eq('e21'))).has('p12',eq('e12'))).has('p13',eq('e13'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p21',eq('e21'))).has('p12',eq('e12'))).has('p13',eq('e13'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p21',eq('e21'))).has('p12',eq('e12'))).has('p23',eq('e23'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p21',eq('e21'))).has('p12',eq('e12'))).has('p23',eq('e23'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p21',eq('e21'))).has('p22',eq('e22'))).has('p13',eq('e13'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p21',eq('e21'))).has('p22',eq('e22'))).has('p13',eq('e13'))).has('p24',eq('e24')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p21',eq('e21'))).has('p22',eq('e22'))).has('p23',eq('e23'))).has('p14',eq('e14')));"
+                + "f6(f5(f4(f3(f2(f1().has('p20',eq('e20'))).has('p21',eq('e21'))).has('p22',eq('e22'))).has('p23',eq('e23'))).has('p24',eq('e24')));"
+                + "r";
+
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAndFollowedByAnd() {
+        return "g.V().has('name',eq('Fred')).has('name',eq('George')).has('age',eq('13')).has('age',eq('14'))";
+    }
+
+
+    @Override
+    protected String getExpectedGremlinForTestAndFollowedByOr() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('name',eq('Fred')).has('name',eq('George'))};"
+                + "f1().has('age',eq('13')).fill(r);"
+                + "f1().has('age',eq('14')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestInitialAlias() {
+        return "def r=(([]) as Set);"
+                + "g.V().as('x').has('name',eq('Fred')).fill(r);"
+                + "g.V().as('x').has('name',eq('George')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestFinalAlias() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',eq('Fred')).as('x').fill(r);"
+                + "g.V().has('name',eq('George')).as('x').fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAliasInMiddle() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',eq('Fred')).as('x').has('age',eq('13')).fill(r);"
+                + "g.V().has('name',eq('Fred')).as('x').has('age',eq('14')).fill(r);"
+                + "g.V().has('name',eq('George')).as('x').has('age',eq('13')).fill(r);"
+                + "g.V().has('name',eq('George')).as('x').has('age',eq('14')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGreminForTestMultipleAliases() {
+        return "def r=(([]) as Set);"
+                + "def f1={GraphTraversal x->x.as('y').fill(r)};"
+                + "f1(g.V().has('name',eq('Fred')).as('x').has('age',eq('13')));"
+                + "f1(g.V().has('name',eq('Fred')).as('x').has('age',eq('14')));"
+                + "f1(g.V().has('name',eq('George')).as('x').has('age',eq('13')));"
+                + "f1(g.V().has('name',eq('George')).as('x').has('age',eq('14')));"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAliasInOrExpr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',eq('Fred')).fill(r);"
+                + "g.V().or(has('name',eq('George')).as('george')).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestAliasInAndExpr() {
+        return "g.V().has('name',eq('Fred')).and(has('name',eq('George')).as('george'))";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestFlatMapExprInAnd() {
+        return "g.V().has('name',eq('Fred')).and(out('knows').has('name',eq('George')))";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestFlatMapExprInOr() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',eq('Fred')).fill(r);"
+                + "g.V().or(out('knows').has('name',eq('George'))).fill(r);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestFieldExpressionPushedToResultExpression() {
+
+        return "def r=(([]) as Set);"
+                + "g.V().has('name',eq('Fred')).fill(r);"
+                + "g.V().or(out('knows').has('name',eq('George'))).fill(r);"
+                + "g.V('').inject(((r) as Vertex[])).values('name')";
+    }
+
+    @Override
+    protected String getExpectedGremlinFortestOrWithNoChildren() {
+        return "def r=(([]) as Set);"
+                + "r";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestFinalAliasNeeded() {
+        return "def r=(([]) as Set);"
+                + "def f1={g.V().has('name',eq('Fred')).as('person').out('livesIn')};"
+                + "def f2={GraphTraversal x->x.as('city').out('state').has('name',eq('Massachusetts')).as('__res').select('person','city','__res').fill(r)};"
+                + "f2(f1().has('name',eq('Chicago')));f2(f1().has('name',eq('Boston')));"
+                + "__(((r) as Map[])).as('__tmp').map({((Map)it.get()).get('person')}).as('person').select('__tmp').map({((Map)it.get()).get('city')}).as('city').select('__tmp').map({((Map)it.get()).get('__res')}).as('__res').path().toList().collect({it.tail()})";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestSimpleRangeExpression() {
+        return "def r=(([]) as Set);"
+                + "def f1={GraphTraversal x->x.has('age',eq('34')).out('eats').has('size',eq('small')).has('color',eq('blue')).range(0,10).fill(r)};"
+                + "f1(g.V().has('name',eq('Fred')));"
+                + "f1(g.V().has('name',eq('George')));"
+                + "g.V('').inject(((r) as Vertex[])).range(0,10).toList().size()";
+    }
+
+    @Override
+    protected String getExpectedGremlinForOptimizeLoopExpression() {
+        return "def r=(([]) as Set);def f1={GraphTraversal x->x.has('name',eq('Fred')).as('label').select('label').fill(r)};"
+                + "f1(g.V().has('__typeName','DataSet'));"
+                + "f1(g.V().has('__superTypeNames','DataSet'));"
+                + "g.V('').inject(((r) as Vertex[])).as('label').repeat(__.in('inputTables').out('outputTables')).emit(has('__typeName',eq('string')).or().has('__superTypeNames',eq('string'))).toList()";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestRangeWithNonZeroOffset() {
+        return "def r=(([]) as Set);" +
+               "g.V().has('__typeName',eq('OMAS_OMRSAsset')).fill(r);" +
+               "g.V().has('__superTypeNames',eq('OMAS_OMRSAsset')).fill(r);" +
+               "g.V('').inject(((r) as Vertex[])).range(5,10).as('inst').select('inst')";
+    }
+
+    @Override
+    protected String getExpectedGremlinForTestRangeWithOrderBy() {
+        return "def r=(([]) as Set);"
+                + "g.V().has('__typeName',eq('OMAS_OMRSAsset')).fill(r);"
+                + "g.V().has('__superTypeNames',eq('OMAS_OMRSAsset')).fill(r);"
+                + "g.V('').inject(((r) as Vertex[])).range(5,10).as('inst').order().by((({it.get().values('name')}) as Function),{a, b->a.toString().toLowerCase() <=> b.toString().toLowerCase()})";
+    }
+
+}

repository/src/test/java/org/apache/atlas/repository/graph/TestIntSequence.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import org.apache.atlas.query.IntSequence;
+
+/**
+ * IntSequence for use in unit tests.
+ *
+ */
+public class TestIntSequence implements IntSequence {
+
+    public static final IntSequence INSTANCE = new TestIntSequence();
+    private TestIntSequence() {
+    }
+    @Override
+    public int next() {
+        return 0;
+    }
+}
\ No newline at end of file

repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala

@@ -886,7 +886,7 @@ class GremlinTest extends BaseGremlinTest {
         .or(id("name").`=`(string("Reporting")))).field("Table").as("tab")
         .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp
     )
-    validateJson(r, "{\n  \"query\":\"DB as db1 where (db1.createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n  \"dataType\":{\n    \"typeName\":\"__tempQueryResultStruct6\",\n    \"attributeDefinitions\":[\n      {\n        \"name\":\"dbName\",\n        \"dataTypeName\":\"string\",\n        \"multiplicity\":{\n          \"lower\":0,\n          \"upper\":1,\n          \"isUnique\":false\n        },\n        \"isComposite\":false,\n        \"isUnique\":false,\n        \"isIndexable\":false,\n        \"reverseAttributeName\":null\n      },\n      {\n        \"name\":\"tabName\",\n        \"dataTypeName\":\"string\",\n        \"multiplicity\":{\n          \"lower\":0,\n          \"upper\":1,\n          \"isUnique\":false\n        },\n        \"isComposite\":false,\n        \"isUnique\":false,\n        \"isIndexable\":false,\n        \"reverseAttributeName\":null\n      }\n    ]\n  },\n  \"rows\":[\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"sales_fact\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"product_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"time_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"customer_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Reporting\",\n      \"tabName\":\"sales_fact_daily_mv\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Reporting\",\n      \"tabName\":\"sales_fact_monthly_mv\"\n    }\n  ]\n}")
+    validateJson(r, "{\n  \"query\":\"DB as db1 where (createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n  \"dataType\":{\n    \"typeName\":\"__tempQueryResultStruct6\",\n    \"attributeDefinitions\":[\n      {\n        \"name\":\"dbName\",\n        \"dataTypeName\":\"string\",\n        \"multiplicity\":{\n          \"lower\":0,\n          \"upper\":1,\n          \"isUnique\":false\n        },\n        \"isComposite\":false,\n        \"isUnique\":false,\n        \"isIndexable\":false,\n        \"reverseAttributeName\":null\n      },\n      {\n        \"name\":\"tabName\",\n        \"dataTypeName\":\"string\",\n        \"multiplicity\":{\n          \"lower\":0,\n          \"upper\":1,\n          \"isUnique\":false\n        },\n        \"isComposite\":false,\n        \"isUnique\":false,\n        \"isIndexable\":false,\n        \"reverseAttributeName\":null\n      }\n    ]\n  },\n  \"rows\":[\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"sales_fact\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"product_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"time_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"customer_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Reporting\",\n      \"tabName\":\"sales_fact_daily_mv\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct6\",\n      \"dbName\":\"Reporting\",\n      \"tabName\":\"sales_fact_monthly_mv\"\n    }\n  ]\n}")
   }
 
   @Test def testJoinAndSelect3 {
@@ -896,7 +896,7 @@ class GremlinTest extends BaseGremlinTest {
         .or(id("db1").hasField("owner"))).field("Table").as("tab")
         .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp
     )
-    validateJson(r, "{\n  \"query\":\"DB as db1 where (db1.createTime > 0) and (db1.name = \\\"Reporting\\\") or db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n  \"dataType\":{\n    \"typeName\":\"__tempQueryResultStruct7\",\n    \"attributeDefinitions\":[\n      {\n        \"name\":\"dbName\",\n        \"dataTypeName\":\"string\",\n        \"multiplicity\":{\n          \"lower\":0,\n          \"upper\":1,\n          \"isUnique\":false\n        },\n        \"isComposite\":false,\n        \"isUnique\":false,\n        \"isIndexable\":false,\n        \"reverseAttributeName\":null\n      },\n      {\n        \"name\":\"tabName\",\n        \"dataTypeName\":\"string\",\n        \"multiplicity\":{\n          \"lower\":0,\n          \"upper\":1,\n          \"isUnique\":false\n        },\n        \"isComposite\":false,\n        \"isUnique\":false,\n        \"isIndexable\":false,\n        \"reverseAttributeName\":null\n      }\n    ]\n  },\n  \"rows\":[\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"sales_fact\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"product_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"time_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"customer_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Reporting\",\n      \"tabName\":\"sales_fact_daily_mv\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Reporting\",\n      \"tabName\":\"sales_fact_monthly_mv\"\n    }\n  ]\n}")
+    validateJson(r, "{\n  \"query\":\"DB as db1 where (createTime > 0) and (name = \\\"Reporting\\\") or db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n  \"dataType\":{\n    \"typeName\":\"__tempQueryResultStruct7\",\n    \"attributeDefinitions\":[\n      {\n        \"name\":\"dbName\",\n        \"dataTypeName\":\"string\",\n        \"multiplicity\":{\n          \"lower\":0,\n          \"upper\":1,\n          \"isUnique\":false\n        },\n        \"isComposite\":false,\n        \"isUnique\":false,\n        \"isIndexable\":false,\n        \"reverseAttributeName\":null\n      },\n      {\n        \"name\":\"tabName\",\n        \"dataTypeName\":\"string\",\n        \"multiplicity\":{\n          \"lower\":0,\n          \"upper\":1,\n          \"isUnique\":false\n        },\n        \"isComposite\":false,\n        \"isUnique\":false,\n        \"isIndexable\":false,\n        \"reverseAttributeName\":null\n      }\n    ]\n  },\n  \"rows\":[\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"sales_fact\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"product_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"time_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Sales\",\n      \"tabName\":\"customer_dim\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Reporting\",\n      \"tabName\":\"sales_fact_daily_mv\"\n    },\n    {\n      \"$typeName$\":\"__tempQueryResultStruct7\",\n      \"dbName\":\"Reporting\",\n      \"tabName\":\"sales_fact_monthly_mv\"\n    }\n  ]\n}")
   }
 
   @Test def testJoinAndSelect4 {

shaded/hbase-client-shaded/pom.xml

@@ -37,6 +37,15 @@
         </dependency>
     </dependencies>
 
+    <dependencyManagement>
+      <dependencies>
+	<dependency>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+          <version>12.0.1</version>
+	</dependency>
+      </dependencies>
+    </dependencyManagement>
     <build>
         <plugins>
             <plugin>

shaded/hbase-server-shaded/pom.xml

@@ -42,7 +42,16 @@
             <artifactId>hbase-server</artifactId>
         </dependency>
     </dependencies>
-
+    <dependencyManagement>
+      <dependencies>
+	<dependency>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+          <version>12.0.1</version>
+	</dependency>
+      </dependencies>
+    </dependencyManagement>
+    
     <build>
         <plugins>
             <plugin>