Skip to content

A
atlas
Commit 5adca841 by Madhan Neethiraj
Options

Revert "ATLAS-1369 - Optimize gremlin queries generated by DSL translator"

This reverts commit aa74c73d.
parent 352f4397
Showing with 100 additions and 2290 deletions
common/src/main/java/org/apache/atlas/groovy/AbstractFunctionExpression.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.groovy;
/**
* Base class for all expression that can have a caller.
*/
public abstract class AbstractFunctionExpression extends AbstractGroovyExpression {
// null for global functions
private GroovyExpression caller;
private TraversalStepType type = TraversalStepType.NONE;
public AbstractFunctionExpression(GroovyExpression target) {
this.caller = target;
}
public AbstractFunctionExpression(TraversalStepType type, GroovyExpression target) {
this.caller = target;
this.type = type;
}
public GroovyExpression getCaller() {
return caller;
}
public void setCaller(GroovyExpression expr) {
caller = expr;
}
public void setType(TraversalStepType type) {
this.type = type;
}
@Override
public TraversalStepType getType() {
return type;
}
}
common/src/main/java/org/apache/atlas/groovy/AbstractGroovyExpression.java
......@@ -33,13 +33,4 @@ public abstract class AbstractGroovyExpression implements GroovyExpression {
return ctx.getQuery();
}
@Override
public TraversalStepType getType() {
return TraversalStepType.NONE;
}
@Override
public GroovyExpression copy() {
return copy(getChildren());
}
}
common/src/main/java/org/apache/atlas/groovy/ArithmeticExpression.java
......@@ -17,8 +17,6 @@
*/
package org.apache.atlas.groovy;
import java.util.List;
import org.apache.atlas.AtlasException;
/**
......@@ -58,14 +56,4 @@ public class ArithmeticExpression extends BinaryExpression {
public ArithmeticExpression(GroovyExpression left, ArithmeticOperator op, GroovyExpression right) {
super(left, op.getGroovyValue(), right);
}
private ArithmeticExpression(GroovyExpression left, String op, GroovyExpression right) {
super(left, op, right);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 2;
return new ArithmeticExpression(newChildren.get(0), op, newChildren.get(1));
}
}
common/src/main/java/org/apache/atlas/groovy/BinaryExpression.java
......@@ -18,9 +18,6 @@
package org.apache.atlas.groovy;
import java.util.Arrays;
import java.util.List;
/**
* Represents any kind of binary expression. This could
* be an arithmetic expression, such as a + 3, a boolean
......@@ -33,7 +30,7 @@ public abstract class BinaryExpression extends AbstractGroovyExpression {
private GroovyExpression left;
private GroovyExpression right;
protected String op;
private String op;
public BinaryExpression(GroovyExpression left, String op, GroovyExpression right) {
this.left = left;
......@@ -51,8 +48,4 @@ public abstract class BinaryExpression extends AbstractGroovyExpression {
right.generateGroovy(context);
}
@Override
public List<GroovyExpression> getChildren() {
return Arrays.asList(left, right);
}
}
common/src/main/java/org/apache/atlas/groovy/CastExpression.java
......@@ -18,9 +18,6 @@
package org.apache.atlas.groovy;
import java.util.Collections;
import java.util.List;
/**
* Groovy expression that represents a cast.
*/
......@@ -31,7 +28,7 @@ public class CastExpression extends AbstractGroovyExpression {
public CastExpression(GroovyExpression expr, String className) {
this.expr = expr;
this.className = className;
this.className =className;
}
@Override
......@@ -44,13 +41,4 @@ public class CastExpression extends AbstractGroovyExpression {
context.append(")");
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.singletonList(expr);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 1;
return new CastExpression(newChildren.get(0), className);
}
}
common/src/main/java/org/apache/atlas/groovy/ClosureExpression.java
......@@ -20,7 +20,6 @@ package org.apache.atlas.groovy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
......@@ -29,79 +28,28 @@ import java.util.List;
*/
public class ClosureExpression extends AbstractGroovyExpression {
/**
* Variable declaration in a closure.
*/
public static class VariableDeclaration {
private String type;
private String varName;
private List<String> varNames = new ArrayList<>();
private GroovyExpression body;
public VariableDeclaration(String type, String varName) {
super();
this.type = type;
this.varName = varName;
}
public VariableDeclaration(String varName) {
this.varName = varName;
}
public void append(GroovyGenerationContext context) {
if (type != null) {
context.append(type);
context.append(" ");
}
context.append(varName);
}
}
private List<VariableDeclaration> vars = new ArrayList<>();
private StatementListExpression body = new StatementListExpression();
public ClosureExpression(String... varNames) {
this(null, varNames);
}
public ClosureExpression(GroovyExpression initialStmt, String... varNames) {
this(Arrays.asList(varNames), initialStmt);
}
public ClosureExpression(List<String> varNames, GroovyExpression initialStmt) {
if (initialStmt != null) {
this.body.addStatement(initialStmt);
}
for (String varName : varNames) {
vars.add(new VariableDeclaration(varName));
}
public ClosureExpression(GroovyExpression body, String... varNames) {
this.body = body;
this.varNames.addAll(Arrays.asList(varNames));
}
public ClosureExpression(GroovyExpression initialStmt, List<VariableDeclaration> varNames) {
if (initialStmt != null) {
this.body.addStatement(initialStmt);
}
vars.addAll(varNames);
}
public void addStatement(GroovyExpression expr) {
body.addStatement(expr);
}
public void addStatements(List<GroovyExpression> exprs) {
body.addStatements(exprs);
}
public void replaceStatement(int index, GroovyExpression newExpr) {
body.replaceStatement(index, newExpr);
public ClosureExpression(List<String> varNames, GroovyExpression body) {
this.body = body;
this.varNames.addAll(varNames);
}
@Override
public void generateGroovy(GroovyGenerationContext context) {
context.append("{");
if (!vars.isEmpty()) {
Iterator<VariableDeclaration> varIt = vars.iterator();
if (!varNames.isEmpty()) {
Iterator<String> varIt = varNames.iterator();
while(varIt.hasNext()) {
VariableDeclaration var = varIt.next();
var.append(context);
String varName = varIt.next();
context.append(varName);
if (varIt.hasNext()) {
context.append(", ");
}
......@@ -110,20 +58,6 @@ public class ClosureExpression extends AbstractGroovyExpression {
}
body.generateGroovy(context);
context.append("}");
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.<GroovyExpression>singletonList(body);
}
public List<GroovyExpression> getStatements() {
return body.getStatements();
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 1;
return new ClosureExpression(newChildren.get(0), vars);
}
}
common/src/main/java/org/apache/atlas/groovy/StatementListExpression.java → common/src/main/java/org/apache/atlas/groovy/CodeBlockExpression.java
......@@ -19,46 +19,35 @@
package org.apache.atlas.groovy;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
/**
* Represents a semi-colon delimited list of Groovy expressions.
* Groovy expression that represents a block of code
* that contains 0 or more statements that are delimited
* by semicolons.
*/
public class StatementListExpression extends AbstractGroovyExpression {
public class CodeBlockExpression extends AbstractGroovyExpression {
private List<GroovyExpression> stmts = new ArrayList<>();
public StatementListExpression() {
}
/**
* @param newChildren
*/
public StatementListExpression(List<GroovyExpression> newChildren) {
stmts.addAll(newChildren);
}
private List<GroovyExpression> body = new ArrayList<>();
public void addStatement(GroovyExpression expr) {
if (expr instanceof StatementListExpression) {
stmts.addAll(((StatementListExpression)expr).getStatements());
} else {
stmts.add(expr);
}
body.add(expr);
}
public void addStatements(List<GroovyExpression> exprs) {
for(GroovyExpression expr : exprs) {
addStatement(expr);
}
body.addAll(exprs);
}
@Override
public void generateGroovy(GroovyGenerationContext context) {
Iterator<GroovyExpression> stmtIt = stmts.iterator();
/*
* the L:{} represents a groovy code block; the label is needed
* to distinguish it from a groovy closure.
*/
context.append("L:{");
Iterator<GroovyExpression> stmtIt = body.iterator();
while(stmtIt.hasNext()) {
GroovyExpression stmt = stmtIt.next();
stmt.generateGroovy(context);
......@@ -66,33 +55,7 @@ public class StatementListExpression extends AbstractGroovyExpression {
context.append(";");
}
}
}
public List<GroovyExpression> getStatements() {
return stmts;
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.unmodifiableList(stmts);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
return new StatementListExpression(newChildren);
}
@Override
public TraversalStepType getType() {
return TraversalStepType.NONE;
}
context.append("}");
/**
* @param oldExpr
* @param newExpr
*/
public void replaceStatement(int index, GroovyExpression newExpr) {
stmts.set(index, newExpr);
}
}
common/src/main/java/org/apache/atlas/groovy/ComparisonExpression.java
......@@ -17,8 +17,6 @@
*/
package org.apache.atlas.groovy;
import java.util.List;
import org.apache.atlas.AtlasException;
/**
......@@ -63,14 +61,4 @@ public class ComparisonExpression extends BinaryExpression {
public ComparisonExpression(GroovyExpression left, ComparisonOperator op, GroovyExpression right) {
super(left, op.getGroovyValue(), right);
}
private ComparisonExpression(GroovyExpression left, String op, GroovyExpression right) {
super(left, op, right);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 2;
return new ComparisonExpression(newChildren.get(0), op, newChildren.get(1));
}
}
common/src/main/java/org/apache/atlas/groovy/ComparisonOperatorExpression.java
......@@ -17,8 +17,6 @@
*/
package org.apache.atlas.groovy;
import java.util.List;
/**
* Represents an expression that compares two expressions using
* the Groovy "spaceship" operator. This is basically the
......@@ -31,10 +29,4 @@ public class ComparisonOperatorExpression extends BinaryExpression {
public ComparisonOperatorExpression(GroovyExpression left, GroovyExpression right) {
super(left, "<=>", right);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 2;
return new ComparisonOperatorExpression(newChildren.get(0), newChildren.get(1));
}
}
common/src/main/java/org/apache/atlas/groovy/FieldExpression.java
......@@ -18,38 +18,27 @@
package org.apache.atlas.groovy;
import java.util.Collections;
import java.util.List;
/**
* Groovy expression that accesses a field in an object.
*/
public class FieldExpression extends AbstractFunctionExpression {
public class FieldExpression extends AbstractGroovyExpression {
private GroovyExpression target;
private String fieldName;
public FieldExpression(GroovyExpression target, String fieldName) {
super(target);
this.target = target;
this.fieldName = fieldName;
}
@Override
public void generateGroovy(GroovyGenerationContext context) {
getCaller().generateGroovy(context);
target.generateGroovy(context);
context.append(".'");
context.append(fieldName);
context.append("'");
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.singletonList(getCaller());
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 1;
return new FieldExpression(newChildren.get(0), fieldName);
}
}
common/src/main/java/org/apache/atlas/groovy/FunctionCallExpression.java
......@@ -20,52 +20,41 @@ package org.apache.atlas.groovy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
/**
* Groovy expression that calls a method on an object.
*/
public class FunctionCallExpression extends AbstractFunctionExpression {
public class FunctionCallExpression extends AbstractGroovyExpression {
// null for global functions
private GroovyExpression target;
private String functionName;
private List<GroovyExpression> arguments = new ArrayList<>();
public FunctionCallExpression(TraversalStepType type, String functionName, GroovyExpression... arguments) {
super(type, null);
public FunctionCallExpression(String functionName, List<? extends GroovyExpression> arguments) {
this.target = null;
this.functionName = functionName;
this.arguments.addAll(Arrays.asList(arguments));
}
public FunctionCallExpression(String functionName, GroovyExpression... arguments) {
super(null);
this.functionName = functionName;
this.arguments.addAll(Arrays.asList(arguments));
this.arguments.addAll(arguments);
}
public FunctionCallExpression(TraversalStepType type, String functionName, List<GroovyExpression> arguments) {
super(type, null);
public FunctionCallExpression(GroovyExpression target, String functionName,
List<? extends GroovyExpression> arguments) {
this.target = target;
this.functionName = functionName;
this.arguments.addAll(arguments);
}
public FunctionCallExpression(GroovyExpression target, String functionName, GroovyExpression... arguments) {
super(target);
public FunctionCallExpression(String functionName, GroovyExpression... arguments) {
this.target = null;
this.functionName = functionName;
this.arguments.addAll(Arrays.asList(arguments));
}
public FunctionCallExpression(TraversalStepType type, GroovyExpression target, String functionName,
List<? extends GroovyExpression> arguments) {
super(type, target);
this.functionName = functionName;
this.arguments.addAll(arguments);
}
public FunctionCallExpression(TraversalStepType type, GroovyExpression target, String functionName,
GroovyExpression... arguments) {
super(type, target);
public FunctionCallExpression(GroovyExpression target, String functionName, GroovyExpression... arguments) {
this.target = target;
this.functionName = functionName;
this.arguments.addAll(Arrays.asList(arguments));
}
......@@ -74,20 +63,11 @@ public class FunctionCallExpression extends AbstractFunctionExpression {
arguments.add(expr);
}
public List<GroovyExpression> getArguments() {
return Collections.unmodifiableList(arguments);
}
public String getFunctionName() {
return functionName;
}
@Override
public void generateGroovy(GroovyGenerationContext context) {
if (getCaller() != null) {
getCaller().generateGroovy(context);
if (target != null) {
target.generateGroovy(context);
context.append(".");
}
context.append(functionName);
......@@ -97,44 +77,10 @@ public class FunctionCallExpression extends AbstractFunctionExpression {
GroovyExpression expr = it.next();
expr.generateGroovy(context);
if (it.hasNext()) {
context.append(",");
context.append(", ");
}
}
context.append(")");
}
@Override
public List<GroovyExpression> getChildren() {
List<GroovyExpression> result = new ArrayList<>(arguments.size() + 1);
if (getCaller() != null) {
result.add(getCaller());
}
result.addAll(arguments);
return result;
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
if (getCaller() == null) {
return new FunctionCallExpression(getType(), functionName, newChildren);
}
GroovyExpression newTarget = newChildren.get(0);
List<GroovyExpression> args = null;
if (newChildren.size() > 1) {
args = newChildren.subList(1, newChildren.size());
} else {
args = Collections.emptyList();
}
return new FunctionCallExpression(getType(), newTarget, functionName, args);
}
public void setArgument(int index, GroovyExpression value) {
if (index < 0 || index >= arguments.size()) {
throw new IllegalArgumentException("Invalid argIndex " + index);
}
arguments.set(index, value);
}
}
common/src/main/java/org/apache/atlas/groovy/GroovyExpression.java
......@@ -18,55 +18,17 @@
package org.apache.atlas.groovy;
import java.util.List;
/**
* Represents an expression in the Groovy programming language, which
* is the language that Gremlin scripts are written and interpreted in.
*/
public interface GroovyExpression {
/**
* Generates a Groovy script from the expression.
* Generates a groovy script from the expression.
*
* @param context
*/
void generateGroovy(GroovyGenerationContext context);
/**
* Gets all of the child expressions of this expression.
* s
* @return
*/
List<GroovyExpression> getChildren();
/**
* Makes a copy of the expression, keeping everything the
* same except its child expressions. These are replaced
* with the provided children. The order of the children
* is important. It is expected that the children provided
* here are updated versions of the children returned by
* getChildren(). The order of the children must be the
* same as the order in which the children were returned
* by getChildren()
*
* @param newChildren
* @return
*/
GroovyExpression copy(List<GroovyExpression> newChildren);
/**
* Makes a shallow copy of the GroovyExpression. This
* is equivalent to copy(getChildren());
*
* @return
*/
GroovyExpression copy();
/**
* Gets the type of traversal step represented by this
* expression (or TraversalStepType.NONE if it is not part of a graph traversal).
*
* @return
*/
TraversalStepType getType();
}
common/src/main/java/org/apache/atlas/groovy/IdentifierExpression.java
......@@ -18,28 +18,18 @@
package org.apache.atlas.groovy;
import java.util.Collections;
import java.util.List;
/**
* Groovy expression that references the variable with the given name.
*
*/
public class IdentifierExpression extends AbstractGroovyExpression {
private TraversalStepType type = TraversalStepType.NONE;
private String varName;
public IdentifierExpression(String varName) {
this.varName = varName;
}
public IdentifierExpression(TraversalStepType type, String varName) {
this.varName = varName;
this.type = type;
}
public String getVariableName() {
return varName;
}
......@@ -49,25 +39,4 @@ public class IdentifierExpression extends AbstractGroovyExpression {
context.append(varName);
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.emptyList();
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.isEmpty();
IdentifierExpression result = new IdentifierExpression(varName);
result.setType(type);
return result;
}
public void setType(TraversalStepType type) {
this.type = type;
}
@Override
public TraversalStepType getType() {
return type;
}
}
common/src/main/java/org/apache/atlas/groovy/LabeledExpression.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.groovy;
import java.util.Collections;
import java.util.List;
/**
* Represents a Groovy expression that has a label.
*/
public class LabeledExpression extends AbstractGroovyExpression {
private String label;
private GroovyExpression expr;
public LabeledExpression(String label, GroovyExpression expr) {
this.label = label;
this.expr = expr;
}
@Override
public void generateGroovy(GroovyGenerationContext context) {
context.append(label);
context.append(":");
expr.generateGroovy(context);
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.singletonList(expr);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 1;
return new LabeledExpression(label, newChildren.get(0));
}
}
common/src/main/java/org/apache/atlas/groovy/ListExpression.java
......@@ -20,7 +20,6 @@ package org.apache.atlas.groovy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
......@@ -58,15 +57,4 @@ public class ListExpression extends AbstractGroovyExpression {
context.append("]");
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.unmodifiableList(values);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
return new ListExpression(newChildren);
}
}
common/src/main/java/org/apache/atlas/groovy/LiteralExpression.java
......@@ -18,15 +18,13 @@
package org.apache.atlas.groovy;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Represents a literal value.
*/
public class LiteralExpression extends AbstractGroovyExpression {
public class LiteralExpression implements GroovyExpression {
public static final LiteralExpression TRUE = new LiteralExpression(true);
public static final LiteralExpression FALSE = new LiteralExpression(false);
......@@ -42,12 +40,6 @@ public class LiteralExpression extends AbstractGroovyExpression {
this.addTypeSuffix = addTypeSuffix;
}
public LiteralExpression(Object value, boolean addTypeSuffix, boolean translateToParameter) {
this.value = value;
this.translateToParameter = translateToParameter;
this.addTypeSuffix = addTypeSuffix;
}
public LiteralExpression(Object value) {
this.value = value;
this.translateToParameter = value instanceof String;
......@@ -94,10 +86,6 @@ public class LiteralExpression extends AbstractGroovyExpression {
}
public Object getValue() {
return value;
}
private String getEscapedValue() {
String escapedValue = (String)value;
escapedValue = escapedValue.replaceAll(Pattern.quote("\\"), Matcher.quoteReplacement("\\\\"));
......@@ -108,15 +96,4 @@ public class LiteralExpression extends AbstractGroovyExpression {
public void setTranslateToParameter(boolean translateToParameter) {
this.translateToParameter = translateToParameter;
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.emptyList();
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 0;
return new LiteralExpression(value, addTypeSuffix, translateToParameter);
}
}
common/src/main/java/org/apache/atlas/groovy/LogicalExpression.java
......@@ -17,8 +17,6 @@
*/
package org.apache.atlas.groovy;
import java.util.List;
/**
* Represents a logical (and/or) expression.
*
......@@ -45,14 +43,4 @@ public class LogicalExpression extends BinaryExpression {
public LogicalExpression(GroovyExpression left, LogicalOperator op, GroovyExpression right) {
super(left, op.getGroovyValue(), right);
}
private LogicalExpression(GroovyExpression left, String op, GroovyExpression right) {
super(left, op, right);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 2;
return new LogicalExpression(newChildren.get(0), op, newChildren.get(1));
}
}
common/src/main/java/org/apache/atlas/groovy/RangeExpression.java
......@@ -18,68 +18,28 @@
package org.apache.atlas.groovy;
import java.util.Collections;
import java.util.List;
/**
* Represents an "exclusive" range expression, e.g. [0..&lt;10].
*/
public class RangeExpression extends AbstractFunctionExpression {
public class RangeExpression extends AbstractGroovyExpression {
private TraversalStepType stepType;
private int startIndex;
private int endIndex;
private GroovyExpression parent;
private int offset;
private int count;
public RangeExpression(TraversalStepType stepType, GroovyExpression parent, int offset, int count) {
super(parent);
this.startIndex = offset;
this.endIndex = count;
this.stepType = stepType;
public RangeExpression(GroovyExpression parent, int offset, int count) {
this.parent = parent;
this.offset = offset;
this.count = count;
}
@Override
public void generateGroovy(GroovyGenerationContext context) {
getCaller().generateGroovy(context);
parent.generateGroovy(context);
context.append(" [");
new LiteralExpression(startIndex).generateGroovy(context);
new LiteralExpression(offset).generateGroovy(context);
context.append("..<");
new LiteralExpression(endIndex).generateGroovy(context);
new LiteralExpression(count).generateGroovy(context);
context.append("]");
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.singletonList(getCaller());
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 1;
return new RangeExpression(stepType, newChildren.get(0), startIndex, endIndex);
}
@Override
public TraversalStepType getType() {
return stepType;
}
public int getStartIndex() {
return startIndex;
}
public void setStartIndex(int startIndex) {
this.startIndex = startIndex;
}
public int getEndIndex() {
return endIndex;
}
public void setEndIndex(int endIndex) {
this.endIndex = endIndex;
}
}
common/src/main/java/org/apache/atlas/groovy/TernaryOperatorExpression.java
......@@ -18,9 +18,6 @@
package org.apache.atlas.groovy;
import java.util.Arrays;
import java.util.List;
/**
* Groovy expression that represents the ternary operator (expr ? trueValue :
* falseValue)
......@@ -32,7 +29,7 @@ public class TernaryOperatorExpression extends AbstractGroovyExpression {
private GroovyExpression falseValue;
public TernaryOperatorExpression(GroovyExpression booleanExpr, GroovyExpression trueValue,
GroovyExpression falseValue) {
GroovyExpression falseValue) {
this.booleanExpr = booleanExpr;
this.trueValue = trueValue;
......@@ -44,9 +41,9 @@ public class TernaryOperatorExpression extends AbstractGroovyExpression {
context.append("((");
booleanExpr.generateGroovy(context);
context.append(")?(");
context.append(") ? (");
trueValue.generateGroovy(context);
context.append("):(");
context.append(") : (");
falseValue.generateGroovy(context);
context.append("))");
}
......@@ -56,20 +53,4 @@ public class TernaryOperatorExpression extends AbstractGroovyExpression {
generateGroovy(context);
return context.getQuery();
}
@Override
public List<GroovyExpression> getChildren() {
return Arrays.asList(booleanExpr, trueValue, falseValue);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 3;
return new TernaryOperatorExpression(newChildren.get(0), newChildren.get(1), newChildren.get(2));
}
@Override
public TraversalStepType getType() {
return trueValue.getType();
}
}
common/src/main/java/org/apache/atlas/groovy/TraversalStepType.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.groovy;
/**
* Types of graph traversal steps. These are based on the traversal steps
* described in the TinkerPop documentation at
* http://tinkerpop.apache.org/docs/current/reference/#graph-traversal-steps.
*/
public enum TraversalStepType {
/**
* Indicates that the expression is not part of a graph traversal.
*/
NONE,
/**
* Indicates that the expression is a
* {@link org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource}.
* This is not technically a graph traversal step. This is the expression the traversal is started from ("g").
*/
SOURCE,
/**
* A Start step adds vertices or edges to the traversal. These include "V", "E", and "inject".
*/
START,
/**
* An End step causes the traversal to be executed. This includes steps such as "toList", "toSet", and "fill"
*/
END,
/**
* Map steps map the current traverser value to exactly one new value. These
* steps include "map" and "select". Here, we make a further distinction
* based on the type of expression that things are being mapped to.
* <p>
* MAP_TO_ELEMENT indicates that the traverser value is being mapped
* to either a Vertex or an Edge.
*/
MAP_TO_ELEMENT,
/**
* Map steps map the current traverser value to exactly one new value. These
* steps include "map" and "select". Here, we make a further distinction
* based on the type of expression that things are being mapped to.
* <p>
* MAP_TO_VALUE indicates that the traverser value is being mapped
* to something that is not a Vertex or an Edge.
*/
MAP_TO_VALUE,
/**
* FlatMap steps map the current value of the traverser to an iterator of objects that
* are streamed to the next step. These are steps like "in, "out", "inE", and
* so forth which map the current value of the traverser from some vertex or edge
* to some other set of vertices or edges that is derived from the original set based
* on the structure of the graph. This also includes "values", which maps a vertex or
* edge to the set of values for a given property. Here, we make a further distinction
* based on the type of expression that things are being mapped to.
* <p>
* FLAT_MAP_TO_ELEMENTS indicates that the traverser value is being mapped
* to something that is a Vertex or an Edge (in, out, outE fall in this category).
*/
FLAT_MAP_TO_ELEMENTS,
/**
* FlatMap steps map the current value of the traverser to an iterator of objects that
* are streamed to the next step. These are steps like "in, "out", "inE", and
* so forth which map the current value of the traverser from some vertex or edge
* to some other set of vertices or edges that is derived from the original set based
* on the structure of the graph. This also includes "values", which maps a vertex or
* edge to the set of values for a given property. Here, we make a further distinction
* based on the type of expression that things are being mapped to.
* <p>
* FLAT_MAP_TO_VALUES indicates that the traverser value is being mapped
* to something that not is a Vertex or an Edge (values falls in this category).
*/
FLAT_MAP_TO_VALUES,
/**
* Filter steps filter things out of the traversal. These include "has", "where",
* "and", "or", and "filter".
*/
FILTER,
/**
* Side effect steps do not affect the traverser value, but do something
* that affects the state of the traverser. These include things such as
* "enablePath()", "as", and "by".
*/
SIDE_EFFECT,
/**
* Branch steps split the traverser, for example, "repeat", "branch", "choose", and "union".
*/
BRANCH,
/**
* Barrier steps in Gremlin force everything before them to be executed
* before moving on to the steps after them. We also use this to indicate
* steps that need to do some aggregation or processing that requires the
* full query result to be present in order for the step to work correctly.
* This includes "range", "group", and "order", and "cap"
*/
BARRIER,
}
common/src/main/java/org/apache/atlas/groovy/TypeCoersionExpression.java
......@@ -18,9 +18,6 @@
package org.apache.atlas.groovy;
import java.util.Collections;
import java.util.List;
/**
* Groovy expression that represents a type coersion (e.g obj as Set).
*/
......@@ -31,29 +28,17 @@ public class TypeCoersionExpression extends AbstractGroovyExpression {
public TypeCoersionExpression(GroovyExpression expr, String className) {
this.expr = expr;
this.className = className;
this.className =className;
}
@Override
public void generateGroovy(GroovyGenerationContext context) {
context.append("((");
context.append("(");
expr.generateGroovy(context);
context.append(")");
context.append(" as ");
context.append(className);
context.append(")");
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.singletonList(expr);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 1;
return new TypeCoersionExpression(newChildren.get(0), className);
}
}
common/src/main/java/org/apache/atlas/groovy/VariableAssignmentExpression.java
......@@ -18,9 +18,6 @@
package org.apache.atlas.groovy;
import java.util.Collections;
import java.util.List;
/**
* Groovy statement that assigns a value to a variable.
*/
......@@ -53,20 +50,9 @@ public class VariableAssignmentExpression extends AbstractGroovyExpression {
context.append(" ");
}
context.append(name);
context.append("=");
context.append(" = ");
value.generateGroovy(context);
}
@Override
public List<GroovyExpression> getChildren() {
return Collections.singletonList(value);
}
@Override
public GroovyExpression copy(List<GroovyExpression> newChildren) {
assert newChildren.size() == 1;
return new VariableAssignmentExpression(name, newChildren.get(0));
}
}
distro/src/conf/atlas-application.properties
......@@ -29,13 +29,6 @@ atlas.graph.storage.hbase.table=apache_atlas_titan
${titan.storage.properties}
# Gremlin Query Optimizer
#
# Enables rewriting gremlin queries to maximize performance. This flag is provided as
# a possible way to work around any defects that are found in the optimizer until they
# are resolved.
#atlas.query.gremlinOptimizerEnabled=true
# Delete handler
#
# This allows the default behavior of doing "soft" deletes to be changed.
......
graphdb/titan0/pom.xml
......@@ -34,7 +34,6 @@
<properties>
<tinkerpop.version>2.6.0</tinkerpop.version>
<titan.version>0.5.4</titan.version>
<guava.version>14.0</guava.version>
</properties>
<dependencies>
......@@ -54,13 +53,6 @@
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<scope>provided</scope>
......
intg/pom.xml
......@@ -60,6 +60,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
......
pom.xml
......@@ -459,7 +459,7 @@
<spring.security.version>3.1.3.RELEASE</spring.security.version>
<spring-ldap-core.version>1.3.1.RELEASE</spring-ldap-core.version>
<javax.servlet.version>3.1.0</javax.servlet.version>
<guava.version>19.0</guava.version>
<guava.version>18.0</guava.version>
<!-- Needed for hooks -->
<aopalliance.version>1.0</aopalliance.version>
......@@ -633,12 +633,6 @@
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<!-- AOP dependencies. -->
<dependency>
<groupId>org.aspectj</groupId>
......
release-log.txt
......@@ -10,7 +10,6 @@ ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(s
ALL CHANGES:
ATLAS-1514 Remove duplicates from class array attribute when target is deleted (dkantor)
ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb)
ATLAS-1513 updated AtlasEntityType with methods to get foreign-key references; added helper methods in AtlasAttribute (mneethiraj via kevalbhatt)
ATLAS-1502 added configuration to restrict entity-types editable via UI (Kalyanikashikar via mneethiraj)
ATLAS-1507 fixed incorrect relationship specified in hive-model
......
repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
......@@ -25,7 +25,6 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.query.GremlinQueryResult;
import org.apache.atlas.query.InputLineageClosureQuery;
import org.apache.atlas.query.OutputLineageClosureQuery;
import org.apache.atlas.query.QueryParams;
......@@ -140,8 +139,7 @@ public class DataSetLineageService implements LineageService {
guid, HIVE_PROCESS_TYPE_NAME,
HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
GremlinQueryResult result = inputsQuery.evaluate();
return inputsQuery.graph(result).toInstanceJson();
return inputsQuery.graph(null).toInstanceJson();
}
@Override
......@@ -158,8 +156,7 @@ public class DataSetLineageService implements LineageService {
new OutputLineageClosureQuery(AtlasClient.DATA_SET_SUPER_TYPE, SELECT_INSTANCE_GUID, guid, HIVE_PROCESS_TYPE_NAME,
HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
GremlinQueryResult result = outputsQuery.evaluate();
return outputsQuery.graph(result).toInstanceJson();
return outputsQuery.graph(null).toInstanceJson();
}
/**
......
repository/src/main/java/org/apache/atlas/gremlin/optimizer/AliasFinder.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.FunctionCallExpression;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.groovy.LiteralExpression;
import org.apache.atlas.groovy.TraversalStepType;
/**
* Finds all aliases in the expression.
*/
public class AliasFinder implements CallHierarchyVisitor {
private List<LiteralExpression> foundAliases = new ArrayList<>();
//Whether a final alias is needed. A final alias is needed
//if there are transformation steps after the last alias in
//the expression. We initialize this to false since a final
//alias is not needed if there are no aliases.
private boolean finalAliasNeeded = false;
@Override
public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
return true;
}
@Override
public void visitNonFunctionCaller(GroovyExpression expr) {
}
@Override
public void visitNullCaller() {
}
private static final Set<TraversalStepType> TRANSFORMATION_STEP_TYPES = new HashSet<>(Arrays.asList(
TraversalStepType.MAP_TO_ELEMENT,
TraversalStepType.MAP_TO_VALUE,
TraversalStepType.FLAT_MAP_TO_ELEMENTS,
TraversalStepType.FLAT_MAP_TO_VALUES,
TraversalStepType.BARRIER,
TraversalStepType.NONE));
@Override
public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
if (functionCall instanceof FunctionCallExpression) {
FunctionCallExpression expr = (FunctionCallExpression)functionCall;
if (expr.getType() == TraversalStepType.SIDE_EFFECT && expr.getFunctionName().equals("as")) {
//We found an alias. This is currently the last expression we've seen
//in our traversal back up the expression tree, so at this point a final
//alias is not needed.
LiteralExpression aliasNameExpr = (LiteralExpression)expr.getArguments().get(0);
foundAliases.add(aliasNameExpr);
finalAliasNeeded=false;
}
}
if(TRANSFORMATION_STEP_TYPES.contains(functionCall.getType())) {
//This step changes the value of the traverser. Now, a final alias
//needs to be added.
if(!foundAliases.isEmpty()) {
finalAliasNeeded = true;
}
}
return true;
}
public List<LiteralExpression> getAliases() {
return foundAliases;
}
public boolean isFinalAliasNeeded() {
return finalAliasNeeded;
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/CallHierarchyVisitor.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.GroovyExpression;
/**
* Call back interface for visiting the call hierarchy of a function call.
*/
public interface CallHierarchyVisitor {
/**
* Visits a function expression before the visit to its caller.
*
* @param expr
*
* @return false to terminate the recursion
*/
boolean preVisitFunctionCaller(AbstractFunctionExpression expr);
/**
* Called when a caller that is not an instance of
* AbstractFunctionExpression is found. This indicates that the deepest
* point in the call hierarchy has been reached.
*
*
*/
void visitNonFunctionCaller(GroovyExpression expr);
/**
* Called when a null caller is found (this happens for static/user-defined
* functions). This indicates that the deepest point in the call hierarchy
* has been reached.
*
*/
void visitNullCaller();
/**
* Visits a function expression after the visit to its caller.
*
* @param expr
*
* @return false to terminate the recursion
*/
boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall);
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/ExpandAndsOptimization.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.ArrayList;
import java.util.List;
import org.apache.atlas.gremlin.GremlinExpressionFactory;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.FunctionCallExpression;
import org.apache.atlas.groovy.GroovyExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Optimizer that pulls has expressions out of an 'and' expression.
*
* For example:
*
* g.V().and(has('x'),has('y')
*
* is optimized to:
*
* g.V().has('x').has('y')
*
* There are certain cases where it is not safe to move an expression out
* of the 'and'. For example, in the expression
*
* g.V().and(has('x').out('y'),has('z'))
*
* has('x').out('y') cannot be moved out of the 'and', since it changes the value of the traverser.
*
* At this time, the ExpandAndsOptimizer is not able to handle this scenario, so we don't extract
* that expression. In this case, the result is:
*
* g.V().has('z').and(has('x').out('y'))
*
* The optimizer will call ExpandAndsOptimization recursively on the children, so
* there is no need to recursively update the children here.
*
* @param expr
* @param context
* @return the expressions that should be unioned together to get the query result
*/
public class ExpandAndsOptimization implements GremlinOptimization {
private static final Logger logger_ = LoggerFactory.getLogger(ExpandAndsOptimization.class);
private final GremlinExpressionFactory factory;
public ExpandAndsOptimization(GremlinExpressionFactory factory) {
this.factory = factory;
}
@Override
public boolean appliesTo(GroovyExpression expr, OptimizationContext contxt) {
return expr instanceof FunctionCallExpression && ((FunctionCallExpression)expr).getFunctionName().equals("and");
}
/**
* Expands the given and expression. There is no need to recursively
* expand the children here. This method is called recursively by
* GremlinQueryOptimier on the children.
*
*/
@Override
public GroovyExpression apply(GroovyExpression expr, OptimizationContext context) {
FunctionCallExpression exprAsFunction = (FunctionCallExpression)expr;
GroovyExpression result = exprAsFunction.getCaller();
List<GroovyExpression> nonExtractableArguments = new ArrayList<>();
for(GroovyExpression argument : exprAsFunction.getArguments()) {
if (GremlinQueryOptimizer.isExtractable(argument)) {
//Set the caller of the deepest expression in the call hierarchy
//of the argument to point to the current result.
//For example, if result is "g.V()" and the updatedArgument is "has('x').has('y')",
//updatedArgument would be a tree like this:
//
// has('y')
// /
// / caller
// |/_
// has('x')
// /
// / caller
// |/_
// (null)
//
//We would set the caller of has('x') to be g.V(), so result would become g.V().has('x').has('y').
//
// Note: This operation is currently done by making a copy of the argument tree. That should
// be changed.
result = GremlinQueryOptimizer.copyWithNewLeafNode(
(AbstractFunctionExpression) argument, result);
} else {
logger_.warn("Found non-extractable argument '{}' in the 'and' expression '{}'",argument.toString(), expr.toString());
nonExtractableArguments.add(argument);
}
}
if (!nonExtractableArguments.isEmpty()) {
//add a final 'and' call with the arguments that could not be extracted
result = factory.generateLogicalExpression(result, "and", nonExtractableArguments);
}
return result;
}
@Override
public boolean isApplyRecursively() {
return true;
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/ExpressionFinder.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.function.Function;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.GroovyExpression;
/**
* Call hierarchy visitor that checks if an expression
* matching the specified criteria is present
* in the call hierarch.
*/
public class ExpressionFinder implements CallHierarchyVisitor {
private final Function<GroovyExpression, Boolean> predicate;
private boolean expressionFound = false;
public ExpressionFinder(Function<GroovyExpression, Boolean> predicate) {
this.predicate = predicate;
}
@Override
public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
if (predicate.apply(expr)) {
expressionFound = true;
return false;
}
return true;
}
@Override
public void visitNonFunctionCaller(GroovyExpression expr) {
if (predicate.apply(expr)) {
expressionFound = true;
}
}
@Override
public void visitNullCaller() {
//nothing to do
}
@Override
public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
//nothing to do
return true;
}
public boolean isExpressionFound() {
return expressionFound;
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/GremlinOptimization.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import org.apache.atlas.groovy.GroovyExpression;
/**
* An optimization that can be applied to a gremlin query.
*/
public interface GremlinOptimization {
/**
* Whether or not this optimization should be applied to the given expression
* @param expr
* @param contxt
* @return
*/
boolean appliesTo(GroovyExpression expr, OptimizationContext contxt);
/**
* Whether or not GremlinQueryOptimizer should call this optimization recursively
* on the updated children.
*/
boolean isApplyRecursively();
/**
* Applies the optimization.
*
* @param expr
* @param context
* @return the optimized expression
*/
GroovyExpression apply(GroovyExpression expr, OptimizationContext context);
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/HasForbiddenType.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.HashSet;
import java.util.Set;
import java.util.function.Function;
import org.apache.atlas.gremlin.GremlinExpressionFactory;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.groovy.TraversalStepType;
/**
* Function that tests whether the expression is an 'or'
* graph traversal function.
*/
public final class HasForbiddenType implements Function<GroovyExpression, Boolean> {
private Set<TraversalStepType> forbiddenTypes = new HashSet<>();
private final GremlinExpressionFactory factory;
public HasForbiddenType(GremlinExpressionFactory factory) {
this.factory = factory;
}
public void addForbiddenType(TraversalStepType type) {
forbiddenTypes.add(type);
}
@Override
public Boolean apply(GroovyExpression expr) {
if(factory.isLeafAnonymousTraversalExpression(expr)) {
return false;
}
return forbiddenTypes.contains(expr.getType());
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/IsOr.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.function.Function;
import org.apache.atlas.groovy.FunctionCallExpression;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.groovy.TraversalStepType;
/**
* Function that tests whether the expression is an 'or'
* graph traversal function.
*/
public final class IsOr implements Function<GroovyExpression, Boolean> {
public static final IsOr INSTANCE = new IsOr();
private IsOr() {
}
@Override
public Boolean apply(GroovyExpression expr) {
if (!(expr instanceof FunctionCallExpression)) {
return false;
}
if (expr.getType() != TraversalStepType.FILTER) {
return false;
}
FunctionCallExpression functionCall = (FunctionCallExpression)expr;
return functionCall.getFunctionName().equals("or");
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/IsOrParent.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.function.Function;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.FunctionCallExpression;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.groovy.TraversalStepType;
/**
* Matches an expression that gets called after calling or(). For example,
* in g.V().or(x,y).toList(), "toList()" is the "or parent", so calling
* "apply()" on this expression would return true and calling it on all
* the other ones would return false.
*/
public final class IsOrParent implements Function<GroovyExpression, Boolean> {
public static final IsOrParent INSTANCE = new IsOrParent();
private IsOrParent() {
}
@Override
public Boolean apply(GroovyExpression expr) {
if (!(expr instanceof AbstractFunctionExpression)) {
return false;
}
AbstractFunctionExpression functionCall = (AbstractFunctionExpression)expr;
GroovyExpression target = functionCall.getCaller();
if (!(target instanceof FunctionCallExpression)) {
return false;
}
if (target.getType() != TraversalStepType.FILTER) {
return false;
}
FunctionCallExpression targetFunction = (FunctionCallExpression)target;
return targetFunction.getFunctionName().equals("or");
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/OptimizationContext.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.ClosureExpression;
import org.apache.atlas.groovy.ClosureExpression.VariableDeclaration;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.groovy.IdentifierExpression;
import org.apache.atlas.groovy.ListExpression;
import org.apache.atlas.groovy.TypeCoersionExpression;
import org.apache.atlas.groovy.VariableAssignmentExpression;
/**
* Maintains state information during gremlin optimization.
*/
public class OptimizationContext {
private static final String TMP_ALIAS_NAME = "__tmp";
private static final String FINAL_ALIAS_NAME = "__res";
private static final String RESULT_VARIABLE = "r";
private final List<GroovyExpression> initialStatements = new ArrayList<>();
private GroovyExpression resultExpression = getResultVariable();
private int counter = 1;
private final Map<String, ClosureExpression> functionBodies = new HashMap<>();
private AbstractFunctionExpression rangeExpression;
public OptimizationContext() {
}
/**
* @return
*/
public List<GroovyExpression> getInitialStatements() {
return initialStatements;
}
public void prependStatement(GroovyExpression expr) {
initialStatements.add(0, expr);
}
public String getUniqueFunctionName() {
return "f" + (counter++);
}
public GroovyExpression getDefineResultVariableStmt() {
GroovyExpression castExpression = new TypeCoersionExpression(new ListExpression(), "Set");
GroovyExpression resultVarDef = new VariableAssignmentExpression(RESULT_VARIABLE, castExpression);
return resultVarDef;
}
public void setResultExpression(GroovyExpression expr) {
resultExpression = expr;
}
public GroovyExpression getResultExpression() {
return resultExpression;
}
public GroovyExpression getResultVariable() {
return new IdentifierExpression(RESULT_VARIABLE);
}
public ClosureExpression getUserDefinedFunctionBody(String functionName) {
return functionBodies.get(functionName);
}
public String addFunctionDefinition(VariableDeclaration decl, GroovyExpression body) {
String functionName = getUniqueFunctionName();
List<VariableDeclaration> decls = (decl == null) ? Collections.<VariableDeclaration>emptyList() : Collections.singletonList(decl);
ClosureExpression bodyClosure = new ClosureExpression(body, decls);
VariableAssignmentExpression expr = new VariableAssignmentExpression(functionName, bodyClosure);
initialStatements.add(expr);
functionBodies.put(functionName, bodyClosure);
return functionName;
}
public String getFinalAliasName() {
return FINAL_ALIAS_NAME;
}
public String getTempAliasName() {
return TMP_ALIAS_NAME;
}
public void setRangeExpression(AbstractFunctionExpression rangeExpression) {
this.rangeExpression = rangeExpression;
}
public AbstractFunctionExpression getRangeExpression() {
return rangeExpression;
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/OrderFinder.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import org.apache.atlas.gremlin.GremlinExpressionFactory;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.GroovyExpression;
/**
* Finds order expression in the call hierarchy.
*
*/
public class OrderFinder implements CallHierarchyVisitor {
private boolean hasOrderExpression;
private GremlinExpressionFactory gremlinFactory;
public OrderFinder(GremlinExpressionFactory gremlinFactory) {
this.gremlinFactory = gremlinFactory;
}
@Override
public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
return true;
}
@Override
public void visitNonFunctionCaller(GroovyExpression expr) {
}
@Override
public void visitNullCaller() {
}
@Override
public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
if (gremlinFactory.isOrderExpression(functionCall)) {
hasOrderExpression = true;
return false;
}
return true;
}
public boolean hasOrderExpression() {
return hasOrderExpression;
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/PathExpressionFinder.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.FunctionCallExpression;
import org.apache.atlas.groovy.GroovyExpression;
/**
* Determines whether an expression contains a path() function.
*/
public class PathExpressionFinder implements CallHierarchyVisitor {
private boolean found = false;
@Override
public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
if(expr instanceof FunctionCallExpression) {
found = ((FunctionCallExpression)expr).getFunctionName().equals("path");
if(found) {
return false;
}
}
return true;
}
@Override
public void visitNonFunctionCaller(GroovyExpression expr) {
}
@Override
public void visitNullCaller() {
}
public boolean isPathExpressionFound() {
return found;
}
@Override
public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
return false;
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/RangeFinder.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.ArrayList;
import java.util.List;
import org.apache.atlas.gremlin.GremlinExpressionFactory;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.GroovyExpression;
/**
* Finds all range expressions in the call hierarchy.
*
*/
public class RangeFinder implements CallHierarchyVisitor {
private List<AbstractFunctionExpression> rangeExpressions = new ArrayList<>();
private GremlinExpressionFactory factory;
public RangeFinder(GremlinExpressionFactory factory) {
this.factory = factory;
}
@Override
public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
return true;
}
@Override
public void visitNonFunctionCaller(GroovyExpression expr) {
}
@Override
public void visitNullCaller() {
}
@Override
public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
if (factory.isRangeExpression(functionCall)) {
rangeExpressions.add(functionCall);
}
return true;
}
public List<AbstractFunctionExpression> getRangeExpressions() {
return rangeExpressions;
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/SplitPointFinder.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.atlas.gremlin.GremlinExpressionFactory;
import org.apache.atlas.groovy.AbstractFunctionExpression;
import org.apache.atlas.groovy.FunctionCallExpression;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.groovy.TraversalStepType;
/**
* This class finds the first place in the expression where the value of the
* traverser is changed from being a vertex to being something else. This is
* important in the "or" optimization logic, since the union operation must be
* done on *vertices* in order to preserve the semantics of the query. In addition,
* expressions that have side effects must be moved as well, so that those
* side effects will be available to the steps that need them.
*/
public class SplitPointFinder implements CallHierarchyVisitor {
//Any steps that change the traverser value to something that is not a vertex or edge
//must be included here, so that the union created by ExpandOrsOptimization
//is done over vertices/edges.
private static final Set<TraversalStepType> TYPES_REQUIRED_IN_RESULT_EXPRESSION = new HashSet<>(
Arrays.asList(
TraversalStepType.BARRIER,
TraversalStepType.BRANCH,
TraversalStepType.SIDE_EFFECT,
TraversalStepType.MAP_TO_VALUE,
TraversalStepType.FLAT_MAP_TO_VALUES,
TraversalStepType.END,
TraversalStepType.NONE));
private final Set<String> requiredAliases = new HashSet<>();
//Exceptions to the requirement that all expressions with a type
//in the above list must be in the result expression. If the
//function name is in this list, it is ok for that expression
//to not be in the result expression. This mechanism allows
//aliases to remain outside the result expression. Other
//exceptions may be found in the future.
private static final Map<TraversalStepType, WhiteList> WHITE_LISTS = new HashMap<>();
static {
WHITE_LISTS.put(TraversalStepType.SIDE_EFFECT, new WhiteList("as"));
}
private final GremlinExpressionFactory factory;
public SplitPointFinder(GremlinExpressionFactory factory) {
this.factory = factory;
}
/**
* Represents a set of function names.
*/
private static final class WhiteList {
private Set<String> allowedFunctionNames = new HashSet<>();
public WhiteList(String... names) {
for(String name : names) {
allowedFunctionNames.add(name);
}
}
public boolean contains(String name) {
return allowedFunctionNames.contains(name);
}
}
private AbstractFunctionExpression splitPoint;
@Override
public boolean preVisitFunctionCaller(AbstractFunctionExpression expr) {
requiredAliases.addAll(factory.getAliasesRequiredByExpression(expr));
return true;
}
@Override
public void visitNonFunctionCaller(GroovyExpression expr) {
}
@Override
public void visitNullCaller() {
}
public AbstractFunctionExpression getSplitPoint() {
return splitPoint;
}
@Override
public boolean postVisitFunctionCaller(AbstractFunctionExpression functionCall) {
String aliasName = factory.getAliasNameIfRelevant(functionCall);
if (splitPoint == null) {
boolean required = isRequiredAlias(aliasName) ||
isRequiredInResultExpression(functionCall);
if (required) {
splitPoint = functionCall;
}
}
removeSeenAlias(aliasName);
return true;
}
private void removeSeenAlias(String aliasName) {
if(aliasName != null) {
requiredAliases.remove(aliasName);
}
}
private boolean isRequiredAlias(String aliasName) {
if(aliasName != null) {
return requiredAliases.contains(aliasName);
}
return false;
}
private boolean isRequiredInResultExpression(AbstractFunctionExpression expr) {
TraversalStepType type = expr.getType();
if (!TYPES_REQUIRED_IN_RESULT_EXPRESSION.contains(type)) {
return false;
}
if(expr instanceof FunctionCallExpression) {
FunctionCallExpression functionCall = (FunctionCallExpression)expr;
//check if the white list permits this function call. If there is
//no white list, all expressions with the current step type must go in the
//result expression.
WhiteList whiteList = WHITE_LISTS.get(type);
if(whiteList != null && whiteList.contains(functionCall.getFunctionName())) {
return false;
}
}
return true;
}
}
repository/src/main/java/org/apache/atlas/gremlin/optimizer/UpdatedExpressions.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.gremlin.optimizer;
import java.util.ArrayList;
import java.util.List;
import org.apache.atlas.groovy.GroovyExpression;
/**
* Represents a list of updated expressions.
*/
public class UpdatedExpressions {
private List<List<GroovyExpression>> updatedChildren = new ArrayList<>();
private boolean changed = false;
public UpdatedExpressions(boolean changed, List<List<GroovyExpression>> updatedChildren) {
this.changed = changed;
this.updatedChildren = updatedChildren;
}
public List<List<GroovyExpression>> getUpdatedChildren() {
return updatedChildren;
}
public boolean hasChanges() {
return changed;
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -67,26 +67,19 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private final AtlasGraph graph;
private DeleteHandler deleteHandler;
private final IAtlasGraphProvider graphProvider;
private final GraphToTypedInstanceMapper graphToInstanceMapper;
private GraphToTypedInstanceMapper graphToInstanceMapper;
@Inject
public GraphBackedMetadataRepository(DeleteHandler deleteHandler) {
this.graphProvider = new AtlasGraphProvider();
this.graphToInstanceMapper = new GraphToTypedInstanceMapper(graphProvider);
this.graph = AtlasGraphProvider.getGraphInstance();
graphToInstanceMapper = new GraphToTypedInstanceMapper(graph);
this.deleteHandler = deleteHandler;
}
//for testing only
public GraphBackedMetadataRepository(IAtlasGraphProvider graphProvider, DeleteHandler deleteHandler) {
this.graphProvider = graphProvider;
this.graphToInstanceMapper = new GraphToTypedInstanceMapper(graphProvider);
this.deleteHandler = deleteHandler;
}
public GraphToTypedInstanceMapper getGraphToInstanceMapper() {
return graphToInstanceMapper;
}
......@@ -201,7 +194,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG.debug("Retrieving entity list for type={}", entityType);
}
AtlasGraphQuery query = getGraph().query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
Iterator<AtlasVertex> results = query.vertices().iterator();
if (!results.hasNext()) {
return Collections.emptyList();
......@@ -436,7 +429,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
requestContext.getDeletedEntityIds());
}
public AtlasGraph getGraph() throws RepositoryException {
return graphProvider.get();
public AtlasGraph getGraph() {
return AtlasGraphProvider.getGraphInstance();
}
}
repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
......@@ -19,7 +19,6 @@ package org.apache.atlas.repository.graph;
import com.google.inject.Singleton;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
......@@ -60,10 +59,10 @@ public final class GraphToTypedInstanceMapper {
private static TypeSystem typeSystem = TypeSystem.getInstance();
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private final IAtlasGraphProvider graphProvider;
private AtlasGraph graph;
public GraphToTypedInstanceMapper(IAtlasGraphProvider graphProvider) {
this.graphProvider = graphProvider;
public GraphToTypedInstanceMapper(AtlasGraph graph) {
this.graph = graph;
}
public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, AtlasVertex instanceVertex)
......@@ -408,7 +407,7 @@ public final class GraphToTypedInstanceMapper {
public ITypedInstance getReferredEntity(String edgeId, IDataType<?> referredType) throws AtlasException {
final AtlasEdge edge = getGraph().getEdge(edgeId);
final AtlasEdge edge = graph.getEdge(edgeId);
if (edge != null) {
final AtlasVertex referredVertex = edge.getInVertex();
if (referredVertex != null) {
......@@ -434,9 +433,5 @@ public final class GraphToTypedInstanceMapper {
}
return null;
}
private AtlasGraph getGraph() throws RepositoryException {
return graphProvider.get();
}
}
repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java
......@@ -24,11 +24,9 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.SoftDeleteHandler;
import org.apache.atlas.repository.graphdb.GraphDatabase;
import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
......@@ -139,6 +137,7 @@ public class AtlasRepositoryConfiguration {
}
}
private static final String GRAPH_DATABASE_IMPLEMENTATION_PROPERTY = "atlas.graphdb.backend";
private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = "org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase";
......@@ -154,22 +153,6 @@ public class AtlasRepositoryConfiguration {
}
/**
* This optimization is configurable as a fail-safe in case issues are found
* with the optimizer in production systems.
*/
public static final String GREMLIN_OPTIMIZER_ENABLED_PROPERTY = "atlas.query.gremlinOptimizerEnabled";
private static final boolean DEFAULT_GREMLIN_OPTIMZER_ENABLED = true;
public static boolean isGremlinOptimizerEnabled() {
try {
return ApplicationProperties.get().getBoolean(GREMLIN_OPTIMIZER_ENABLED_PROPERTY, DEFAULT_GREMLIN_OPTIMZER_ENABLED);
} catch (AtlasException e) {
LOG.error("Could not determine value of " + GREMLIN_OPTIMIZER_ENABLED_PROPERTY + ". Defaulting to " + DEFAULT_GREMLIN_OPTIMZER_ENABLED, e);
return DEFAULT_GREMLIN_OPTIMZER_ENABLED;
}
}
/**
* Get the list of operations which are configured to be skipped from auditing
* Valid format is HttpMethod:URL eg: GET:Version
* @return list of string
......
repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala 100644 → 100755
......@@ -32,19 +32,15 @@ import scala.collection.JavaConversions.bufferAsJavaList
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import org.apache.atlas.gremlin.GremlinExpressionFactory
import org.apache.atlas.gremlin.optimizer.GremlinQueryOptimizer
import org.apache.atlas.groovy.CastExpression
import org.apache.atlas.groovy.ClosureExpression
import org.apache.atlas.groovy.LabeledExpression
import org.apache.atlas.groovy.CodeBlockExpression
import org.apache.atlas.groovy.FunctionCallExpression
import org.apache.atlas.groovy.GroovyExpression
import org.apache.atlas.groovy.GroovyGenerationContext
import org.apache.atlas.groovy.IdentifierExpression
import org.apache.atlas.groovy.ListExpression
import org.apache.atlas.groovy.LiteralExpression
import org.apache.atlas.groovy.TraversalStepType
import org.apache.atlas.query.Expressions.AliasExpression
import org.apache.atlas.query.Expressions.ArithmeticExpression
import org.apache.atlas.query.Expressions.BackReference
......@@ -82,10 +78,7 @@ import org.apache.atlas.query.Expressions.MaxExpression
import org.apache.atlas.query.Expressions.MinExpression
import org.apache.atlas.query.Expressions.SumExpression
import org.apache.atlas.query.Expressions.CountExpression
import org.apache.atlas.util.AtlasRepositoryConfiguration
import java.util.HashSet
trait IntSequence {
def next: Int
}
......@@ -127,69 +120,6 @@ trait SelectExpressionHandling {
}
}
// Removes back references in comparison expressions that are
// right after an alias expression.
//
//For example:
// .as('x').and(select('x').has(y),...) is changed to
// .as('x').and(has(y),...)
//
//This allows the "has" to be extracted out of the and/or by
//the GremlinQueryOptimizer so the index can be used to evaluate
//the predicate.
val RemoveUnneededBackReferences : PartialFunction[Expression, Expression] = {
case filterExpr@FilterExpression(aliasExpr@AliasExpression(_,aliasName), filterChild) => {
val updatedChild = removeUnneededBackReferences(filterChild, aliasName)
val changed = !(updatedChild eq filterChild)
if(changed) {
FilterExpression(aliasExpr, updatedChild)
}
else {
filterExpr
}
}
case x => x
}
def removeUnneededBackReferences(expr: Expression, outerAlias: String) : Expression = expr match {
case logicalExpr@LogicalExpression(logicalOp,children) => {
var changed : Boolean = false;
val updatedChildren : List[Expression] = children.map { child =>
val updatedChild = removeUnneededBackReferences(child, outerAlias);
changed |= ! (updatedChild eq child);
updatedChild
}
if(changed) {
LogicalExpression(logicalOp,updatedChildren)
}
else {
logicalExpr
}
}
case comparisonExpr@ComparisonExpression(_,_,_) => {
var changed = false
val updatedLeft = removeUnneededBackReferences(comparisonExpr.left, outerAlias);
changed |= !( updatedLeft eq comparisonExpr.left);
val updatedRight = removeUnneededBackReferences(comparisonExpr.right, outerAlias);
changed |= !(updatedRight eq comparisonExpr.right);
if (changed) {
ComparisonExpression(comparisonExpr.symbol, updatedLeft, updatedRight)
} else {
comparisonExpr
}
}
case FieldExpression(fieldName, fieldInfo, Some(br @ BackReference(brAlias, _, _))) if outerAlias.equals(brAlias) => {
//Remove the back reference, since the thing it references is right in front
//of the comparison expression we're in
FieldExpression(fieldName, fieldInfo, None)
}
case x => x
}
//in groupby, convert alias expressions defined in the group by child to BackReferences
//in the groupby list and selectList.
val AddBackReferencesToGroupBy : PartialFunction[Expression, Expression] = {
......@@ -526,10 +456,7 @@ class GremlinTranslator(expr: Expression,
return translateLiteralValue(l.dataType, l);
}
case list: ListLiteral[_] => {
//Here, we are creating a Groovy list literal expression ([value1, value2, value3]). Because
//of this, any gremlin query expressions within the list must start with an anonymous traversal.
//We set 'inClosure' to true in this case to make that happen.
val values : java.util.List[GroovyExpression] = translateList(list.rawValue, true);
val values : java.util.List[GroovyExpression] = translateList(list.rawValue, true); //why hard coded
return new ListExpression(values);
}
case in@TraitInstanceExpression(child) => {
......@@ -566,7 +493,7 @@ class GremlinTranslator(expr: Expression,
case limitOffset@LimitExpression(child, limit, offset) => {
val childExpr = genQuery(parent, child, inClosure);
val totalResultRows = limit.value + offset.value;
return GremlinExpressionFactory.INSTANCE.generateRangeExpression(childExpr, offset.value, totalResultRows);
return GremlinExpressionFactory.INSTANCE.generateLimitExpression(childExpr, offset.value, totalResultRows);
}
case count@CountExpression() => {
val listExpr = GremlinExpressionFactory.INSTANCE.getClosureArgumentValue();
......@@ -694,7 +621,8 @@ class GremlinTranslator(expr: Expression,
def genFullQuery(expr: Expression, hasSelect: Boolean): String = {
var q : GroovyExpression = new FunctionCallExpression(TraversalStepType.START, new IdentifierExpression(TraversalStepType.SOURCE, "g"),"V");
var q : GroovyExpression = new FunctionCallExpression(new IdentifierExpression("g"),"V");
val debug:Boolean = false
if(gPersistenceBehavior.addGraphVertexPrefix(preStatements)) {
......@@ -703,23 +631,15 @@ class GremlinTranslator(expr: Expression,
q = genQuery(q, expr, false)
q = GremlinExpressionFactory.INSTANCE.generateToListExpression(q);
q = new FunctionCallExpression(q, "toList");
q = gPersistenceBehavior.getGraph().addOutputTransformationPredicate(q, hasSelect, expr.isInstanceOf[PathExpression]);
var overallExpression = new CodeBlockExpression();
overallExpression.addStatements(preStatements);
overallExpression.addStatement(q)
overallExpression.addStatements(postStatements);
if(AtlasRepositoryConfiguration.isGremlinOptimizerEnabled()) {
q = GremlinQueryOptimizer.getInstance().optimize(q);
}
val closureExpression = new ClosureExpression();
closureExpression.addStatements(preStatements);
closureExpression.addStatement(q)
closureExpression.addStatements(postStatements);
val overallExpression = new LabeledExpression("L", closureExpression);
val qryStr = generateGremlin(overallExpression);
var qryStr = generateGremlin(overallExpression);
if(debug) {
println(" query " + qryStr)
......@@ -746,7 +666,6 @@ class GremlinTranslator(expr: Expression,
e1 = e1.transformUp(addAliasToLoopInput())
e1 = e1.transformUp(instanceClauseToTop(e1))
e1 = e1.transformUp(traitClauseWithInstanceForTop(e1))
e1 = e1.transformUp(RemoveUnneededBackReferences)
//Following code extracts the select expressions from expression tree.
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -508,9 +508,6 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"from hive_db limit 3 offset 1", 2},
{"hive_db", 3},
{"hive_db where hive_db.name=\"Reporting\"", 1},
{"hive_db where hive_db.name=\"Reporting\" or hive_db.name=\"Sales\" or hive_db.name=\"Logging\" limit 1 offset 1", 1},
{"hive_db where hive_db.name=\"Reporting\" or hive_db.name=\"Sales\" or hive_db.name=\"Logging\" limit 1 offset 2", 1},
{"hive_db where hive_db.name=\"Reporting\" or hive_db.name=\"Sales\" or hive_db.name=\"Logging\" limit 2 offset 1", 2},
{"hive_db where hive_db.name=\"Reporting\" limit 10 ", 1},
{"hive_db hive_db.name = \"Reporting\"", 1},
{"hive_db where hive_db.name=\"Reporting\" select name, owner", 1},
......
repository/src/test/java/org/apache/atlas/repository/graph/TestIntSequence.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import org.apache.atlas.query.IntSequence;
/**
* IntSequence for use in unit tests.
*
*/
public class TestIntSequence implements IntSequence {
public static final IntSequence INSTANCE = new TestIntSequence();
private TestIntSequence() {
}
@Override
public int next() {
return 0;
}
}
\ No newline at end of file
repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
......@@ -886,7 +886,7 @@ class GremlinTest extends BaseGremlinTest {
.or(id("name").`=`(string("Reporting")))).field("Table").as("tab")
.select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp
)
validateJson(r, "{\n \"query\":\"DB as db1 where (createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct6\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct6\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
}
@Test def testJoinAndSelect3 {
......@@ -896,7 +896,7 @@ class GremlinTest extends BaseGremlinTest {
.or(id("db1").hasField("owner"))).field("Table").as("tab")
.select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp
)
validateJson(r, "{\n \"query\":\"DB as db1 where (createTime > 0) and (name = \\\"Reporting\\\") or db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct7\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) and (db1.name = \\\"Reporting\\\") or db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct7\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
}
@Test def testJoinAndSelect4 {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment