Skip to content

A
atlas
Commit 5614bf0d by Ashutosh Mestry Committed by Madhan Neethiraj
Options

ATLAS-2229: DSL implementation using ANTLR #3 - Select, GroupBy, OrderBy

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent faeecf10
Showing with 331 additions and 149 deletions
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
...@@ -720,8 +720,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -720,8 +720,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private AttributeSearchResult toAttributesResult(List results, GremlinQuery query) { private AttributeSearchResult toAttributesResult(List results, GremlinQuery query) {
AttributeSearchResult ret = new AttributeSearchResult(); AttributeSearchResult ret = new AttributeSearchResult();
List<String> names = extractNames(results); // List<String> names = extractNames(results);
List<List<Object>> values = extractValues(results); // List<List<Object>> values = extractValues(results);
List<String> names = (List<String>) results.get(0);
List<List<Object>> values = extractValues(results.subList(1, results.size()));
ret.setName(names); ret.setName(names);
ret.setValues(values); ret.setValues(values);
......
repository/src/main/java/org/apache/atlas/query/DSLVisitor.java
...@@ -21,13 +21,12 @@ package org.apache.atlas.query; ...@@ -21,13 +21,12 @@ package org.apache.atlas.query;
import org.apache.atlas.query.antlr4.AtlasDSLParser.*; import org.apache.atlas.query.antlr4.AtlasDSLParser.*;
import org.apache.atlas.query.antlr4.AtlasDSLParserBaseVisitor; import org.apache.atlas.query.antlr4.AtlasDSLParserBaseVisitor;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Objects;
public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> {
private static final Logger LOG = LoggerFactory.getLogger(DSLVisitor.class); private static final Logger LOG = LoggerFactory.getLogger(DSLVisitor.class);
...@@ -68,7 +67,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { ...@@ -68,7 +67,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> {
} }
queryProcessor.addLimit(ctx.limitClause().NUMBER().toString(), queryProcessor.addLimit(ctx.limitClause().NUMBER().toString(),
(ctx.offsetClause() == null ? "0" : ctx.offsetClause().NUMBER().getText())); (ctx.offsetClause() == null ? "0" : ctx.offsetClause().NUMBER().getText()));
return super.visitLimitOffset(ctx); return super.visitLimitOffset(ctx);
} }
...@@ -78,17 +77,46 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { ...@@ -78,17 +77,46 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> {
LOG.debug("=> DSLVisitor.visitSelectExpr({})", ctx); LOG.debug("=> DSLVisitor.visitSelectExpr({})", ctx);
} }
// Select can have only attributes, aliased attributes or aggregate functions
// Groupby attr also represent select expr, no processing is needed in that case
// visit groupBy would handle the select expr appropriately
if (!(ctx.getParent() instanceof GroupByExpressionContext)) { if (!(ctx.getParent() instanceof GroupByExpressionContext)) {
List<Pair<String, String>> items = new ArrayList<>(); String[] items = new String[ctx.selectExpression().size()];
for (int i = 0; i < ctx.selectExpression().size(); i++) { String[] labels = new String[ctx.selectExpression().size()];
String idf = ctx.selectExpression(i).expr().getText();
String alias = (ctx.selectExpression(i).K_AS() != null) ? QueryProcessor.SelectExprMetadata selectExprMetadata = new QueryProcessor.SelectExprMetadata();
ctx.selectExpression(i).identifier().getText() : "";
items.add(new MutablePair<String, String>(idf, alias)); for (int i = 0; i < ctx.selectExpression().size(); i++) {
SelectExpressionContext selectExpression = ctx.selectExpression(i);
CountClauseContext countClause = selectExpression.expr().compE().countClause();
SumClauseContext sumClause = selectExpression.expr().compE().sumClause();
MinClauseContext minClause = selectExpression.expr().compE().minClause();
MaxClauseContext maxClause = selectExpression.expr().compE().maxClause();
IdentifierContext identifier = selectExpression.identifier();
labels[i] = identifier != null ? identifier.getText() : selectExpression.getText();
if (Objects.nonNull(countClause)) {
items[i] = "count";
selectExprMetadata.setCountIdx(i);
} else if (Objects.nonNull(sumClause)) {
items[i] = sumClause.expr().getText();
selectExprMetadata.setSumIdx(i);
} else if (Objects.nonNull(minClause)) {
items[i] = minClause.expr().getText();
selectExprMetadata.setMinIdx(i);
} else if (Objects.nonNull(maxClause)) {
items[i] = maxClause.expr().getText();
selectExprMetadata.setMaxIdx(i);
} else {
items[i] = selectExpression.expr().getText();
}
} }
queryProcessor.addSelect(items); selectExprMetadata.setItems(items);
selectExprMetadata.setLabels(labels);
queryProcessor.addSelect(selectExprMetadata);
} }
return super.visitSelectExpr(ctx); return super.visitSelectExpr(ctx);
} }
......
repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
...@@ -189,6 +189,14 @@ public class IdentifierHelper { ...@@ -189,6 +189,14 @@ public class IdentifierHelper {
return isPrimitive; return isPrimitive;
} }
public boolean isAttribute() {
return isAttribute;
}
public String getAttributeName() {
return attributeName;
}
public String getEdgeLabel() { public String getEdgeLabel() {
return edgeLabel; return edgeLabel;
} }
......
repository/src/main/java/org/apache/atlas/query/QueryProcessor.java
...@@ -21,20 +21,29 @@ import com.google.common.annotations.VisibleForTesting; ...@@ -21,20 +21,29 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.query.Expressions.Expression; import org.apache.atlas.query.Expressions.Expression;
import org.apache.atlas.type.*; import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import javax.inject.Inject; import javax.inject.Inject;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.StringJoiner;
import java.util.stream.Stream;
public class QueryProcessor { public class QueryProcessor {
private static final Logger LOG = LoggerFactory.getLogger(QueryProcessor.class); private static final Logger LOG = LoggerFactory.getLogger(QueryProcessor.class);
...@@ -42,13 +51,18 @@ public class QueryProcessor { ...@@ -42,13 +51,18 @@ public class QueryProcessor {
private final int DEFAULT_QUERY_RESULT_LIMIT = 25; private final int DEFAULT_QUERY_RESULT_LIMIT = 25;
private final int DEFAULT_QUERY_RESULT_OFFSET = 0; private final int DEFAULT_QUERY_RESULT_OFFSET = 0;
private final List<String> errorList = new ArrayList<>();
private final GremlinClauseList queryClauses = new GremlinClauseList();
private int providedLimit = DEFAULT_QUERY_RESULT_LIMIT;
private int providedOffset = DEFAULT_QUERY_RESULT_OFFSET;
private boolean hasSelect = false;
private boolean isSelectNoop = false;
private boolean hasGrpBy = false;
private final org.apache.atlas.query.Lookup lookup;
private final boolean isNestedQuery; private final boolean isNestedQuery;
private final List<String> errorList = new ArrayList<>();
private final GremlinClauseList queryClauses = new GremlinClauseList();
private int providedLimit = DEFAULT_QUERY_RESULT_LIMIT;
private int providedOffset = DEFAULT_QUERY_RESULT_OFFSET;
private int currentStep; private int currentStep;
private final org.apache.atlas.query.Lookup lookup;
private Context context; private Context context;
@Inject @Inject
...@@ -83,15 +97,6 @@ public class QueryProcessor { ...@@ -83,15 +97,6 @@ public class QueryProcessor {
return expression.isReady(); return expression.isReady();
} }
private void init() {
if (!isNestedQuery) {
add(GremlinClause.G);
add(GremlinClause.V);
} else {
add(GremlinClause.NESTED_START);
}
}
public void addFrom(String typeName) { public void addFrom(String typeName) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("addFrom(typeName={})", typeName); LOG.debug("addFrom(typeName={})", typeName);
...@@ -117,17 +122,6 @@ public class QueryProcessor { ...@@ -117,17 +122,6 @@ public class QueryProcessor {
} }
} }
private void introduceType(IdentifierHelper.Advice ia) {
if (!ia.isPrimitive() && ia.getIntroduceType()) {
add(GremlinClause.OUT, ia.getEdgeLabel());
context.registerActive(ia.getTypeName());
}
}
private IdentifierHelper.Advice getAdvice(String actualTypeName) {
return IdentifierHelper.create(context, lookup, actualTypeName);
}
public void addFromProperty(String typeName, String attribute) { public void addFromProperty(String typeName, String attribute) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("addFromProperty(typeName={}, attribute={})", typeName, attribute); LOG.debug("addFromProperty(typeName={}, attribute={})", typeName, attribute);
...@@ -202,51 +196,60 @@ public class QueryProcessor { ...@@ -202,51 +196,60 @@ public class QueryProcessor {
queryClauses.add(GremlinClause.OR, StringUtils.join(clauses, ',')); queryClauses.add(GremlinClause.OR, StringUtils.join(clauses, ','));
} }
public void addSelect(List<Pair<String, String>> items) { public void addSelect(SelectExprMetadata selectExprMetadata) {
String[] items = selectExprMetadata.getItems();
String[] labels = selectExprMetadata.getLabels();
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("addSelect(items.length={})", items != null ? items.size() : -1); LOG.debug("addSelect(items.length={})", items != null ? items.length : 0);
} }
StringBuilder sb = new StringBuilder(); if (items != null) {
for (int i = 0; i < items.size(); i++) { for (int i = 0; i < items.length; i++) {
IdentifierHelper.Advice ia = getAdvice(items.get(i).getLeft()); IdentifierHelper.Advice ia = getAdvice(items[i]);
if(StringUtils.isNotEmpty(items.get(i).getRight())) {
context.aliasMap.put(items.get(i).getRight(), ia.getQualifiedName());
}
if(!ia.isPrimitive() && ia.getIntroduceType()) { if(!labels[i].equals(items[i])) {
add(GremlinClause.OUT, ia.getEdgeLabel()); context.aliasMap.put(labels[i], ia.getQualifiedName());
add(GremlinClause.AS, getCurrentStep()); }
addSelectClause(getCurrentStep());
incrementCurrentStep();
} else {
sb.append(quoted(ia.getQualifiedName()));
}
if (i != items.size() - 1) { if (i == selectExprMetadata.getCountIdx()) {
sb.append(","); items[i] = GremlinClause.INLINE_COUNT.get();
} else if (i == selectExprMetadata.getMinIdx()) {
items[i] = GremlinClause.INLINE_MIN.get(ia.getQualifiedName(), ia.getQualifiedName());
} else if (i == selectExprMetadata.getMaxIdx()) {
items[i] = GremlinClause.INLINE_MAX.get(ia.getQualifiedName(), ia.getQualifiedName());
} else if (i == selectExprMetadata.getSumIdx()) {
items[i] = GremlinClause.INLINE_SUM.get(ia.getQualifiedName(), ia.getQualifiedName());
} else {
if (!ia.isPrimitive() && ia.getIntroduceType()) {
add(GremlinClause.OUT, ia.getEdgeLabel());
context.registerActive(ia.getTypeName());
int dotIdx = ia.get().indexOf(".");
if (dotIdx != -1) {
IdentifierHelper.Advice iax = getAdvice(ia.get());
items[i] = GremlinClause.INLINE_GET_PROPERTY.get(iax.getQualifiedName());
} else {
isSelectNoop = true;
}
} else {
items[i] = GremlinClause.INLINE_GET_PROPERTY.get(ia.getQualifiedName());
}
}
} }
}
if (!StringUtils.isEmpty(sb.toString())) { // If GroupBy clause exists then the query spits out a List<Map<String, List<AtlasVertex>>> otherwise the query returns List<AtlasVertex>
addValueMapClause(sb.toString()); // Different transformations are needed for DSLs with groupby and w/o groupby
} GremlinClause transformationFn;
} if (isSelectNoop) {
transformationFn = GremlinClause.SELECT_EXPR_NOOP_FN;
} else {
transformationFn = hasGrpBy ? GremlinClause.SELECT_WITH_GRPBY_HELPER_FN : GremlinClause.SELECT_EXPR_HELPER_FN;
}
queryClauses.add(0, transformationFn, getJoinedQuotedStr(labels), String.join(",", items));
queryClauses.add(GremlinClause.INLINE_TRANSFORM_CALL);
private void addSelectClause(String s) { hasSelect = true;
if (LOG.isDebugEnabled()) {
LOG.debug("addSelectClause(s={})", s);
} }
add(GremlinClause.SELECT, s);
}
private String getCurrentStep() {
return String.format("s%d", currentStep);
}
private void incrementCurrentStep() {
currentStep++;
} }
public QueryProcessor createNestedProcessor() { public QueryProcessor createNestedProcessor() {
...@@ -255,14 +258,6 @@ public class QueryProcessor { ...@@ -255,14 +258,6 @@ public class QueryProcessor {
return qp; return qp;
} }
private void addValueMapClause(String s) {
if (LOG.isDebugEnabled()) {
LOG.debug("addValueMapClause(s={})", s);
}
add(GremlinClause.VALUE_MAP, s);
}
public void addFromAlias(String typeName, String alias) { public void addFromAlias(String typeName, String alias) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("addFromAlias(typeName={}, alias={})", typeName, alias); LOG.debug("addFromAlias(typeName={}, alias={})", typeName, alias);
...@@ -281,19 +276,6 @@ public class QueryProcessor { ...@@ -281,19 +276,6 @@ public class QueryProcessor {
add(GremlinClause.AS, stepName); add(GremlinClause.AS, stepName);
} }
private void add(GremlinClause clause, String... args) {
queryClauses.add(new GremlinClauseValue(clause, clause.get(args)));
}
private void addRangeClause(String startIndex, String endIndex) {
if (LOG.isDebugEnabled()) {
LOG.debug("addRangeClause(startIndex={}, endIndex={})", startIndex, endIndex);
}
add(GremlinClause.RANGE, startIndex, startIndex, endIndex);
}
public void addGroupBy(String item) { public void addGroupBy(String item) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("addGroupBy(item={})", item); LOG.debug("addGroupBy(item={})", item);
...@@ -301,16 +283,7 @@ public class QueryProcessor { ...@@ -301,16 +283,7 @@ public class QueryProcessor {
add(GremlinClause.GROUP); add(GremlinClause.GROUP);
addByClause(item, false); addByClause(item, false);
} hasGrpBy = true;
private void addByClause(String name, boolean descr) {
if (LOG.isDebugEnabled()) {
LOG.debug("addByClause(name={})", name, descr);
}
IdentifierHelper.Advice ia = getAdvice(name);
add((!descr) ? GremlinClause.BY : GremlinClause.BY_DESC,
ia.getQualifiedName());
} }
public void addLimit(String limit, String offset) { public void addLimit(String limit, String offset) {
...@@ -335,27 +308,38 @@ public class QueryProcessor { ...@@ -335,27 +308,38 @@ public class QueryProcessor {
addLimit(Integer.toString(providedLimit), Integer.toString(providedOffset)); addLimit(Integer.toString(providedLimit), Integer.toString(providedOffset));
} }
updatePosition(GremlinClause.LIMIT);
add(GremlinClause.TO_LIST); add(GremlinClause.TO_LIST);
updatePosition(GremlinClause.INLINE_TRANSFORM_CALL);
} }
public String getText() { public String getText() {
String ret;
String[] items = new String[queryClauses.size()]; String[] items = new String[queryClauses.size()];
for (int i = 0; i < queryClauses.size(); i++) { int startIdx = hasSelect ? 1 : 0;
int endIdx = hasSelect ? queryClauses.size() - 1 : queryClauses.size();
for (int i = startIdx; i < endIdx; i++) {
items[i] = queryClauses.getValue(i); items[i] = queryClauses.getValue(i);
} }
String ret = StringUtils.join(items, "."); if (hasSelect) {
String body = StringUtils.join(Stream.of(items).filter(Objects::nonNull).toArray(), ".");
String inlineFn = queryClauses.getValue(queryClauses.size() - 1);
String funCall = String.format(inlineFn, body);
ret = queryClauses.getValue(0) + funCall;
} else {
ret = String.join(".", items);
}
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("getText() => {}", ret); LOG.debug("getText() => {}", ret);
} }
return ret; return ret;
} }
public boolean hasSelect() { public boolean hasSelect() {
return (queryClauses.hasClause(GremlinClause.VALUE_MAP) != -1); return hasSelect;
} }
public void addOrderBy(String name, boolean isDesc) { public void addOrderBy(String name, boolean isDesc) {
...@@ -365,19 +349,73 @@ public class QueryProcessor { ...@@ -365,19 +349,73 @@ public class QueryProcessor {
add(GremlinClause.ORDER); add(GremlinClause.ORDER);
addByClause(name, isDesc); addByClause(name, isDesc);
updateSelectClausePosition();
} }
private void updateSelectClausePosition() { private void updatePosition(GremlinClause clause) {
int selectClauseIndex = queryClauses.hasClause(GremlinClause.VALUE_MAP); int index = queryClauses.hasClause(clause);
if(-1 == selectClauseIndex) { if(-1 == index) {
return; return;
} }
GremlinClauseValue gcv = queryClauses.remove(selectClauseIndex); GremlinClauseValue gcv = queryClauses.remove(index);
queryClauses.add(gcv); queryClauses.add(gcv);
} }
private void init() {
if (!isNestedQuery) {
add(GremlinClause.G);
add(GremlinClause.V);
} else {
add(GremlinClause.NESTED_START);
}
}
private void introduceType(IdentifierHelper.Advice ia) {
if (!ia.isPrimitive() && ia.getIntroduceType()) {
add(GremlinClause.OUT, ia.getEdgeLabel());
context.registerActive(ia.getTypeName());
}
}
private IdentifierHelper.Advice getAdvice(String actualTypeName) {
return IdentifierHelper.create(context, lookup, actualTypeName);
}
private String getJoinedQuotedStr(String[] elements) {
StringJoiner joiner = new StringJoiner(",");
Arrays.stream(elements).map(x -> "'" + x + "'").forEach(joiner::add);
return joiner.toString();
}
private void add(GremlinClause clause, String... args) {
queryClauses.add(new GremlinClauseValue(clause, clause.get(args)));
}
private void add(int idx, GremlinClause clause, String... args) {
queryClauses.add(idx, new GremlinClauseValue(clause, clause.get(args)));
}
private void addRangeClause(String startIndex, String endIndex) {
if (LOG.isDebugEnabled()) {
LOG.debug("addRangeClause(startIndex={}, endIndex={})", startIndex, endIndex);
}
if (hasSelect) {
add(queryClauses.size() - 1, GremlinClause.RANGE, startIndex, startIndex, endIndex);
} else {
add(GremlinClause.RANGE, startIndex, startIndex, endIndex);
}
}
private void addByClause(String name, boolean descr) {
if (LOG.isDebugEnabled()) {
LOG.debug("addByClause(name={})", name, descr);
}
IdentifierHelper.Advice ia = getAdvice(name);
add((!descr) ? GremlinClause.BY : GremlinClause.BY_DESC, ia.getQualifiedName());
}
private enum GremlinClause { private enum GremlinClause {
AS("as('%s')"), AS("as('%s')"),
BY("by('%s')"), BY("by('%s')"),
...@@ -407,6 +445,17 @@ public class QueryProcessor { ...@@ -407,6 +445,17 @@ public class QueryProcessor {
TEXT_PREFIX("has('%s', org.janusgraph.core.attribute.Text.textPrefix(%s))"), TEXT_PREFIX("has('%s', org.janusgraph.core.attribute.Text.textPrefix(%s))"),
TEXT_SUFFIX("has('%s', org.janusgraph.core.attribute.Text.textRegex(\".*\" + %s))"), TEXT_SUFFIX("has('%s', org.janusgraph.core.attribute.Text.textRegex(\".*\" + %s))"),
TRAIT("has('__traitNames', within('%s'))"), TRAIT("has('__traitNames', within('%s'))"),
SELECT_EXPR_NOOP_FN("def f(r){ r }; "),
SELECT_EXPR_HELPER_FN("def f(r){ return [[%s]].plus(r.collect({[%s]})).unique(); }; "),
SELECT_WITH_GRPBY_HELPER_FN("def f(r){ return [[%s]].plus(r.collect({it.values()}).flatten().collect({[%s]})).unique(); }; "),
INLINE_COUNT("r.size()"),
INLINE_SUM("r.sum({it.value('%s')}).value('%s')"),
INLINE_MAX("r.max({it.value('%s')}).value('%s')"),
INLINE_MIN("r.min({it.value('%s')}).value('%s')"),
INLINE_GET_PROPERTY("it.value('%s')"),
INLINE_OUT_VERTEX("it.out('%s')"),
INLINE_OUT_VERTEX_VALUE("it.out('%s').value('%s')"), // This might require more closure introduction :(
INLINE_TRANSFORM_CALL("f(%s)"),
V("V()"), V("V()"),
VALUE_MAP("valueMap(%s)"); VALUE_MAP("valueMap(%s)");
...@@ -452,14 +501,30 @@ public class QueryProcessor { ...@@ -452,14 +501,30 @@ public class QueryProcessor {
list.add(g); list.add(g);
} }
public void add(int idx, GremlinClauseValue g) {
list.add(idx, g);
}
public void add(GremlinClauseValue g, AtlasEntityType t) { public void add(GremlinClauseValue g, AtlasEntityType t) {
add(g); add(g);
} }
public void add(int idx, GremlinClauseValue g, AtlasEntityType t) {
add(idx, g);
}
public void add(GremlinClause clause, String... args) { public void add(GremlinClause clause, String... args) {
list.add(new GremlinClauseValue(clause, clause.get(args))); list.add(new GremlinClauseValue(clause, clause.get(args)));
} }
public void add(int i, GremlinClause clause, String... args) {
list.add(i, new GremlinClauseValue(clause, clause.get(args)));
}
public GremlinClauseValue getAt(int i) {
return list.get(i);
}
public String getValue(int i) { public String getValue(int i) {
return list.get(i).value; return list.get(i).value;
} }
...@@ -500,8 +565,8 @@ public class QueryProcessor { ...@@ -500,8 +565,8 @@ public class QueryProcessor {
static class Context { static class Context {
private final List<String> errorList; private final List<String> errorList;
org.apache.atlas.query.Lookup lookup; org.apache.atlas.query.Lookup lookup;
private AtlasType activeType;
Map<String, String> aliasMap = new HashMap<>(); Map<String, String> aliasMap = new HashMap<>();
private AtlasType activeType;
public Context(List<String> errorList, org.apache.atlas.query.Lookup lookup) { public Context(List<String> errorList, org.apache.atlas.query.Lookup lookup) {
this.lookup = lookup; this.lookup = lookup;
...@@ -682,16 +747,71 @@ public class QueryProcessor { ...@@ -682,16 +747,71 @@ public class QueryProcessor {
@Override @Override
public boolean isDate(Context context, String attributeName) { public boolean isDate(Context context, String attributeName) {
AtlasEntityType et = context.getActiveEntityType(); AtlasEntityType et = context.getActiveEntityType();
if(et == null) { if (et == null) {
return false; return false;
} }
AtlasType attr = et.getAttributeType(attributeName); AtlasType attr = et.getAttributeType(attributeName);
if(attr == null) { return attr != null && attr.getTypeName().equals(AtlasBaseTypeDef.ATLAS_TYPE_DATE);
return false;
} }
}
static class SelectExprMetadata {
private String[] items;
private String[] labels;
private int countIdx = -1;
private int sumIdx = -1;
private int maxIdx = -1;
private int minIdx = -1;
public String[] getItems() {
return items;
}
public int getCountIdx() {
return countIdx;
}
public void setCountIdx(final int countIdx) {
this.countIdx = countIdx;
}
public int getSumIdx() {
return sumIdx;
}
public void setSumIdx(final int sumIdx) {
this.sumIdx = sumIdx;
}
public int getMaxIdx() {
return maxIdx;
}
public void setMaxIdx(final int maxIdx) {
this.maxIdx = maxIdx;
}
public int getMinIdx() {
return minIdx;
}
public void setMinIdx(final int minIdx) {
this.minIdx = minIdx;
}
public String[] getLabels() {
return labels;
}
public void setItems(final String[] items) {
this.items = items;
}
return attr.getTypeName().equals("date"); public void setLabels(final String[] labels) {
this.labels = labels;
} }
} }
} }
repository/src/test/java/org/apache/atlas/query/QueryProcessorTest.java
...@@ -32,9 +32,7 @@ import java.io.ByteArrayInputStream; ...@@ -32,9 +32,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
...@@ -72,8 +70,8 @@ public class QueryProcessorTest { ...@@ -72,8 +70,8 @@ public class QueryProcessorTest {
@Test @Test
public void DBHasName() { public void DBHasName() {
String expected = "g.V().has('__typeName', 'DB').has('DB.name').limit(25).toList()"; String expected = "g.V().has('__typeName', 'DB').has('DB.name').limit(25).toList()";
// verify("DB has name", expected); verify("DB has name", expected);
verify("DB where DB has name", expected); verify("DB where DB has name", expected);
} }
@Test @Test
...@@ -83,15 +81,19 @@ public class QueryProcessorTest { ...@@ -83,15 +81,19 @@ public class QueryProcessorTest {
@Test @Test
public void DBasDSelect() { public void DBasDSelect() {
String expected = "g.V().has('__typeName', 'DB').as('d').valueMap('DB.name','DB.owner')"; String expected = "def f(r){ return [['d.name','d.owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }; f(g.V().has('__typeName', 'DB').as('d')";
verify("DB as d select d.name, d.owner", expected + ".limit(25).toList()"); verify("DB as d select d.name, d.owner", expected + ".limit(25).toList())");
verify("DB as d select d.name, d.owner limit 10", expected + ".limit(10).toList()"); verify("DB as d select d.name, d.owner limit 10", expected + ".limit(10).toList())");
} }
@Test @Test
public void tableSelectColumns() { public void tableSelectColumns() {
verify("Table select columns limit 10", "g.V().has('__typeName', 'Table').out('__Table.columns').as('s0').select('s0').limit(10).toList()"); String exMain = "g.V().has('__typeName', 'Table').out('__Table.columns').limit(10).toList()";
verify("Table select db.name", "g.V().has('__typeName', 'Table').out('__DB.Table').as('s0').select('s0').limit(25).toList()"); String exSel = "def f(r){ r }";
verify("Table select columns limit 10", getExpected(exSel, exMain));
String exMain2 = "g.V().has('__typeName', 'Table').out('__Table.db').limit(25).toList()";
verify("Table select db.name", getExpected(exSel, exMain2));
} }
@Test(enabled = false) @Test(enabled = false)
...@@ -112,9 +114,16 @@ public class QueryProcessorTest { ...@@ -112,9 +114,16 @@ public class QueryProcessorTest {
verify("from DB orderby name", expected); verify("from DB orderby name", expected);
verify("from DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()"); verify("from DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()");
verify("DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()"); verify("DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()");
verify("DB as d select name, owner orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').valueMap('DB.name','DB.owner').limit(3).toList()");
String exSel = "def f(r){ return [['d.name','d.owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }";
String exMain = "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner)').limit(25).toList()";
verify("DB as d select d.name, d.owner orderby (d.owner) limit 25", getExpected(exSel, exMain));
String exMain2 = "g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.createTime', gt('1388563200000'))).order().by('Table.createTime').limit(25).toList()";
String exSel2 = "def f(r){ return [['_col_0','_col_1']].plus(r.collect({[it.value('Table.name'),it.value('Table.createTime')]})).unique(); }";
verify("Table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 orderby _col_1", verify("Table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 orderby _col_1",
"g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.createTime', gt('1388563200000'))).order().by('Table.createTime').valueMap('Table.name','Table.createTime').limit(25).toList()"); getExpected(exSel2, exMain2));
} }
@Test @Test
...@@ -124,47 +133,55 @@ public class QueryProcessorTest { ...@@ -124,47 +133,55 @@ public class QueryProcessorTest {
@Test @Test
public void fromDBSelect() { public void fromDBSelect() {
verify("from DB select DB.name, DB.owner", "g.V().has('__typeName', 'DB').valueMap('DB.name','DB.owner').limit(25).toList()"); String expected = "def f(r){ return [['DB.name','DB.owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }; f(g.V().has('__typeName', 'DB').limit(25).toList())";
verify("from DB select DB.name, DB.owner", expected);
} }
@Test @Test
public void fromDBSelectGroupBy() { public void fromDBGroupBy() {
verify("from DB groupby (DB.owner)", "g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList()"); verify("from DB groupby (DB.owner)", "g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList()");
} }
@Test @Test
public void whereClauseTextContains() { public void whereClauseTextContains() {
String expected = "g.V().has('__typeName', 'DB').has('DB.name', eq(\"Reporting\")).valueMap('DB.name','DB.owner').limit(25).toList()"; String exMain = "g.V().has('__typeName', 'DB').has('DB.name', eq(\"Reporting\")).limit(25).toList()";
verify("from DB where name = \"Reporting\" select name, owner", expected); String exSel = "def f(r){ return [['name','owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }";
verify("from DB where name = \"Reporting\" select name, owner", getExpected(exSel, exMain));
verify("from DB where (name = \"Reporting\") select name, owner", getExpected(exSel, exMain));
verify("Table where Asset.name like \"Tab*\"", verify("Table where Asset.name like \"Tab*\"",
"g.V().has('__typeName', 'Table').has('Table.name', org.janusgraph.core.attribute.Text.textRegex(\"Tab.*\")).limit(25).toList()"); "g.V().has('__typeName', 'Table').has('Table.name', org.janusgraph.core.attribute.Text.textRegex(\"Tab.*\")).limit(25).toList()");
verify("from DB where (name = \"Reporting\") select name, owner", expected);
verify("from Table where (db.name = \"Reporting\")", verify("from Table where (db.name = \"Reporting\")",
"g.V().has('__typeName', 'Table').out('__DB.Table').has('DB.name', eq(\"Reporting\")).in('__DB.Table').limit(25).toList()"); "g.V().has('__typeName', 'Table').out('__Table.db').has('DB.name', eq(\"Reporting\")).in('__Table.db').limit(25).toList()");
} }
@Test @Test
public void whereClauseWithAsTextContains() { public void whereClauseWithAsTextContains() {
verify("Table as t where t.name = \"testtable_1\" select t.name, t.owner)", String exSel = "def f(r){ return [['t.name','t.owner']].plus(r.collect({[it.value('Table.name'),it.value('Table.owner')]})).unique(); }";
"g.V().has('__typeName', 'Table').as('t').has('Table.name', eq(\"testtable_1\")).valueMap('Table.name','Table.owner').limit(25).toList()"); String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.name', eq(\"testtable_1\")).limit(25).toList()";
verify("Table as t where t.name = \"testtable_1\" select t.name, t.owner)", getExpected(exSel, exMain));
} }
@Test @Test
public void whereClauseWithDateCompare() { public void whereClauseWithDateCompare() {
verify("Table as t where t.createdTime = \"2017-12-12T02:35:58.440Z\" select t.name, t.owner)", String exSel = "def f(r){ return [['t.name','t.owner']].plus(r.collect({[it.value('Table.name'),it.value('Table.owner')]})).unique(); }";
"g.V().has('__typeName', 'Table').as('t').has('Table.createdTime', eq('1513046158440')).valueMap('Table.name','Table.owner').limit(25).toList()"); String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.createdTime', eq('1513046158440')).limit(25).toList()";
verify("Table as t where t.createdTime = \"2017-12-12T02:35:58.440Z\" select t.name, t.owner)", getExpected(exSel, exMain));
} }
@Test @Test
public void multipleWhereClauses() { public void multipleWhereClauses() {
verify("Table where name=\"sales_fact\", columns as c select c.owner, c.name, c.dataType", String exSel = "def f(r){ return [['c.owner','c.name','c.dataType']].plus(r.collect({[it.value('Column.owner'),it.value('Column.name'),it.value('Column.dataType')]})).unique(); }";
"g.V().has('__typeName', 'Table').has('Table.name', eq(\"sales_fact\")).out('__Table.columns').as('c').valueMap('Column.owner','Column.name','Column.dataType').limit(25).toList()"); String exMain = "g.V().has('__typeName', 'Table').has('Table.name', eq(\"sales_fact\")).out('__Table.columns').as('c').limit(25).toList()";
verify("Table where name=\"sales_fact\", columns as c select c.owner, c.name, c.dataType", getExpected(exSel, exMain));
;
} }
@Test @Test
public void subType() { public void subType() {
verify("Asset select name, owner", String exMain = "g.V().has('__typeName', within('Asset','Table')).limit(25).toList()";
"g.V().has('__typeName', within('Asset','Table')).valueMap('Asset.name','Asset.owner').limit(25).toList()"); String exSel = "def f(r){ return [['name','owner']].plus(r.collect({[it.value('Asset.name'),it.value('Asset.owner')]})).unique(); }";
verify("Asset select name, owner", getExpected(exSel, exMain));
} }
@Test @Test
...@@ -236,7 +253,7 @@ public class QueryProcessorTest { ...@@ -236,7 +253,7 @@ public class QueryProcessorTest {
verify("hive_db where hive_db.name='Reporting' and hive_db.createTime < '2017-12-12T02:35:58.440Z'", verify("hive_db where hive_db.name='Reporting' and hive_db.createTime < '2017-12-12T02:35:58.440Z'",
"g.V().has('__typeName', 'hive_db').and(__.has('hive_db.name', eq('Reporting')),__.has('hive_db.createTime', lt('1513046158440'))).limit(25).toList()"); "g.V().has('__typeName', 'hive_db').and(__.has('hive_db.name', eq('Reporting')),__.has('hive_db.createTime', lt('1513046158440'))).limit(25).toList()");
verify("Table where db.name='Sales' and db.clusterName='cl1'", verify("Table where db.name='Sales' and db.clusterName='cl1'",
"g.V().has('__typeName', 'Table').and(__.out('__DB.Table').has('DB.name', eq('Sales')).in('__DB.Table'),__.out('__DB.Table').has('DB.clusterName', eq('cl1')).in('__DB.Table')).limit(25).toList()"); "g.V().has('__typeName', 'Table').and(__.out('__Table.db').has('DB.name', eq('Sales')).in('__Table.db'),__.out('__Table.db').has('DB.clusterName', eq('cl1')).in('__Table.db')).limit(25).toList()");
} }
private void verify(String dsl, String expectedGremlin) { private void verify(String dsl, String expectedGremlin) {
...@@ -245,6 +262,10 @@ public class QueryProcessorTest { ...@@ -245,6 +262,10 @@ public class QueryProcessorTest {
assertEquals(actualGremlin, expectedGremlin); assertEquals(actualGremlin, expectedGremlin);
} }
private String getExpected(String select, String main) {
return String.format("%s; f(%s)", select, main);
}
private AtlasDSLParser.QueryContext getParsedQuery(String query) { private AtlasDSLParser.QueryContext getParsedQuery(String query) {
AtlasDSLParser.QueryContext queryContext = null; AtlasDSLParser.QueryContext queryContext = null;
InputStream stream = new ByteArrayInputStream(query.getBytes()); InputStream stream = new ByteArrayInputStream(query.getBytes());
...@@ -276,8 +297,9 @@ public class QueryProcessorTest { ...@@ -276,8 +297,9 @@ public class QueryProcessorTest {
qv.visit(queryContext); qv.visit(queryContext);
queryProcessor.close(); queryProcessor.close();
assertTrue(StringUtils.isNotEmpty(queryProcessor.getText())); String s = queryProcessor.getText();
return queryProcessor.getText(); assertTrue(StringUtils.isNotEmpty(s));
return s;
} }
private static class TestLookup implements org.apache.atlas.query.Lookup { private static class TestLookup implements org.apache.atlas.query.Lookup {
...@@ -324,6 +346,8 @@ public class QueryProcessorTest { ...@@ -324,6 +346,8 @@ public class QueryProcessorTest {
public String getRelationshipEdgeLabel(QueryProcessor.Context context, String attributeName) { public String getRelationshipEdgeLabel(QueryProcessor.Context context, String attributeName) {
if (attributeName.equalsIgnoreCase("columns")) if (attributeName.equalsIgnoreCase("columns"))
return "__Table.columns"; return "__Table.columns";
if (attributeName.equalsIgnoreCase("db"))
return "__Table.db";
else else
return "__DB.Table"; return "__DB.Table";
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment