Skip to content

A
atlas
Commit b5989c84 by Ashutosh Mestry Committed by apoorvnaik
Options

Bugfixes for ATLAS-2333, ATLAS-2334

ATLAS-2229: ValueArray clause implementation. ATLAS-2229: DSL valueArray clause, added unit tests. ATLAS-2229: DSLVisitor refactoring. ATLAS-2229: Fixed REST call with classification. ATLAS-2229: GroupBy, Select, Min, Max, Count ATLAS-2229: Additional unit tests for min, max, count. Updated test framework. ATLAS-2229: Handling case where query is invalid. ATLAS-2229: Updates related to boolean value handling. ATLAS-2229: Count, Sum, Max, Min operations implemented. ATLAS-2229: Addressed review comments. Signed-off-by: 's avatarapoorvnaik <apoorvnaik@apache.org>
parent ab316c9b
Showing with 1224 additions and 632 deletions
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
...@@ -19,7 +19,6 @@ package org.apache.atlas.discovery; ...@@ -19,7 +19,6 @@ package org.apache.atlas.discovery;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.ApplicationProperties; import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.SortOrder; import org.apache.atlas.SortOrder;
...@@ -36,6 +35,7 @@ import org.apache.atlas.model.profile.AtlasUserSavedSearch; ...@@ -36,6 +35,7 @@ import org.apache.atlas.model.profile.AtlasUserSavedSearch;
import org.apache.atlas.query.AtlasDSL; import org.apache.atlas.query.AtlasDSL;
import org.apache.atlas.query.GremlinQuery; import org.apache.atlas.query.GremlinQuery;
import org.apache.atlas.query.QueryParams; import org.apache.atlas.query.QueryParams;
import org.apache.atlas.query.antlr4.AtlasDSLParser;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graph.GraphHelper;
...@@ -67,21 +67,16 @@ import javax.inject.Inject; ...@@ -67,21 +67,16 @@ import javax.inject.Inject;
import javax.script.Bindings; import javax.script.Bindings;
import javax.script.ScriptEngine; import javax.script.ScriptEngine;
import javax.script.ScriptException; import javax.script.ScriptException;
import java.util.ArrayList; import java.util.*;
import java.util.Arrays;
import java.util.Collection; import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND;
import java.util.Collections; import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
import java.util.HashMap; import static org.apache.atlas.AtlasErrorCode.UNKNOWN_TYPENAME;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.AtlasErrorCode.*;
import static org.apache.atlas.SortOrder.ASCENDING; import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.SortOrder.DESCENDING; import static org.apache.atlas.SortOrder.DESCENDING;
import static org.apache.atlas.model.TypeCategory.*; import static org.apache.atlas.model.TypeCategory.ARRAY;
import static org.apache.atlas.model.TypeCategory.MAP;
import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE; import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
...@@ -177,7 +172,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -177,7 +172,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, boolean excludeDeletedEntities, int limit, int offset) public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, boolean excludeDeletedEntities, int limit, int offset)
throws AtlasBaseException { throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT); AtlasSearchResult ret = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT);
QueryParams params = validateSearchParams(limit, offset); QueryParams params = QueryParams.getNormalizedParams(limit, offset);
AtlasIndexQuery idxQuery = toAtlasIndexQuery(fullTextQuery); AtlasIndexQuery idxQuery = toAtlasIndexQuery(fullTextQuery);
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
...@@ -200,7 +195,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -200,7 +195,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
LOG.debug("Executing basic search query: {} with type: {} and classification: {}", query, typeName, classification); LOG.debug("Executing basic search query: {} with type: {} and classification: {}", query, typeName, classification);
} }
final QueryParams params = validateSearchParams(limit, offset); final QueryParams params = QueryParams.getNormalizedParams(limit, offset);
Set<String> typeNames = null; Set<String> typeNames = null;
Set<String> classificationNames = null; Set<String> classificationNames = null;
String attrQualifiedName = null; String attrQualifiedName = null;
...@@ -417,7 +412,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -417,7 +412,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@GraphTransaction @GraphTransaction
public AtlasSearchResult searchWithParameters(SearchParameters searchParameters) throws AtlasBaseException { public AtlasSearchResult searchWithParameters(SearchParameters searchParameters) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(searchParameters); AtlasSearchResult ret = new AtlasSearchResult(searchParameters);
final QueryParams params = validateSearchParams(searchParameters.getLimit(),searchParameters.getOffset()); final QueryParams params = QueryParams.getNormalizedParams(searchParameters.getLimit(),searchParameters.getOffset());
searchParameters.setLimit(params.limit()); searchParameters.setLimit(params.limit());
searchParameters.setOffset(params.offset()); searchParameters.setOffset(params.offset());
...@@ -554,7 +549,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -554,7 +549,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
} }
} }
QueryParams params = validateSearchParams(limit, offset); QueryParams params = QueryParams.getNormalizedParams(limit, offset);
ScriptEngine scriptEngine = graph.getGremlinScriptEngine(); ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
Bindings bindings = scriptEngine.createBindings(); Bindings bindings = scriptEngine.createBindings();
Set<String> states = getEntityStates(); Set<String> states = getEntityStates();
...@@ -675,8 +670,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -675,8 +670,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
} }
private GremlinQuery toGremlinQuery(String query, int limit, int offset) throws AtlasBaseException { private GremlinQuery toGremlinQuery(String query, int limit, int offset) throws AtlasBaseException {
QueryParams params = validateSearchParams(limit, offset); QueryParams params = QueryParams.getNormalizedParams(limit, offset);
GremlinQuery gremlinQuery = new AtlasDSL.Translator(AtlasDSL.Parser.parse(query), typeRegistry, params.offset(), params.limit()).translate(); GremlinQuery gremlinQuery = new AtlasDSL.Translator(query, typeRegistry, params.offset(), params.limit()).translate();
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Translated Gremlin Query: {}", gremlinQuery.queryStr()); LOG.debug("Translated Gremlin Query: {}", gremlinQuery.queryStr());
...@@ -685,23 +680,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -685,23 +680,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return gremlinQuery; return gremlinQuery;
} }
private QueryParams validateSearchParams(int limitParam, int offsetParam) {
int defaultLimit = AtlasConfiguration.SEARCH_DEFAULT_LIMIT.getInt();
int maxLimit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt();
int limit = defaultLimit;
if (limitParam > 0 && limitParam <= maxLimit) {
limit = limitParam;
}
int offset = 0;
if (offsetParam > 0) {
offset = offsetParam;
}
return new QueryParams(limit, offset);
}
private AtlasIndexQuery toAtlasIndexQuery(String fullTextQuery) { private AtlasIndexQuery toAtlasIndexQuery(String fullTextQuery) {
String graphQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, fullTextQuery); String graphQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, fullTextQuery);
return graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery); return graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery);
...@@ -709,8 +687,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -709,8 +687,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private AttributeSearchResult toAttributesResult(List results, GremlinQuery query) { private AttributeSearchResult toAttributesResult(List results, GremlinQuery query) {
AttributeSearchResult ret = new AttributeSearchResult(); AttributeSearchResult ret = new AttributeSearchResult();
// List<String> names = extractNames(results);
// List<List<Object>> values = extractValues(results);
List<String> names = (List<String>) results.get(0); List<String> names = (List<String>) results.get(0);
List<List<Object>> values = extractValues(results.subList(1, results.size())); List<List<Object>> values = extractValues(results.subList(1, results.size()));
...@@ -920,6 +896,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -920,6 +896,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override @Override
public String getDslQueryUsingTypeNameClassification(String query, String typeName, String classification) { public String getDslQueryUsingTypeNameClassification(String query, String typeName, String classification) {
final String whereDSLKeyword = "where"; final String whereDSLKeyword = "where";
final String isaDSLKeyword = "isa";
final String isDSLKeyword = "is";
final String limitDSLKeyword = "limit"; final String limitDSLKeyword = "limit";
final String whereFormat = whereDSLKeyword + " %s"; final String whereFormat = whereDSLKeyword + " %s";
...@@ -928,7 +906,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -928,7 +906,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
if (StringUtils.isNotEmpty(typeName)) { if (StringUtils.isNotEmpty(typeName)) {
if(StringUtils.isNotEmpty(query)) { if(StringUtils.isNotEmpty(query)) {
String s = query.toLowerCase(); String s = query.toLowerCase();
if(!s.startsWith(whereDSLKeyword) && !s.startsWith(limitDSLKeyword)) { if(!s.startsWith(whereDSLKeyword) &&
!s.startsWith(limitDSLKeyword) &&
!s.startsWith(isaDSLKeyword) &&
!s.startsWith(isDSLKeyword)) {
queryStr = String.format(whereFormat, query); queryStr = String.format(whereFormat, query);
} }
} }
...@@ -939,7 +920,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -939,7 +920,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
if (StringUtils.isNotEmpty(classification)) { if (StringUtils.isNotEmpty(classification)) {
// isa works with a type name only - like hive_column isa PII; it doesn't work with more complex query // isa works with a type name only - like hive_column isa PII; it doesn't work with more complex query
if (StringUtils.isEmpty(query)) { if (StringUtils.isEmpty(query)) {
queryStr += (" isa " + classification); queryStr += String.format("%s %s %s", queryStr, isaDSLKeyword, classification);
} }
} }
return queryStr; return queryStr;
......
repository/src/main/java/org/apache/atlas/query/AtlasDSL.java
...@@ -53,7 +53,7 @@ public class AtlasDSL { ...@@ -53,7 +53,7 @@ public class AtlasDSL {
return RESERVED_KEYWORDS.contains(word); return RESERVED_KEYWORDS.contains(word);
} }
public static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException { private static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException {
AtlasDSLParser.QueryContext ret; AtlasDSLParser.QueryContext ret;
try { try {
...@@ -88,7 +88,6 @@ public class AtlasDSL { ...@@ -88,7 +88,6 @@ public class AtlasDSL {
@Override @Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine, final String msg, final RecognitionException e) { public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine, final String msg, final RecognitionException e) {
// TODO: Capture multiple datapoints
isValid = false; isValid = false;
errorMsg = msg; errorMsg = msg;
} }
...@@ -108,7 +107,11 @@ public class AtlasDSL { ...@@ -108,7 +107,11 @@ public class AtlasDSL {
private final int offset; private final int offset;
private final int limit; private final int limit;
public Translator(final AtlasDSLParser.QueryContext queryContext, AtlasTypeRegistry typeRegistry, int offset, int limit) { public Translator(String query, AtlasTypeRegistry typeRegistry, int offset, int limit) throws AtlasBaseException {
this(Parser.parse(query), typeRegistry, offset, limit);
}
private Translator(final AtlasDSLParser.QueryContext queryContext, AtlasTypeRegistry typeRegistry, int offset, int limit) {
this.queryContext = queryContext; this.queryContext = queryContext;
this.typeRegistry = typeRegistry; this.typeRegistry = typeRegistry;
this.offset = offset; this.offset = offset;
...@@ -116,20 +119,46 @@ public class AtlasDSL { ...@@ -116,20 +119,46 @@ public class AtlasDSL {
} }
public GremlinQuery translate() { public GremlinQuery translate() {
GremlinQueryComposer gremlinQueryComposer = new GremlinQueryComposer(typeRegistry); QueryMetadata queryMetadata = new QueryMetadata(queryContext);
GremlinQueryComposer gremlinQueryComposer = new GremlinQueryComposer(typeRegistry, queryMetadata, limit, offset);
if (offset >= 0) {
if (!gremlinQueryComposer.hasLimitOffset()) {
gremlinQueryComposer.addLimit(Integer.toString(limit), Integer.toString(offset));
}
}
DSLVisitor dslVisitor = new DSLVisitor(gremlinQueryComposer); DSLVisitor dslVisitor = new DSLVisitor(gremlinQueryComposer);
// Now process the Query and collect translation in
queryContext.accept(dslVisitor); queryContext.accept(dslVisitor);
return new GremlinQuery(gremlinQueryComposer.get(), queryMetadata.hasSelect());
}
}
public static class QueryMetadata {
private boolean hasSelect;
private boolean hasGroupBy;
private boolean hasOrderBy;
private boolean hasLimitOffset;
public QueryMetadata(AtlasDSLParser.QueryContext queryContext) {
hasSelect = queryContext != null && queryContext.selectClause() != null;
hasGroupBy = queryContext != null && queryContext.groupByExpression() != null;
hasOrderBy = queryContext != null && queryContext.orderByExpr() != null;
hasLimitOffset = queryContext != null && queryContext.limitOffset() != null;
}
public boolean hasSelect() {
return hasSelect;
}
public boolean hasGroupBy() {
return hasGroupBy;
}
public boolean hasOrderBy() {
return hasOrderBy;
}
public boolean hasLimitOffset() {
return hasLimitOffset;
}
return new GremlinQuery(gremlinQueryComposer.get(), gremlinQueryComposer.hasSelect()); public boolean needTransformation() {
return (hasGroupBy && hasSelect && hasOrderBy) || (hasGroupBy && hasOrderBy) || hasSelect;
} }
} }
} }
repository/src/main/java/org/apache/atlas/query/DSLVisitor.java
...@@ -66,7 +66,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> { ...@@ -66,7 +66,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
LOG.debug("=> DSLVisitor.visitLimitOffset({})", ctx); LOG.debug("=> DSLVisitor.visitLimitOffset({})", ctx);
} }
gremlinQueryComposer.addLimit(ctx.limitClause().NUMBER().toString(), gremlinQueryComposer.addLimit(ctx.limitClause().NUMBER().getText(),
(ctx.offsetClause() == null ? "0" : ctx.offsetClause().NUMBER().getText())); (ctx.offsetClause() == null ? "0" : ctx.offsetClause().NUMBER().getText()));
return super.visitLimitOffset(ctx); return super.visitLimitOffset(ctx);
} }
...@@ -85,7 +85,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> { ...@@ -85,7 +85,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
String[] items = new String[ctx.selectExpression().size()]; String[] items = new String[ctx.selectExpression().size()];
String[] labels = new String[ctx.selectExpression().size()]; String[] labels = new String[ctx.selectExpression().size()];
GremlinQueryComposer.SelectExprMetadata selectExprMetadata = new GremlinQueryComposer.SelectExprMetadata(); SelectClauseComposer selectClauseComposer = new SelectClauseComposer();
for (int i = 0; i < ctx.selectExpression().size(); i++) { for (int i = 0; i < ctx.selectExpression().size(); i++) {
SelectExpressionContext selectExpression = ctx.selectExpression(i); SelectExpressionContext selectExpression = ctx.selectExpression(i);
...@@ -99,24 +99,24 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> { ...@@ -99,24 +99,24 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
if (Objects.nonNull(countClause)) { if (Objects.nonNull(countClause)) {
items[i] = "count"; items[i] = "count";
selectExprMetadata.setCountIdx(i); selectClauseComposer.setCountIdx(i);
} else if (Objects.nonNull(sumClause)) { } else if (Objects.nonNull(sumClause)) {
items[i] = sumClause.expr().getText(); items[i] = sumClause.expr().getText();
selectExprMetadata.setSumIdx(i); selectClauseComposer.setSumIdx(i);
} else if (Objects.nonNull(minClause)) { } else if (Objects.nonNull(minClause)) {
items[i] = minClause.expr().getText(); items[i] = minClause.expr().getText();
selectExprMetadata.setMinIdx(i); selectClauseComposer.setMinIdx(i);
} else if (Objects.nonNull(maxClause)) { } else if (Objects.nonNull(maxClause)) {
items[i] = maxClause.expr().getText(); items[i] = maxClause.expr().getText();
selectExprMetadata.setMaxIdx(i); selectClauseComposer.setMaxIdx(i);
} else { } else {
items[i] = selectExpression.expr().getText(); items[i] = selectExpression.expr().getText();
} }
} }
selectExprMetadata.setItems(items); selectClauseComposer.setItems(items);
selectExprMetadata.setLabels(labels); selectClauseComposer.setLabels(labels);
gremlinQueryComposer.addSelect(selectExprMetadata); gremlinQueryComposer.addSelect(selectClauseComposer);
} }
return super.visitSelectExpr(ctx); return super.visitSelectExpr(ctx);
} }
...@@ -251,8 +251,16 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> { ...@@ -251,8 +251,16 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
if (comparisonClause != null) { if (comparisonClause != null) {
String lhs = comparisonClause.arithE(0).getText(); String lhs = comparisonClause.arithE(0).getText();
String op = comparisonClause.operator().getText().toUpperCase(); String op, rhs;
String rhs = comparisonClause.arithE(1).getText(); AtomEContext atomECtx = comparisonClause.arithE(1).multiE().atomE();
if(atomECtx.literal() == null ||
(atomECtx.literal() != null && atomECtx.literal().valueArray() == null)) {
op = comparisonClause.operator().getText().toUpperCase();
rhs = comparisonClause.arithE(1).getText();
} else {
op = "in";
rhs = getInClause(atomECtx);
}
gremlinQueryComposer.addWhere(lhs, op, rhs); gremlinQueryComposer.addWhere(lhs, op, rhs);
} else { } else {
...@@ -260,4 +268,16 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> { ...@@ -260,4 +268,16 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
} }
} }
} }
private String getInClause(AtomEContext atomEContext) {
StringBuilder sb = new StringBuilder();
ValueArrayContext valueArrayContext = atomEContext.literal().valueArray();
int startIdx = 1;
int endIdx = valueArrayContext.children.size() - 1;
for (int i = startIdx; i < endIdx; i++) {
sb.append(valueArrayContext.getChild(i));
}
return sb.toString();
}
} }
repository/src/main/java/org/apache/atlas/query/GremlinClause.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.query;
enum GremlinClause {
AS("as('%s')"),
DEDUP("dedup()"),
G("g"),
GROUP_BY("group().by('%s')"),
HAS("has('%s', %s)"),
HAS_OPERATOR("has('%s', %s(%s))"),
HAS_PROPERTY("has('%s')"),
WHERE("where(%s)"),
HAS_NOT_PROPERTY("hasNot('%s')"),
HAS_TYPE("has('__typeName', '%s')"),
HAS_TYPE_WITHIN("has('__typeName', within(%s))"),
HAS_WITHIN("has('%s', within(%s))"),
IN("in('%s')"),
OR("or(%s)"),
AND("and(%s)"),
NESTED_START("__"),
NESTED_HAS_OPERATOR("has('%s', %s(%s))"),
LIMIT("limit(%s)"),
ORDER_BY("order().by('%s')"),
ORDER_BY_DESC("order().by('%s', decr)"),
OUT("out('%s')"),
RANGE("range(%s, %s + %s)"),
SELECT("select('%s')"),
TO_LIST("toList()"),
TEXT_CONTAINS("has('%s', org.janusgraph.core.attribute.Text.textRegex(%s))"),
TEXT_PREFIX("has('%s', org.janusgraph.core.attribute.Text.textPrefix(%s))"),
TEXT_SUFFIX("has('%s', org.janusgraph.core.attribute.Text.textRegex(\".*\" + %s))"),
TRAIT("has('__traitNames', within('%s'))"),
SELECT_NOOP_FN("def f(r){ r }; "),
SELECT_FN("def f(r){ t=[[%s]]; %s r.each({t.add([%s])}); t.unique(); }; "),
SELECT_ONLY_AGG_FN("def f(r){ t=[[%s]]; %s t.add([%s]); t;}; "),
SELECT_ONLY_AGG_GRP_FN("def f(l){ t=[[%s]]; l.get(0).each({k,r -> L:{ %s t.add([%s]); } }); t; }; "),
SELECT_MULTI_ATTR_GRP_FN("def f(l){ t=[[%s]]; l.get(0).each({k,r -> L:{ %s r.each({t.add([%s])}) } }); t.unique(); }; "),
INLINE_ASSIGNMENT("def %s=%s;"),
INLINE_LIST_RANGE("[%s..<%s]"),
INLINE_COUNT("r.size()"),
INLINE_SUM("r.sum({it.value('%s')})"),
INLINE_MAX("r.max({it.value('%s')}).value('%s')"),
INLINE_MIN("r.min({it.value('%s')}).value('%s')"),
INLINE_GET_PROPERTY("it.value('%s')"),
INLINE_TRANSFORM_CALL("f(%s)"),
V("V()"),
VALUE_MAP("valueMap(%s)");
private final String template;
GremlinClause(String template) {
this.template = template;
}
String get() {
return template;
}
String get(String... args) {
return (args == null || args.length == 0) ? template : String.format(template, args);
}
}
repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
...@@ -18,15 +18,8 @@ ...@@ -18,15 +18,8 @@
package org.apache.atlas.query; package org.apache.atlas.query;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes;
import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
...@@ -36,13 +29,11 @@ import org.slf4j.LoggerFactory; ...@@ -36,13 +29,11 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject; import javax.inject.Inject;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.StringJoiner;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
...@@ -52,50 +43,46 @@ public class GremlinQueryComposer { ...@@ -52,50 +43,46 @@ public class GremlinQueryComposer {
private final int DEFAULT_QUERY_RESULT_LIMIT = 25; private final int DEFAULT_QUERY_RESULT_LIMIT = 25;
private final int DEFAULT_QUERY_RESULT_OFFSET = 0; private final int DEFAULT_QUERY_RESULT_OFFSET = 0;
private final List<String> errorList = new ArrayList<>(); private final List<String> errorList = new ArrayList<>();
private final GremlinClauseList queryClauses = new GremlinClauseList(); private final GremlinClauseList queryClauses = new GremlinClauseList();
private final Lookup lookup; private final Lookup lookup;
private final boolean isNestedQuery; private final boolean isNestedQuery;
private int providedLimit = DEFAULT_QUERY_RESULT_LIMIT; private final AtlasDSL.QueryMetadata queryMetadata;
private int providedOffset = DEFAULT_QUERY_RESULT_OFFSET; private int providedLimit = DEFAULT_QUERY_RESULT_LIMIT;
private boolean hasSelect = false; private int providedOffset = DEFAULT_QUERY_RESULT_OFFSET;
private boolean isSelectNoop = false; private Context context;
private boolean hasGroupBy = false;
private boolean hasOrderBy = false;
private boolean hasLimitOffset = false;
private String offset = null;
private String limit = null;
private Context context;
private SelectExprMetadata selectExprMetadata;
@Inject @Inject
public GremlinQueryComposer(AtlasTypeRegistry typeRegistry) { public GremlinQueryComposer(AtlasTypeRegistry typeRegistry, final AtlasDSL.QueryMetadata qmd) {
isNestedQuery = false; isNestedQuery = false;
lookup = new RegistryBasedLookup(errorList, typeRegistry); lookup = new RegistryBasedLookup(errorList, typeRegistry);
context = new Context(errorList, lookup); this.context = new Context(errorList, lookup);
queryMetadata = qmd;
init(); init();
} }
public GremlinQueryComposer(AtlasTypeRegistry typeRegistry, int limit, int offset) { public GremlinQueryComposer(AtlasTypeRegistry typeRegistry, final AtlasDSL.QueryMetadata queryMetadata, int limit, int offset) {
this(typeRegistry); this(typeRegistry, queryMetadata);
providedLimit = limit; providedLimit = limit;
providedOffset = offset < 0 ? DEFAULT_QUERY_RESULT_OFFSET : offset; providedOffset = offset < 0 ? DEFAULT_QUERY_RESULT_OFFSET : offset;
} }
@VisibleForTesting @VisibleForTesting
GremlinQueryComposer(Lookup lookup, Context context) { GremlinQueryComposer(Lookup lookup, Context context, final AtlasDSL.QueryMetadata qmd) {
this.isNestedQuery = false; this.isNestedQuery = false;
this.lookup = lookup; this.lookup = lookup;
this.context = context; this.context = context;
this.queryMetadata = qmd;
init(); init();
} }
public GremlinQueryComposer(Lookup registryLookup, boolean isNestedQuery) { public GremlinQueryComposer(Lookup registryLookup, final AtlasDSL.QueryMetadata qmd, boolean isNestedQuery) {
this.isNestedQuery = isNestedQuery; this.isNestedQuery = isNestedQuery;
this.lookup = registryLookup; this.lookup = registryLookup;
this.queryMetadata = qmd;
init(); init();
} }
...@@ -134,10 +121,9 @@ public class GremlinQueryComposer { ...@@ -134,10 +121,9 @@ public class GremlinQueryComposer {
addFrom(typeName); addFrom(typeName);
add(GremlinClause.HAS_PROPERTY, add(GremlinClause.HAS_PROPERTY,
IdentifierHelper.getQualifiedName(lookup, context, attribute)); IdentifierHelper.getQualifiedName(lookup, context, attribute));
} }
public void addFromIsA(String typeName, String traitName) { public void addFromIsA(String typeName, String traitName) {
addFrom(typeName); addFrom(typeName);
add(GremlinClause.TRAIT, traitName); add(GremlinClause.TRAIT, traitName);
...@@ -148,103 +134,89 @@ public class GremlinQueryComposer { ...@@ -148,103 +134,89 @@ public class GremlinQueryComposer {
LOG.debug("addWhere(lhs={}, operator={}, rhs={})", lhs, operator, rhs); LOG.debug("addWhere(lhs={}, operator={}, rhs={})", lhs, operator, rhs);
} }
String currentType = context.getActiveTypeName(); String currentType = context.getActiveTypeName();
SearchParameters.Operator op = SearchParameters.Operator.fromString(operator); SearchParameters.Operator op = SearchParameters.Operator.fromString(operator);
IdentifierHelper.Advice org = null; IdentifierHelper.Advice org = null;
IdentifierHelper.Advice lhsI = getAdvice(lhs); IdentifierHelper.Advice lhsI = getAdvice(lhs);
if(!lhsI.isPrimitive()) { if (!lhsI.isPrimitive()) {
introduceType(lhsI); introduceType(lhsI);
org = lhsI; org = lhsI;
lhsI = getAdvice(lhs); lhsI = getAdvice(lhs);
} }
if(lhsI.isDate()) { if (lhsI.isDate()) {
rhs = parseDate(rhs); rhs = parseDate(rhs);
} }
rhs = addQuotesIfNecessary(rhs); rhs = addQuotesIfNecessary(rhs);
if(op == SearchParameters.Operator.LIKE) { if (op == SearchParameters.Operator.LIKE) {
add(GremlinClause.TEXT_CONTAINS, lhsI.getQualifiedName(), rhs.replace("*", ".*").replace('?', '.')); add(GremlinClause.TEXT_CONTAINS, lhsI.getQualifiedName(), getFixedRegEx(rhs));
} else if(op == SearchParameters.Operator.IN) { } else if (op == SearchParameters.Operator.IN) {
add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), "within", rhs); add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), "within", rhs);
} else { } else {
add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), op.getSymbols()[1], rhs); add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), op.getSymbols()[1], rhs);
} }
if(org != null && !org.isPrimitive() && org.getIntroduceType()) { if (org != null && org.getIntroduceType()) {
add(GremlinClause.DEDUP); add(GremlinClause.DEDUP);
add(GremlinClause.IN, org.getEdgeLabel()); add(GremlinClause.IN, org.getEdgeLabel());
context.registerActive(currentType); context.registerActive(currentType);
} }
} }
private String getFixedRegEx(String rhs) {
return rhs.replace("*", ".*").replace('?', '.');
}
public void addAndClauses(List<String> clauses) { public void addAndClauses(List<String> clauses) {
queryClauses.add(GremlinClause.AND, StringUtils.join(clauses, ',')); queryClauses.add(GremlinClause.AND, String.join(",", clauses));
} }
public void addOrClauses(List<String> clauses) { public void addOrClauses(List<String> clauses) {
queryClauses.add(GremlinClause.OR, StringUtils.join(clauses, ',')); queryClauses.add(GremlinClause.OR, String.join(",", clauses));
}
public void addSelect(SelectClauseComposer selectClauseComposer) {
process(selectClauseComposer);
if (!(queryMetadata.hasOrderBy() && queryMetadata.hasGroupBy())) {
addSelectTransformation(selectClauseComposer);
}
this.context.setSelectClauseComposer(selectClauseComposer);
} }
public void addSelect(SelectExprMetadata selectExprMetadata) { private void process(SelectClauseComposer scc) {
String[] items = selectExprMetadata.getItems();
String[] labels = selectExprMetadata.getLabels();
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("addSelect(items.length={})", items != null ? items.length : 0); LOG.debug("addSelect(items.length={})", scc.getItems() != null ? scc.getItems().length : 0);
} }
if (items != null) { if (scc.getItems() == null) {
for (int i = 0; i < items.length; i++) { return;
IdentifierHelper.Advice ia = getAdvice(items[i]); }
if(!labels[i].equals(items[i])) { for (int i = 0; i < scc.getItems().length; i++) {
context.aliasMap.put(labels[i], ia.getQualifiedName()); IdentifierHelper.Advice ia = getAdvice(scc.getItem(i));
}
if (i == selectExprMetadata.getCountIdx()) { if (!scc.getItem(i).equals(scc.getLabel(i))) {
items[i] = GremlinClause.INLINE_COUNT.get(); context.addAlias(scc.getLabel(i), ia.getQualifiedName());
} else if (i == selectExprMetadata.getMinIdx()) {
items[i] = GremlinClause.INLINE_MIN.get(ia.getQualifiedName(), ia.getQualifiedName());
} else if (i == selectExprMetadata.getMaxIdx()) {
items[i] = GremlinClause.INLINE_MAX.get(ia.getQualifiedName(), ia.getQualifiedName());
} else if (i == selectExprMetadata.getSumIdx()) {
items[i] = GremlinClause.INLINE_SUM.get(ia.getQualifiedName(), ia.getQualifiedName());
} else {
if (!ia.isPrimitive() && ia.getIntroduceType()) {
add(GremlinClause.OUT, ia.getEdgeLabel());
context.registerActive(ia.getTypeName());
int dotIdx = ia.get().indexOf(".");
if (dotIdx != -1) {
IdentifierHelper.Advice iax = getAdvice(ia.get());
items[i] = GremlinClause.INLINE_GET_PROPERTY.get(iax.getQualifiedName());
} else {
isSelectNoop = true;
}
} else {
items[i] = GremlinClause.INLINE_GET_PROPERTY.get(ia.getQualifiedName());
}
}
} }
// If GroupBy clause exists then the query spits out a List<Map<String, List<AtlasVertex>>> otherwise the query returns List<AtlasVertex> if (scc.updateAsApplicable(i, ia.getQualifiedName())) {
// Different transformations are needed for DSLs with groupby and w/o groupby continue;
GremlinClause transformationFn;
if (isSelectNoop) {
transformationFn = GremlinClause.SELECT_EXPR_NOOP_FN;
} else {
transformationFn = hasGroupBy ? GremlinClause.SELECT_WITH_GRPBY_HELPER_FN : GremlinClause.SELECT_EXPR_HELPER_FN;
} }
queryClauses.add(0, transformationFn, getJoinedQuotedStr(labels), String.join(",", items));
queryClauses.add(GremlinClause.INLINE_TRANSFORM_CALL);
hasSelect = true; if (introduceType(ia)) {
this.selectExprMetadata = selectExprMetadata; scc.isSelectNoop = !ia.hasParts();
if(ia.hasParts()) {
scc.assign(i, getAdvice(ia.get()).getQualifiedName(), GremlinClause.INLINE_GET_PROPERTY);
}
} else {
scc.assign(i, ia.getQualifiedName(), GremlinClause.INLINE_GET_PROPERTY);
}
} }
} }
public GremlinQueryComposer createNestedProcessor() { public GremlinQueryComposer createNestedProcessor() {
GremlinQueryComposer qp = new GremlinQueryComposer(lookup, true); GremlinQueryComposer qp = new GremlinQueryComposer(lookup, queryMetadata, true);
qp.context = this.context; qp.context = this.context;
return qp; return qp;
} }
...@@ -259,12 +231,12 @@ public class GremlinQueryComposer { ...@@ -259,12 +231,12 @@ public class GremlinQueryComposer {
context.registerAlias(alias); context.registerAlias(alias);
} }
public void addAsClause(String stepName) { public void addAsClause(String alias) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("addAsClause(stepName={})", stepName); LOG.debug("addAsClause(stepName={})", alias);
} }
add(GremlinClause.AS, stepName); add(GremlinClause.AS, alias);
} }
public void addGroupBy(String item) { public void addGroupBy(String item) {
...@@ -273,7 +245,6 @@ public class GremlinQueryComposer { ...@@ -273,7 +245,6 @@ public class GremlinQueryComposer {
} }
addGroupByClause(item); addGroupByClause(item);
hasGroupBy = true;
} }
public void addLimit(String limit, String offset) { public void addLimit(String limit, String offset) {
...@@ -281,47 +252,48 @@ public class GremlinQueryComposer { ...@@ -281,47 +252,48 @@ public class GremlinQueryComposer {
LOG.debug("addLimit(limit={}, offset={})", limit, offset); LOG.debug("addLimit(limit={}, offset={})", limit, offset);
} }
this.limit = limit; if (offset.equalsIgnoreCase("0")) {
this.offset = offset; add(GremlinClause.LIMIT, limit);
} else {
addRangeClause(offset, limit);
}
}
hasLimitOffset = true; public void addDefaultLimit() {
addLimit(Integer.toString(providedLimit), Integer.toString(providedOffset));
} }
public String get() { public String get() {
close(); close();
String ret; String items[] = getFormattedClauses(queryMetadata.needTransformation());
String[] items = new String[queryClauses.size()]; return queryMetadata.needTransformation() ?
getTransformedClauses(items) :
boolean needTransformation = needTransformation(); String.join(".", items);
int startIdx = needTransformation ? 1 : 0; }
int endIdx = needTransformation ? queryClauses.size() - 1 : queryClauses.size();
for (int i = startIdx; i < endIdx; i++) {
items[i] = queryClauses.getValue(i);
}
if (needTransformation) { private String getTransformedClauses(String[] items) {
String body = String.join(".", Stream.of(items).filter(Objects::nonNull).collect(Collectors.toList())); String ret;
String inlineFn = queryClauses.getValue(queryClauses.size() - 1); String body = String.join(".", Stream.of(items).filter(Objects::nonNull).collect(Collectors.toList()));
String funCall = String.format(inlineFn, body); String inlineFn = queryClauses.getValue(queryClauses.size() - 1);
ret = queryClauses.getValue(0) + funCall; String funCall = String.format(inlineFn, body);
if (isNestedQuery) {
ret = String.join(".", queryClauses.getValue(0), funCall);
} else { } else {
ret = String.join(".", items); ret = queryClauses.getValue(0) + funCall;
}
if (LOG.isDebugEnabled()) {
LOG.debug("get() => {}", ret);
} }
return ret; return ret;
} }
public boolean hasSelect() { private String[] getFormattedClauses(boolean needTransformation) {
return hasSelect; String[] items = new String[queryClauses.size()];
} int startIdx = needTransformation ? 1 : 0;
int endIdx = needTransformation ? queryClauses.size() - 1 : queryClauses.size();
public boolean hasLimitOffset() { for (int i = startIdx; i < endIdx; i++) {
return hasLimitOffset; items[i] = queryClauses.getValue(i);
}
return items;
} }
public void addOrderBy(String name, boolean isDesc) { public void addOrderBy(String name, boolean isDesc) {
...@@ -329,34 +301,47 @@ public class GremlinQueryComposer { ...@@ -329,34 +301,47 @@ public class GremlinQueryComposer {
LOG.debug("addOrderBy(name={}, isDesc={})", name, isDesc); LOG.debug("addOrderBy(name={}, isDesc={})", name, isDesc);
} }
AtlasAttribute attribute = ((AtlasStructType) context.getActiveType()).getAttribute(getAttributeName(name)); IdentifierHelper.Advice ia = getAdvice(name);
if (hasGroupBy) { if (queryMetadata.hasSelect() && queryMetadata.hasGroupBy()) {
GremlinClause transformationFn = isDesc ? GremlinClause.GRPBY_ORDERBY_DESC_HELPER_FN : GremlinClause.GRPBY_ORDERBY_ASC_HELPER_FN; addOrderByClause(ia.getQualifiedName(), isDesc);
add(0, transformationFn, attribute.getQualifiedName(), attribute.getQualifiedName()); moveToLast(GremlinClause.GROUP_BY);
add(GremlinClause.INLINE_TRANSFORM_CALL);
addSelectTransformation(this.context.selectClauseComposer);
} else if (queryMetadata.hasGroupBy()) {
addOrderByClause(ia.getQualifiedName(), isDesc);
moveToLast(GremlinClause.GROUP_BY);
} else { } else {
addOrderByClause(attribute.getQualifiedName(), isDesc); addOrderByClause(ia.getQualifiedName(), isDesc);
} }
hasOrderBy = true;
} }
private String getAttributeName(String fqdn) { private static String quoted(String rhs) {
final String ret; return IdentifierHelper.getQuoted(rhs);
int lastSepIdx = fqdn.lastIndexOf('.');
return lastSepIdx == -1 ? fqdn : fqdn.substring(lastSepIdx + 1);
} }
private boolean needTransformation() { private void addSelectTransformation(final SelectClauseComposer selectClauseComposer) {
return (hasGroupBy && hasSelect && hasOrderBy) || (hasGroupBy && hasOrderBy) || hasSelect; GremlinClause fn;
} if (selectClauseComposer.isSelectNoop) {
fn = GremlinClause.SELECT_NOOP_FN;
} else if (queryMetadata.hasGroupBy()){
fn = selectClauseComposer.onlyAggregators() ?
GremlinClause.SELECT_ONLY_AGG_GRP_FN :
GremlinClause.SELECT_MULTI_ATTR_GRP_FN;
private static String quoted(String rhs) { } else {
return IdentifierHelper.getQuoted(rhs); fn = selectClauseComposer.onlyAggregators() ?
GremlinClause.SELECT_ONLY_AGG_FN :
GremlinClause.SELECT_FN;
}
queryClauses.add(0, fn,
selectClauseComposer.getLabelHeader(),
selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString(): "",
selectClauseComposer.getItemsString());
queryClauses.add(GremlinClause.INLINE_TRANSFORM_CALL);
} }
private String addQuotesIfNecessary(String rhs) { private String addQuotesIfNecessary(String rhs) {
if(IdentifierHelper.isTrueOrFalse(rhs)) return rhs;
if(IdentifierHelper.isQuoted(rhs)) return rhs; if(IdentifierHelper.isQuoted(rhs)) return rhs;
return quoted(rhs); return quoted(rhs);
} }
...@@ -369,20 +354,11 @@ public class GremlinQueryComposer { ...@@ -369,20 +354,11 @@ public class GremlinQueryComposer {
} }
private void close() { private void close() {
// No limits or toList() need to be added to the nested queries if (isNestedQuery)
if (isNestedQuery) return; return;
if (hasLimitOffset) { if (!queryMetadata.hasLimitOffset()) {
// If there are any aggregator functions then implicit limits shouldn't be applied addDefaultLimit();
if (selectExprMetadata == null || !selectExprMetadata.hasAggregatorFunction()) {
if (offset.equalsIgnoreCase("0")) {
add(GremlinClause.LIMIT, limit);
} else {
addRangeClause(offset, limit);
}
} else {
LOG.warn("Query has aggregator function. Performance might be slow for large dataset");
}
} }
if (queryClauses.isEmpty()) { if (queryClauses.isEmpty()) {
...@@ -390,14 +366,14 @@ public class GremlinQueryComposer { ...@@ -390,14 +366,14 @@ public class GremlinQueryComposer {
return; return;
} }
updatePosition(GremlinClause.LIMIT); moveToLast(GremlinClause.LIMIT);
add(GremlinClause.TO_LIST); add(GremlinClause.TO_LIST);
updatePosition(GremlinClause.INLINE_TRANSFORM_CALL); moveToLast(GremlinClause.INLINE_TRANSFORM_CALL);
} }
private void updatePosition(GremlinClause clause) { private void moveToLast(GremlinClause clause) {
int index = queryClauses.hasClause(clause); int index = queryClauses.hasClause(clause);
if(-1 == index) { if (-1 == index) {
return; return;
} }
...@@ -414,25 +390,19 @@ public class GremlinQueryComposer { ...@@ -414,25 +390,19 @@ public class GremlinQueryComposer {
} }
} }
private void introduceType(IdentifierHelper.Advice ia) { private boolean introduceType(IdentifierHelper.Advice ia) {
if (!ia.isPrimitive() && ia.getIntroduceType()) { if (ia.getIntroduceType()) {
add(GremlinClause.OUT, ia.getEdgeLabel()); add(GremlinClause.OUT, ia.getEdgeLabel());
context.registerActive(ia.getTypeName()); context.registerActive(ia.getTypeName());
} }
return ia.getIntroduceType();
} }
private IdentifierHelper.Advice getAdvice(String actualTypeName) { private IdentifierHelper.Advice getAdvice(String actualTypeName) {
return IdentifierHelper.create(context, lookup, actualTypeName); return IdentifierHelper.create(context, lookup, actualTypeName);
} }
private String getJoinedQuotedStr(String[] elements) {
StringJoiner joiner = new StringJoiner(",");
Arrays.stream(elements)
.map(x -> x.contains("'") ? "\"" + x + "\"" : "'" + x + "'")
.forEach(joiner::add);
return joiner.toString();
}
private void add(GremlinClause clause, String... args) { private void add(GremlinClause clause, String... args) {
queryClauses.add(new GremlinClauseValue(clause, clause.get(args))); queryClauses.add(new GremlinClauseValue(clause, clause.get(args)));
} }
...@@ -446,7 +416,7 @@ public class GremlinQueryComposer { ...@@ -446,7 +416,7 @@ public class GremlinQueryComposer {
LOG.debug("addRangeClause(startIndex={}, endIndex={})", startIndex, endIndex); LOG.debug("addRangeClause(startIndex={}, endIndex={})", startIndex, endIndex);
} }
if (hasSelect) { if (queryMetadata.hasSelect()) {
add(queryClauses.size() - 1, GremlinClause.RANGE, startIndex, startIndex, endIndex); add(queryClauses.size() - 1, GremlinClause.RANGE, startIndex, startIndex, endIndex);
} else { } else {
add(GremlinClause.RANGE, startIndex, startIndex, endIndex); add(GremlinClause.RANGE, startIndex, startIndex, endIndex);
...@@ -471,64 +441,6 @@ public class GremlinQueryComposer { ...@@ -471,64 +441,6 @@ public class GremlinQueryComposer {
add(GremlinClause.GROUP_BY, ia.getQualifiedName()); add(GremlinClause.GROUP_BY, ia.getQualifiedName());
} }
private enum GremlinClause {
AS("as('%s')"),
DEDUP("dedup()"),
G("g"),
GROUP_BY("group().by('%s')"),
HAS("has('%s', %s)"),
HAS_OPERATOR("has('%s', %s(%s))"),
HAS_PROPERTY("has('%s')"),
WHERE("where(%s)"),
HAS_NOT_PROPERTY("hasNot('%s')"),
HAS_TYPE("has('__typeName', '%s')"),
HAS_TYPE_WITHIN("has('__typeName', within(%s))"),
HAS_WITHIN("has('%s', within(%s))"),
IN("in('%s')"),
OR("or(%s)"),
AND("and(%s)"),
NESTED_START("__"),
NESTED_HAS_OPERATOR("has('%s', %s(%s))"),
LIMIT("limit(%s)"),
ORDER_BY("order().by('%s')"),
ORDER_BY_DESC("order().by('%s', decr)"),
OUT("out('%s')"),
RANGE("range(%s, %s + %s)"),
SELECT("select('%s')"),
TO_LIST("toList()"),
TEXT_CONTAINS("has('%s', org.janusgraph.core.attribute.Text.textRegex(%s))"),
TEXT_PREFIX("has('%s', org.janusgraph.core.attribute.Text.textPrefix(%s))"),
TEXT_SUFFIX("has('%s', org.janusgraph.core.attribute.Text.textRegex(\".*\" + %s))"),
TRAIT("has('__traitNames', within('%s'))"),
SELECT_EXPR_NOOP_FN("def f(r){ r }; "),
SELECT_EXPR_HELPER_FN("def f(r){ return [[%s]].plus(r.collect({[%s]})).unique(); }; "),
SELECT_WITH_GRPBY_HELPER_FN("def f(r){ return [[%s]].plus(r.collect({it.values()}).flatten().collect({[%s]})).unique(); }; "),
GRPBY_ORDERBY_ASC_HELPER_FN("def f(r){ m=r.get(0); m.each({ k,v -> m[k] = v.sort{a,b -> a.value('%s') <=> b.value('%s')}}); r }; "),
GRPBY_ORDERBY_DESC_HELPER_FN("def f(r){ m=r.get(0); m.each({ k,v -> m[k] = v.sort{a,b -> b.value('%s') <=> a.value('%s')}}); r; }; "),
INLINE_COUNT("r.size()"),
INLINE_SUM("r.sum({it.value('%s')}).value('%s')"),
INLINE_MAX("r.max({it.value('%s')}).value('%s')"),
INLINE_MIN("r.min({it.value('%s')}).value('%s')"),
INLINE_GET_PROPERTY("it.value('%s')"),
INLINE_OUT_VERTEX("it.out('%s')"),
INLINE_OUT_VERTEX_VALUE("it.out('%s').value('%s')"), // This might require more closure introduction :(
INLINE_TRANSFORM_CALL("f(%s)"),
V("V()"),
VALUE_MAP("valueMap(%s)");
private final String format;
GremlinClause(String format) {
this.format = format;
}
String get(String... args) {
return (args == null || args.length == 0) ?
format :
String.format(format, args);
}
}
private static class GremlinClauseValue { private static class GremlinClauseValue {
private final GremlinClause clause; private final GremlinClause clause;
private final String value; private final String value;
...@@ -604,7 +516,14 @@ public class GremlinQueryComposer { ...@@ -604,7 +516,14 @@ public class GremlinQueryComposer {
} }
public boolean isEmpty() { public boolean isEmpty() {
return list.size() == 0; return list.size() == 0 || containsGVLimit();
}
private boolean containsGVLimit() {
return list.size() == 3 &&
list.get(0).clause == GremlinClause.G &&
list.get(1).clause == GremlinClause.V &&
list.get(2).clause == GremlinClause.LIMIT;
} }
public void clear() { public void clear() {
...@@ -624,6 +543,7 @@ public class GremlinQueryComposer { ...@@ -624,6 +543,7 @@ public class GremlinQueryComposer {
Lookup lookup; Lookup lookup;
Map<String, String> aliasMap = new HashMap<>(); Map<String, String> aliasMap = new HashMap<>();
private AtlasType activeType; private AtlasType activeType;
private SelectClauseComposer selectClauseComposer;
public Context(List<String> errorList, Lookup lookup) { public Context(List<String> errorList, Lookup lookup) {
this.lookup = lookup; this.lookup = lookup;
...@@ -659,12 +579,7 @@ public class GremlinQueryComposer { ...@@ -659,12 +579,7 @@ public class GremlinQueryComposer {
} }
public void registerAlias(String alias) { public void registerAlias(String alias) {
if(aliasMap.containsKey(alias)) { addAlias(alias, getActiveTypeName());
errorList.add(String.format("Duplicate alias found: %s for type %s already present.", alias, getActiveEntityType()));
return;
}
aliasMap.put(alias, getActiveTypeName());
} }
public boolean hasAlias(String alias) { public boolean hasAlias(String alias) {
...@@ -678,210 +593,18 @@ public class GremlinQueryComposer { ...@@ -678,210 +593,18 @@ public class GremlinQueryComposer {
public boolean isEmpty() { public boolean isEmpty() {
return activeType == null; return activeType == null;
} }
}
private static class RegistryBasedLookup implements Lookup {
private final List<String> errorList;
private final AtlasTypeRegistry typeRegistry;
public RegistryBasedLookup(List<String> errorList, AtlasTypeRegistry typeRegistry) {
this.errorList = errorList;
this.typeRegistry = typeRegistry;
}
@Override
public AtlasType getType(String typeName) {
try {
return typeRegistry.getType(typeName);
} catch (AtlasBaseException e) {
addError(e.getMessage());
}
return null; public void setSelectClauseComposer(SelectClauseComposer selectClauseComposer) {
this.selectClauseComposer = selectClauseComposer;
} }
@Override public void addAlias(String alias, String typeName) {
public String getQualifiedName(Context context, String name) { if(aliasMap.containsKey(alias)) {
try { errorList.add(String.format("Duplicate alias found: %s for type %s already present.", alias, getActiveEntityType()));
AtlasEntityType et = context.getActiveEntityType(); return;
if(et == null) {
return "";
}
return et.getQualifiedAttributeName(name);
} catch (AtlasBaseException e) {
addError(e.getMessage());
}
return "";
}
@Override
public boolean isPrimitive(Context context, String attributeName) {
AtlasEntityType et = context.getActiveEntityType();
if(et == null) {
return false;
}
AtlasType attr = et.getAttributeType(attributeName);
if(attr == null) {
return false;
}
TypeCategory attrTypeCategory = attr.getTypeCategory();
return (attrTypeCategory != null) && (attrTypeCategory == TypeCategory.PRIMITIVE || attrTypeCategory == TypeCategory.ENUM);
}
@Override
public String getRelationshipEdgeLabel(Context context, String attributeName) {
AtlasEntityType et = context.getActiveEntityType();
if(et == null) {
return "";
}
AtlasAttribute attr = et.getAttribute(attributeName);
return (attr != null) ? attr.getRelationshipEdgeLabel() : "";
}
@Override
public boolean hasAttribute(Context context, String typeName) {
return (context.getActiveEntityType() != null) && context.getActiveEntityType().getAttribute(typeName) != null;
}
@Override
public boolean doesTypeHaveSubTypes(Context context) {
return (context.getActiveEntityType() != null && context.getActiveEntityType().getAllSubTypes().size() > 0);
}
@Override
public String getTypeAndSubTypes(Context context) {
String[] str = context.getActiveEntityType() != null ?
context.getActiveEntityType().getTypeAndAllSubTypes().toArray(new String[]{}) :
new String[]{};
if(str.length == 0) {
return null;
}
String[] quoted = new String[str.length];
for (int i = 0; i < str.length; i++) {
quoted[i] = quoted(str[i]);
}
return StringUtils.join(quoted, ",");
}
@Override
public boolean isTraitType(Context context) {
return (context.getActiveType() != null &&
context.getActiveType().getTypeCategory() == TypeCategory.CLASSIFICATION);
}
@Override
public String getTypeFromEdge(Context context, String item) {
AtlasEntityType et = context.getActiveEntityType();
if(et == null) {
return "";
}
AtlasAttribute attr = et.getAttribute(item);
if(attr == null) {
return null;
}
AtlasType at = attr.getAttributeType();
if(at.getTypeCategory() == TypeCategory.ARRAY) {
AtlasArrayType arrType = ((AtlasArrayType)at);
return ((AtlasBuiltInTypes.AtlasObjectIdType) arrType.getElementType()).getObjectType();
}
return context.getActiveEntityType().getAttribute(item).getTypeName();
}
@Override
public boolean isDate(Context context, String attributeName) {
AtlasEntityType et = context.getActiveEntityType();
if (et == null) {
return false;
} }
AtlasType attr = et.getAttributeType(attributeName); aliasMap.put(alias, typeName);
return attr != null && attr.getTypeName().equals(AtlasBaseTypeDef.ATLAS_TYPE_DATE);
}
protected void addError(String s) {
errorList.add(s);
}
}
static class SelectExprMetadata {
private String[] items;
private String[] labels;
private int countIdx = -1;
private int sumIdx = -1;
private int maxIdx = -1;
private int minIdx = -1;
private boolean hasAggregator = false;
public String[] getItems() {
return items;
}
public void setItems(final String[] items) {
this.items = items;
}
public int getCountIdx() {
return countIdx;
}
public void setCountIdx(final int countIdx) {
this.countIdx = countIdx;
setHasAggregator();
}
public int getSumIdx() {
return sumIdx;
}
public void setSumIdx(final int sumIdx) {
this.sumIdx = sumIdx;
setHasAggregator();
}
public int getMaxIdx() {
return maxIdx;
}
public void setMaxIdx(final int maxIdx) {
this.maxIdx = maxIdx;
setHasAggregator();
}
public int getMinIdx() {
return minIdx;
}
public void setMinIdx(final int minIdx) {
this.minIdx = minIdx;
setHasAggregator();
}
public String[] getLabels() {
return labels;
}
public void setLabels(final String[] labels) {
this.labels = labels;
}
public boolean hasAggregatorFunction(){
return hasAggregator;
}
private void setHasAggregator() {
hasAggregator = true;
} }
} }
} }
repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
...@@ -89,6 +89,10 @@ public class IdentifierHelper { ...@@ -89,6 +89,10 @@ public class IdentifierHelper {
return String.format("'%s'", s); return String.format("'%s'", s);
} }
public static boolean isTrueOrFalse(String rhs) {
return rhs.equalsIgnoreCase("true") || rhs.equalsIgnoreCase("false");
}
public static class Advice { public static class Advice {
private String raw; private String raw;
private String actual; private String actual;
...@@ -123,7 +127,7 @@ public class IdentifierHelper { ...@@ -123,7 +127,7 @@ public class IdentifierHelper {
updateTypeInfo(lookup, context); updateTypeInfo(lookup, context);
isTrait = lookup.isTraitType(context); isTrait = lookup.isTraitType(context);
updateEdgeInfo(lookup, context); updateEdgeInfo(lookup, context);
introduceType = !context.hasAlias(parts[0]); introduceType = !isPrimitive() && !context.hasAlias(parts[0]);
updateSubTypes(lookup, context); updateSubTypes(lookup, context);
} }
} }
...@@ -237,12 +241,12 @@ public class IdentifierHelper { ...@@ -237,12 +241,12 @@ public class IdentifierHelper {
return actual; return actual;
} }
public boolean isNewContext() {
return newContext;
}
public boolean isDate() { public boolean isDate() {
return isDate; return isDate;
} }
}
public boolean hasParts() {
return parts.length > 1;
}
}
} }
repository/src/main/java/org/apache/atlas/query/QueryParams.java
...@@ -18,6 +18,8 @@ ...@@ -18,6 +18,8 @@
package org.apache.atlas.query; package org.apache.atlas.query;
import org.apache.atlas.AtlasConfiguration;
public class QueryParams { public class QueryParams {
private int limit; private int limit;
private int offset; private int offset;
...@@ -47,4 +49,21 @@ public class QueryParams { ...@@ -47,4 +49,21 @@ public class QueryParams {
public void offset(int offset) { public void offset(int offset) {
this.offset = offset; this.offset = offset;
} }
public static QueryParams getNormalizedParams(int suppliedLimit, int suppliedOffset) {
int defaultLimit = AtlasConfiguration.SEARCH_DEFAULT_LIMIT.getInt();
int maxLimit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt();
int limit = defaultLimit;
if (suppliedLimit > 0 && suppliedLimit <= maxLimit) {
limit = suppliedLimit;
}
int offset = 0;
if (suppliedOffset > 0) {
offset = suppliedOffset;
}
return new QueryParams(limit, offset);
}
} }
repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.query;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.type.*;
import org.apache.commons.lang.StringUtils;
import java.util.List;
class RegistryBasedLookup implements Lookup {
private final List<String> errorList;
private final AtlasTypeRegistry typeRegistry;
public RegistryBasedLookup(List<String> errorList, AtlasTypeRegistry typeRegistry) {
this.errorList = errorList;
this.typeRegistry = typeRegistry;
}
@Override
public AtlasType getType(String typeName) {
try {
return typeRegistry.getType(typeName);
} catch (AtlasBaseException e) {
addError(e.getMessage());
}
return null;
}
@Override
public String getQualifiedName(GremlinQueryComposer.Context context, String name) {
try {
AtlasEntityType et = context.getActiveEntityType();
if(et == null) {
return "";
}
return et.getQualifiedAttributeName(name);
} catch (AtlasBaseException e) {
addError(e.getMessage());
}
return "";
}
@Override
public boolean isPrimitive(GremlinQueryComposer.Context context, String attributeName) {
AtlasEntityType et = context.getActiveEntityType();
if(et == null) {
return false;
}
AtlasType attr = et.getAttributeType(attributeName);
if(attr == null) {
return false;
}
TypeCategory attrTypeCategory = attr.getTypeCategory();
return (attrTypeCategory != null) && (attrTypeCategory == TypeCategory.PRIMITIVE || attrTypeCategory == TypeCategory.ENUM);
}
@Override
public String getRelationshipEdgeLabel(GremlinQueryComposer.Context context, String attributeName) {
AtlasEntityType et = context.getActiveEntityType();
if(et == null) {
return "";
}
AtlasStructType.AtlasAttribute attr = et.getAttribute(attributeName);
return (attr != null) ? attr.getRelationshipEdgeLabel() : "";
}
@Override
public boolean hasAttribute(GremlinQueryComposer.Context context, String typeName) {
return (context.getActiveEntityType() != null) && context.getActiveEntityType().getAttribute(typeName) != null;
}
@Override
public boolean doesTypeHaveSubTypes(GremlinQueryComposer.Context context) {
return (context.getActiveEntityType() != null && context.getActiveEntityType().getAllSubTypes().size() > 0);
}
@Override
public String getTypeAndSubTypes(GremlinQueryComposer.Context context) {
String[] str = context.getActiveEntityType() != null ?
context.getActiveEntityType().getTypeAndAllSubTypes().toArray(new String[]{}) :
new String[]{};
if(str.length == 0) {
return null;
}
String[] quoted = new String[str.length];
for (int i = 0; i < str.length; i++) {
quoted[i] = IdentifierHelper.getQuoted(str[i]);
}
return StringUtils.join(quoted, ",");
}
@Override
public boolean isTraitType(GremlinQueryComposer.Context context) {
return (context.getActiveType() != null &&
context.getActiveType().getTypeCategory() == TypeCategory.CLASSIFICATION);
}
@Override
public String getTypeFromEdge(GremlinQueryComposer.Context context, String item) {
AtlasEntityType et = context.getActiveEntityType();
if(et == null) {
return "";
}
AtlasStructType.AtlasAttribute attr = et.getAttribute(item);
if(attr == null) {
return null;
}
AtlasType at = attr.getAttributeType();
if(at.getTypeCategory() == TypeCategory.ARRAY) {
AtlasArrayType arrType = ((AtlasArrayType)at);
return ((AtlasBuiltInTypes.AtlasObjectIdType) arrType.getElementType()).getObjectType();
}
return context.getActiveEntityType().getAttribute(item).getTypeName();
}
@Override
public boolean isDate(GremlinQueryComposer.Context context, String attributeName) {
AtlasEntityType et = context.getActiveEntityType();
if (et == null) {
return false;
}
AtlasType attr = et.getAttributeType(attributeName);
return attr != null && attr.getTypeName().equals(AtlasBaseTypeDef.ATLAS_TYPE_DATE);
}
protected void addError(String s) {
errorList.add(s);
}
}
repository/src/main/java/org/apache/atlas/query/SelectClauseComposer.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.query;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.StringJoiner;
class SelectClauseComposer {
private String[] labels;
private String[] items;
private Map<String, String> itemAssignmentExprs;
private int countIdx = -1;
private int sumIdx = -1;
private int maxIdx = -1;
private int minIdx = -1;
private int aggCount = 0;
public boolean isSelectNoop;
public SelectClauseComposer() {}
public String[] getItems() {
return items;
}
public void setItems(final String[] items) {
this.items = items;
}
public boolean updateAsApplicable(int currentIndex, String qualifiedName) {
boolean ret = false;
if (currentIndex == getCountIdx()) {
ret = assign(currentIndex, "count",
GremlinClause.INLINE_COUNT.get(), GremlinClause.INLINE_ASSIGNMENT);
} else if (currentIndex == getMinIdx()) {
ret = assign(currentIndex, "min", qualifiedName,
GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_MIN);
} else if (currentIndex == getMaxIdx()) {
ret = assign(currentIndex, "max", qualifiedName,
GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_MAX);
} else if (currentIndex == getSumIdx()) {
ret = assign(currentIndex, "sum", qualifiedName,
GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_SUM);
}
return ret;
}
private boolean assign(String item, String assignExpr) {
if (itemAssignmentExprs == null) {
itemAssignmentExprs = new LinkedHashMap<>();
}
itemAssignmentExprs.put(item, assignExpr);
return true;
}
public boolean assign(int i, String qualifiedName, GremlinClause clause) {
items[i] = clause.get(qualifiedName);
return true;
}
private boolean assign(int i, String s, String p1, GremlinClause clause) {
items[i] = s;
return assign(items[i], clause.get(s, p1));
}
private boolean assign(int i, String s, String p1, GremlinClause inline, GremlinClause clause) {
items[i] = s;
return assign(items[i], inline.get(s, clause.get(p1, p1)));
}
private int getCountIdx() {
return countIdx;
}
public void setCountIdx(final int countIdx) {
this.countIdx = countIdx;
aggCount++;
}
private int getSumIdx() {
return sumIdx;
}
public void setSumIdx(final int sumIdx) {
this.sumIdx = sumIdx;
aggCount++;
}
private int getMaxIdx() {
return maxIdx;
}
public void setMaxIdx(final int maxIdx) {
this.maxIdx = maxIdx;
aggCount++;
}
private int getMinIdx() {
return minIdx;
}
public void setMinIdx(final int minIdx) {
this.minIdx = minIdx;
aggCount++;
}
public String[] getLabels() {
return labels;
}
public void setLabels(final String[] labels) {
this.labels = labels;
}
public boolean hasAssignmentExpr() {
return itemAssignmentExprs != null && !itemAssignmentExprs.isEmpty();
}
public boolean onlyAggregators() {
return aggCount > 0 && aggCount == items.length;
}
public String getLabelHeader() {
return getJoinedQuotedStr(getLabels());
}
public String getItemsString() {
return String.join(",", getItems());
}
public String getAssignmentExprString(){
return String.join(" ", itemAssignmentExprs.values());
}
private String getJoinedQuotedStr(String[] elements) {
StringJoiner joiner = new StringJoiner(",");
Arrays.stream(elements)
.map(x -> x.contains("'") ? "\"" + x + "\"" : "'" + x + "'")
.forEach(joiner::add);
return joiner.toString();
}
public String getItem(int i) {
return items[i];
}
public String getLabel(int i) {
return labels[i];
}
}
repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLLexer.java
// Generated from repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLLexer.g4 by ANTLR 4.7
package org.apache.atlas.query.antlr4; package org.apache.atlas.query.antlr4;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RuntimeMetaData; import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Vocabulary; import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.VocabularyImpl; import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.atn.ATNDeserializer; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.atn.LexerATNSimulator;
import org.antlr.v4.runtime.atn.PredictionContextCache;
import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.misc.*;
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
public class AtlasDSLLexer extends Lexer { public class AtlasDSLLexer extends Lexer {
......
repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParser.g4
...@@ -27,9 +27,9 @@ operator: (K_LT | K_LTE | K_EQ | K_NEQ | K_GT | K_GTE | K_LIKE) ; ...@@ -27,9 +27,9 @@ operator: (K_LT | K_LTE | K_EQ | K_NEQ | K_GT | K_GTE | K_LIKE) ;
sortOrder: K_ASC | K_DESC ; sortOrder: K_ASC | K_DESC ;
valueArray: K_LBRACKET STRING (K_COMMA STRING)* K_RBRACKET ; valueArray: K_LBRACKET ID (K_COMMA ID)* K_RBRACKET ;
literal: BOOL | NUMBER | FLOATING_NUMBER | (STRING | valueArray) ; literal: BOOL | NUMBER | FLOATING_NUMBER | (ID | valueArray) ;
// Composite rules // Composite rules
limitClause: K_LIMIT NUMBER ; limitClause: K_LIMIT NUMBER ;
......
repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParser.java
// Generated from repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParser.g4 by ANTLR 4.7
package org.apache.atlas.query.antlr4; package org.apache.atlas.query.antlr4;
import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.atn.*;
import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.dfa.DFA;
...@@ -239,9 +238,9 @@ public class AtlasDSLParser extends Parser { ...@@ -239,9 +238,9 @@ public class AtlasDSLParser extends Parser {
public static class ValueArrayContext extends ParserRuleContext { public static class ValueArrayContext extends ParserRuleContext {
public TerminalNode K_LBRACKET() { return getToken(AtlasDSLParser.K_LBRACKET, 0); } public TerminalNode K_LBRACKET() { return getToken(AtlasDSLParser.K_LBRACKET, 0); }
public List<TerminalNode> STRING() { return getTokens(AtlasDSLParser.STRING); } public List<TerminalNode> ID() { return getTokens(AtlasDSLParser.ID); }
public TerminalNode STRING(int i) { public TerminalNode ID(int i) {
return getToken(AtlasDSLParser.STRING, i); return getToken(AtlasDSLParser.ID, i);
} }
public TerminalNode K_RBRACKET() { return getToken(AtlasDSLParser.K_RBRACKET, 0); } public TerminalNode K_RBRACKET() { return getToken(AtlasDSLParser.K_RBRACKET, 0); }
public List<TerminalNode> K_COMMA() { return getTokens(AtlasDSLParser.K_COMMA); } public List<TerminalNode> K_COMMA() { return getTokens(AtlasDSLParser.K_COMMA); }
...@@ -269,7 +268,7 @@ public class AtlasDSLParser extends Parser { ...@@ -269,7 +268,7 @@ public class AtlasDSLParser extends Parser {
setState(82); setState(82);
match(K_LBRACKET); match(K_LBRACKET);
setState(83); setState(83);
match(STRING); match(ID);
setState(88); setState(88);
_errHandler.sync(this); _errHandler.sync(this);
_la = _input.LA(1); _la = _input.LA(1);
...@@ -279,7 +278,7 @@ public class AtlasDSLParser extends Parser { ...@@ -279,7 +278,7 @@ public class AtlasDSLParser extends Parser {
setState(84); setState(84);
match(K_COMMA); match(K_COMMA);
setState(85); setState(85);
match(STRING); match(ID);
} }
} }
setState(90); setState(90);
...@@ -305,7 +304,7 @@ public class AtlasDSLParser extends Parser { ...@@ -305,7 +304,7 @@ public class AtlasDSLParser extends Parser {
public TerminalNode BOOL() { return getToken(AtlasDSLParser.BOOL, 0); } public TerminalNode BOOL() { return getToken(AtlasDSLParser.BOOL, 0); }
public TerminalNode NUMBER() { return getToken(AtlasDSLParser.NUMBER, 0); } public TerminalNode NUMBER() { return getToken(AtlasDSLParser.NUMBER, 0); }
public TerminalNode FLOATING_NUMBER() { return getToken(AtlasDSLParser.FLOATING_NUMBER, 0); } public TerminalNode FLOATING_NUMBER() { return getToken(AtlasDSLParser.FLOATING_NUMBER, 0); }
public TerminalNode STRING() { return getToken(AtlasDSLParser.STRING, 0); } public TerminalNode ID() { return getToken(AtlasDSLParser.ID, 0); }
public ValueArrayContext valueArray() { public ValueArrayContext valueArray() {
return getRuleContext(ValueArrayContext.class,0); return getRuleContext(ValueArrayContext.class,0);
} }
...@@ -349,16 +348,16 @@ public class AtlasDSLParser extends Parser { ...@@ -349,16 +348,16 @@ public class AtlasDSLParser extends Parser {
} }
break; break;
case K_LBRACKET: case K_LBRACKET:
case STRING: case ID:
enterOuterAlt(_localctx, 4); enterOuterAlt(_localctx, 4);
{ {
setState(98); setState(98);
_errHandler.sync(this); _errHandler.sync(this);
switch (_input.LA(1)) { switch (_input.LA(1)) {
case STRING: case ID:
{ {
setState(96); setState(96);
match(STRING); match(ID);
} }
break; break;
case K_LBRACKET: case K_LBRACKET:
...@@ -496,30 +495,23 @@ public class AtlasDSLParser extends Parser { ...@@ -496,30 +495,23 @@ public class AtlasDSLParser extends Parser {
case BOOL: case BOOL:
case K_LBRACKET: case K_LBRACKET:
case ID: case ID:
case STRING:
enterOuterAlt(_localctx, 1); enterOuterAlt(_localctx, 1);
{ {
setState(110); setState(110);
_errHandler.sync(this); _errHandler.sync(this);
switch (_input.LA(1)) { switch ( getInterpreter().adaptivePredict(_input,3,_ctx) ) {
case ID: case 1:
{ {
setState(108); setState(108);
identifier(); identifier();
} }
break; break;
case NUMBER: case 2:
case FLOATING_NUMBER:
case BOOL:
case K_LBRACKET:
case STRING:
{ {
setState(109); setState(109);
literal(); literal();
} }
break; break;
default:
throw new NoViableAltException(this);
} }
} }
break; break;
...@@ -1491,25 +1483,19 @@ public class AtlasDSLParser extends Parser { ...@@ -1491,25 +1483,19 @@ public class AtlasDSLParser extends Parser {
{ {
setState(208); setState(208);
_errHandler.sync(this); _errHandler.sync(this);
switch (_input.LA(1)) { switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
case ID: case 1:
{ {
setState(206); setState(206);
identifier(); identifier();
} }
break; break;
case NUMBER: case 2:
case FLOATING_NUMBER:
case BOOL:
case K_LBRACKET:
case STRING:
{ {
setState(207); setState(207);
literal(); literal();
} }
break; break;
default:
throw new NoViableAltException(this);
} }
setState(210); setState(210);
match(K_AS); match(K_AS);
...@@ -1621,25 +1607,19 @@ public class AtlasDSLParser extends Parser { ...@@ -1621,25 +1607,19 @@ public class AtlasDSLParser extends Parser {
{ {
setState(221); setState(221);
_errHandler.sync(this); _errHandler.sync(this);
switch (_input.LA(1)) { switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
case ID: case 1:
{ {
setState(219); setState(219);
identifier(); identifier();
} }
break; break;
case NUMBER: case 2:
case FLOATING_NUMBER:
case BOOL:
case K_LBRACKET:
case STRING:
{ {
setState(220); setState(220);
literal(); literal();
} }
break; break;
default:
throw new NoViableAltException(this);
} }
} }
break; break;
...@@ -2029,7 +2009,7 @@ public class AtlasDSLParser extends Parser { ...@@ -2029,7 +2009,7 @@ public class AtlasDSLParser extends Parser {
setState(261); setState(261);
_errHandler.sync(this); _errHandler.sync(this);
_la = _input.LA(1); _la = _input.LA(1);
while ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << NUMBER) | (1L << FLOATING_NUMBER) | (1L << BOOL) | (1L << K_LPAREN) | (1L << K_LBRACKET) | (1L << K_FROM) | (1L << K_WHERE) | (1L << K_MAX) | (1L << K_MIN) | (1L << K_SUM) | (1L << K_COUNT) | (1L << ID) | (1L << STRING))) != 0)) { while ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << NUMBER) | (1L << FLOATING_NUMBER) | (1L << BOOL) | (1L << K_LPAREN) | (1L << K_LBRACKET) | (1L << K_FROM) | (1L << K_WHERE) | (1L << K_MAX) | (1L << K_MIN) | (1L << K_SUM) | (1L << K_COUNT) | (1L << ID))) != 0)) {
{ {
{ {
setState(258); setState(258);
...@@ -2226,10 +2206,10 @@ public class AtlasDSLParser extends Parser { ...@@ -2226,10 +2206,10 @@ public class AtlasDSLParser extends Parser {
"\2\64\u00d2\3\2\2\2\66\u00d7\3\2\2\28\u00e1\3\2\2\2:\u00e3\3\2\2\2<\u00e6"+ "\2\64\u00d2\3\2\2\2\66\u00d7\3\2\2\28\u00e1\3\2\2\2:\u00e3\3\2\2\2<\u00e6"+
"\3\2\2\2>\u00ea\3\2\2\2@\u00ed\3\2\2\2B\u00f4\3\2\2\2D\u00f6\3\2\2\2F"+ "\3\2\2\2>\u00ea\3\2\2\2@\u00ed\3\2\2\2B\u00f4\3\2\2\2D\u00f6\3\2\2\2F"+
"\u00fb\3\2\2\2H\u0103\3\2\2\2J\u010c\3\2\2\2L\u010e\3\2\2\2NO\7\62\2\2"+ "\u00fb\3\2\2\2H\u0103\3\2\2\2J\u010c\3\2\2\2L\u010e\3\2\2\2NO\7\62\2\2"+
"O\3\3\2\2\2PQ\t\2\2\2Q\5\3\2\2\2RS\t\3\2\2S\7\3\2\2\2TU\7\23\2\2UZ\7\63"+ "O\3\3\2\2\2PQ\t\2\2\2Q\5\3\2\2\2RS\t\3\2\2S\7\3\2\2\2TU\7\23\2\2UZ\7\62"+
"\2\2VW\7\t\2\2WY\7\63\2\2XV\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2[]\3"+ "\2\2VW\7\t\2\2WY\7\62\2\2XV\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2[]\3"+
"\2\2\2\\Z\3\2\2\2]^\7\25\2\2^\t\3\2\2\2_g\7\b\2\2`g\7\6\2\2ag\7\7\2\2"+ "\2\2\2\\Z\3\2\2\2]^\7\25\2\2^\t\3\2\2\2_g\7\b\2\2`g\7\6\2\2ag\7\7\2\2"+
"be\7\63\2\2ce\5\b\5\2db\3\2\2\2dc\3\2\2\2eg\3\2\2\2f_\3\2\2\2f`\3\2\2"+ "be\7\62\2\2ce\5\b\5\2db\3\2\2\2dc\3\2\2\2eg\3\2\2\2f_\3\2\2\2f`\3\2\2"+
"\2fa\3\2\2\2fd\3\2\2\2g\13\3\2\2\2hi\7 \2\2ij\7\6\2\2j\r\3\2\2\2kl\7\'"+ "\2fa\3\2\2\2fd\3\2\2\2g\13\3\2\2\2hi\7 \2\2ij\7\6\2\2j\r\3\2\2\2kl\7\'"+
"\2\2lm\7\6\2\2m\17\3\2\2\2nq\5\2\2\2oq\5\n\6\2pn\3\2\2\2po\3\2\2\2qw\3"+ "\2\2lm\7\6\2\2m\17\3\2\2\2nq\5\2\2\2oq\5\n\6\2pn\3\2\2\2po\3\2\2\2qw\3"+
"\2\2\2rs\7\22\2\2st\5,\27\2tu\7\24\2\2uw\3\2\2\2vp\3\2\2\2vr\3\2\2\2w"+ "\2\2\2rs\7\22\2\2st\5,\27\2tu\7\24\2\2uw\3\2\2\2vp\3\2\2\2vr\3\2\2\2w"+
......
repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParserBaseVisitor.java
// Generated from repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParser.g4 by ANTLR 4.7
package org.apache.atlas.query.antlr4; package org.apache.atlas.query.antlr4;
import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor; import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
......
repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParserVisitor.java
// Generated from repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParser.g4 by ANTLR 4.7
package org.apache.atlas.query.antlr4; package org.apache.atlas.query.antlr4;
import org.antlr.v4.runtime.tree.ParseTreeVisitor; import org.antlr.v4.runtime.tree.ParseTreeVisitor;
......
repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
...@@ -17,12 +17,18 @@ ...@@ -17,12 +17,18 @@
*/ */
package org.apache.atlas.query; package org.apache.atlas.query;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.TokenStream;
import org.apache.atlas.TestModules; import org.apache.atlas.TestModules;
import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.discovery.EntityDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.query.antlr4.AtlasDSLLexer;
import org.apache.atlas.query.antlr4.AtlasDSLParser;
import org.apache.atlas.runner.LocalSolrRunner; import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider; import org.testng.annotations.DataProvider;
...@@ -31,7 +37,22 @@ import org.testng.annotations.Test; ...@@ -31,7 +37,22 @@ import org.testng.annotations.Test;
import javax.inject.Inject; import javax.inject.Inject;
import static org.testng.Assert.*; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class) @Guice(modules = TestModules.TestOnlyModule.class)
public class DSLQueriesTest extends BasicTestSetup { public class DSLQueriesTest extends BasicTestSetup {
...@@ -257,6 +278,11 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -257,6 +278,11 @@ public class DSLQueriesTest extends BasicTestSetup {
{"hive_table where name='sales_fact', db where name='Reporting'", 0}, {"hive_table where name='sales_fact', db where name='Reporting'", 0},
{"hive_table where name='sales_fact', db where name='Reporting' limit 10", 0}, {"hive_table where name='sales_fact', db where name='Reporting' limit 10", 0},
{"hive_table where name='sales_fact', db where name='Reporting' limit 10 offset 1", 0}, {"hive_table where name='sales_fact', db where name='Reporting' limit 10 offset 1", 0},
{"hive_db as d where owner = ['John ETL', 'Jane BI']", 2},
{"hive_db as d where owner = ['John ETL', 'Jane BI'] limit 10", 2},
{"hive_db as d where owner = ['John ETL', 'Jane BI'] limit 10 offset 1", 1},
}; };
} }
...@@ -360,12 +386,389 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -360,12 +386,389 @@ public class DSLQueriesTest extends BasicTestSetup {
}; };
} }
@DataProvider(name = "minMaxCountProvider")
private Object[][] minMaxCountQueries() {
return new Object[][]{
{"from hive_db groupby (owner) select count() ",
new FieldValueValidator()
.withFieldNames("count()")
.withExpectedValues(1)
.withExpectedValues(1)
.withExpectedValues(1) },
// FIXME
// { "from hive_db groupby (owner, name) select Asset.owner, Asset.name, count()",
// new FieldValueValidator()
// .withFieldNames("Asset.owner", "Asset.name", "count()")
// .withExpectedValues("Jane BI", "Reporting", 1)
// .withExpectedValues("Tim ETL", "Logging", 1)
// .withExpectedValues("John ETL", "Sales", 1) },
{ "from hive_db groupby (owner) select count() ",
new FieldValueValidator()
.withFieldNames("count()").
withExpectedValues(1).
withExpectedValues(1).
withExpectedValues(1) },
{ "from hive_db groupby (owner) select Asset.owner, count() ",
new FieldValueValidator()
.withFieldNames("Asset.owner", "count()")
.withExpectedValues("Jane BI", 1)
.withExpectedValues("Tim ETL", 1)
.withExpectedValues("John ETL", 1) },
{ "from hive_db groupby (owner) select count() ",
new FieldValueValidator()
.withFieldNames("count()")
.withExpectedValues(1)
.withExpectedValues(1)
.withExpectedValues(1) },
{ "from hive_db groupby (owner) select Asset.owner, count() ",
new FieldValueValidator()
.withFieldNames("Asset.owner", "count()")
.withExpectedValues("Jane BI", 1)
.withExpectedValues("Tim ETL", 1)
.withExpectedValues("John ETL", 1) },
{ "from hive_db groupby (owner) select Asset.owner, max(Asset.name) ",
new FieldValueValidator()
.withFieldNames("Asset.owner", "max(Asset.name)")
.withExpectedValues("Tim ETL", "Logging")
.withExpectedValues("Jane BI", "Reporting")
.withExpectedValues("John ETL", "Sales") },
{ "from hive_db groupby (owner) select max(Asset.name) ",
new FieldValueValidator()
.withFieldNames("max(Asset.name)")
.withExpectedValues("Logging")
.withExpectedValues("Reporting")
.withExpectedValues("Sales") },
{ "from hive_db groupby (owner) select owner, Asset.name, min(Asset.name) ",
new FieldValueValidator()
.withFieldNames("owner", "Asset.name", "min(Asset.name)")
.withExpectedValues("Tim ETL", "Logging", "Logging")
.withExpectedValues("Jane BI", "Reporting", "Reporting")
.withExpectedValues("John ETL", "Sales", "Sales") },
{ "from hive_db groupby (owner) select owner, min(Asset.name) ",
new FieldValueValidator()
.withFieldNames("owner", "min(Asset.name)")
.withExpectedValues("Tim ETL", "Logging")
.withExpectedValues("Jane BI", "Reporting")
.withExpectedValues("John ETL", "Sales") },
{ "from hive_db groupby (owner) select min(name) ",
new FieldValueValidator()
.withFieldNames("min(name)")
.withExpectedValues("Reporting")
.withExpectedValues("Logging")
.withExpectedValues("Sales") },
{ "from hive_db groupby (owner) select min('name') ",
new FieldValueValidator()
.withFieldNames("min('name')")
.withExpectedValues("name")
.withExpectedValues("name")
.withExpectedValues("name") },
{ "from hive_db select count() ",
new FieldValueValidator()
.withFieldNames("count()")
.withExpectedValues(3) },
{ "from Person select count() as 'count', max(Person.age) as 'max', min(Person.age) as 'min'",
new FieldValueValidator()
.withFieldNames("'count'", "'max'", "'min'")
.withExpectedValues(50, 0, 4) },
{ "from Person select count() as 'count', sum(Person.age) as 'sum'",
new FieldValueValidator()
.withFieldNames("'count'", "'sum'")
.withExpectedValues(4, 86) },
// tests to ensure that group by works with order by and limit
// FIXME:
// { "from hive_db groupby (owner) select min(name) orderby name limit 2 ",
// new FieldValueValidator()
// .withFieldNames("min(name)")
// .withExpectedValues("Logging")
// .withExpectedValues("Reporting") },
// { "from hive_db groupby (owner) select min(name) orderby name desc limit 2 ",
// new FieldValueValidator()
// .withFieldNames("min(name)")
// .withExpectedValues("Reporting")
// .withExpectedValues("Sales") }
};
}
@Test(dataProvider = "minMaxCountProvider")
public void minMaxCount(String query, FieldValueValidator fv) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
assertSearchResult(searchResult, fv);
}
@Test(dataProvider = "likeQueriesProvider") @Test(dataProvider = "likeQueriesProvider")
public void likeQueries(String query, int expected) throws AtlasBaseException { public void likeQueries(String query, int expected) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0); AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
assertSearchResult(searchResult, expected); assertSearchResult(searchResult, expected);
} }
@Test
public void classification() {
String expected = "g.V().has('__traitNames', within('PII')).limit(25).toList()";
verify("PII", expected);
}
@Test
public void dimension() {
String expected = "g.V().has('__typeName', 'hive_table').has('__traitNames', within('Dimension')).limit(25).toList()";
verify("hive_table isa Dimension", expected);
verify("hive_table is Dimension", expected);
verify("hive_table where hive_table is Dimension", expected);
// Not supported since it requires two singleSrcQuery, one for isa clause other for where clause
// verify("Table isa Dimension where name = 'sales'",
// "g.V().has('__typeName', 'Table').has('__traitNames', within('Dimension')).has('Table.name', eq('sales')).limit(25).toList()");
}
@Test
public void fromDB() {
verify("from hive_db", "g.V().has('__typeName', 'hive_db').limit(25).toList()");
verify("from hive_db limit 10", "g.V().has('__typeName', 'hive_db').limit(10).toList()");
verify("hive_db limit 10", "g.V().has('__typeName', 'hive_db').limit(10).toList()");
}
@Test
public void hasName() {
String expected = "g.V().has('__typeName', within('DataSet','hive_column_lineage','Infrastructure','Asset','Process','hive_table','hive_column','hive_db','hive_process')).has('Asset.name').limit(25).toList()";
verify("Asset has name", expected);
verify("Asset where Asset has name", expected);
}
@Test
public void simpleAlias() {
verify("Asset as a", "g.V().has('__typeName', within('DataSet','hive_column_lineage','Infrastructure','Asset','Process','hive_table','hive_column','hive_db','hive_process')).as('a').limit(25).toList()");
}
@Test
public void selectQueries() {
String expected = "def f(r){ t=[['d.name','d.owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); }; " +
"f(g.V().has('__typeName', within('DataSet','hive_column_lineage','Infrastructure','Asset','Process','hive_table','hive_column','hive_db','hive_process')).as('d')";
verify("Asset as d select d.name, d.owner", expected + ".limit(25).toList())");
verify("Asset as d select d.name, d.owner limit 10", expected + ".limit(10).toList())");
}
@Test
public void tableSelectColumns() {
String exMain = "g.V().has('__typeName', 'hive_table').out('__hive_table.columns').limit(10).toList()";
String exSel = "def f(r){ r };";
String exSel1 = "def f(r){ t=[['db.name']]; r.each({t.add([it.value('Asset.name')])}); t.unique(); };";
verify("hive_table select columns limit 10", getExpected(exSel, exMain));
String exMain2 = "g.V().has('__typeName', 'hive_table').out('__hive_table.db').limit(25).toList()";
verify("hive_table select db", getExpected(exSel, exMain2));
String exMain3 = "g.V().has('__typeName', 'hive_table').out('__hive_table.db').limit(25).toList()";
verify("hive_table select db.name", getExpected(exSel1, exMain3));
}
@Test(enabled = false)
public void SelectLimit() {
verify("from hive_db limit 5", "g.V().has('__typeName', 'hive_db').limit(5).toList()");
verify("from hive_db limit 5 offset 2", "g.V().has('__typeName', 'hive_db').range(2, 7).toList()");
}
@Test
public void orderBy() {
String expected = "g.V().has('__typeName', 'hive_db').order().by('Asset.name').limit(25).toList()";
verify("hive_db orderby name", expected);
verify("from hive_db orderby name", expected);
verify("from hive_db as d orderby d.owner limit 3", "g.V().has('__typeName', 'hive_db').as('d').order().by('Asset.owner').limit(3).toList()");
verify("hive_db as d orderby d.owner limit 3", "g.V().has('__typeName', 'hive_db').as('d').order().by('Asset.owner').limit(3).toList()");
String exSel = "def f(r){ t=[['d.name','d.owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); };";
String exMain = "g.V().has('__typeName', 'hive_db').as('d').order().by('Asset.owner').limit(25).toList()";
verify("hive_db as d select d.name, d.owner orderby (d.owner) limit 25", getExpected(exSel, exMain));
String exMain2 = "g.V().has('__typeName', 'hive_table').and(__.has('Asset.name', eq(\"sales_fact\")),__.has('hive_table.createTime', gt('1388563200000'))).order().by('hive_table.createTime').limit(25).toList()";
String exSel2 = "def f(r){ t=[['_col_0','_col_1']]; r.each({t.add([it.value('Asset.name'),it.value('hive_table.createTime')])}); t.unique(); };";
verify("hive_table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 orderby _col_1",
getExpected(exSel2, exMain2));
}
@Test
public void fromDBOrderByNameDesc() {
verify("from hive_db orderby name DESC", "g.V().has('__typeName', 'hive_db').order().by('Asset.name', decr).limit(25).toList()");
}
@Test
public void fromDBSelect() {
String expected = "def f(r){ t=[['Asset.name','Asset.owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); };" +
" f(g.V().has('__typeName', 'hive_db').limit(25).toList())";
verify("from hive_db select Asset.name, Asset.owner", expected);
expected = "def f(r){ t=[['min(name)','max(owner)']]; " +
"def min=r.min({it.value('Asset.name')}).value('Asset.name'); " +
"def max=r.max({it.value('Asset.owner')}).value('Asset.owner'); " +
"t.add([min,max]); t;}; " +
"f(g.V().has('__typeName', 'hive_db').limit(25).toList())";
verify("hive_db select min(name), max(owner)", expected);
expected = "def f(r){ t=[['owner','min(name)','max(owner)']]; " +
"def min=r.min({it.value('Asset.name')}).value('Asset.name'); " +
"def max=r.max({it.value('Asset.owner')}).value('Asset.owner'); " +
"r.each({t.add([it.value('Asset.owner'),min,max])}); t.unique(); }; " +
"f(g.V().has('__typeName', 'hive_db').limit(25).toList())";
verify("hive_db select owner, min(name), max(owner)", expected);
}
@Test
public void fromDBGroupBy() {
verify("from hive_db groupby (Asset.owner)", "g.V().has('__typeName', 'hive_db').group().by('Asset.owner').limit(25).toList()");
}
@Test
public void whereClauseTextContains() {
String exMain = "g.V().has('__typeName', 'hive_db').has('Asset.name', eq(\"Reporting\")).limit(25).toList()";
String exSel = "def f(r){ t=[['name','owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); };";
verify("from hive_db where name = \"Reporting\" select name, owner", getExpected(exSel, exMain));
verify("from hive_db where (name = \"Reporting\") select name, owner", getExpected(exSel, exMain));
verify("hive_table where Asset.name like \"Tab*\"",
"g.V().has('__typeName', 'hive_table').has('Asset.name', org.janusgraph.core.attribute.Text.textRegex(\"Tab.*\")).limit(25).toList()");
verify("from hive_table where (db.name = \"Reporting\")",
"g.V().has('__typeName', 'hive_table').out('__hive_table.db').has('Asset.name', eq(\"Reporting\")).dedup().in('__hive_table.db').limit(25).toList()");
}
@Test
public void whereClauseWithAsTextContains() {
String exSel = "def f(r){ t=[['t.name','t.owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); };";
String exMain = "g.V().has('__typeName', 'hive_table').as('t').has('Asset.name', eq(\"testtable_1\")).limit(25).toList()";
verify("hive_table as t where t.name = \"testtable_1\" select t.name, t.owner)", getExpected(exSel, exMain));
}
@Test
public void whereClauseWithDateCompare() {
String exSel = "def f(r){ t=[['t.name','t.owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); };";
String exMain = "g.V().has('__typeName', 'hive_table').as('t').has('hive_table.createTime', eq('1513046158440')).limit(25).toList()";
verify("hive_table as t where t.createTime = \"2017-12-12T02:35:58.440Z\" select t.name, t.owner)", getExpected(exSel, exMain));
}
@Test
public void subType() {
String exMain = "g.V().has('__typeName', within('DataSet','hive_column_lineage','Infrastructure','Asset','Process','hive_table','hive_column','hive_db','hive_process')).limit(25).toList()";
String exSel = "def f(r){ t=[['name','owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); };";
verify("Asset select name, owner", getExpected(exSel, exMain));
}
@Test
public void TraitWithSpace() {
verify("`Log Data`", "g.V().has('__traitNames', within('Log Data')).limit(25).toList()");
}
@Test
public void nestedQueries() {
verify("hive_table where name=\"sales_fact\" or name=\"testtable_1\"",
"g.V().has('__typeName', 'hive_table').or(__.has('Asset.name', eq(\"sales_fact\")),__.has('Asset.name', eq(\"testtable_1\"))).limit(25).toList()");
verify("hive_table where name=\"sales_fact\" and name=\"testtable_1\"",
"g.V().has('__typeName', 'hive_table').and(__.has('Asset.name', eq(\"sales_fact\")),__.has('Asset.name', eq(\"testtable_1\"))).limit(25).toList()");
verify("hive_table where name=\"sales_fact\" or name=\"testtable_1\" or name=\"testtable_2\"",
"g.V().has('__typeName', 'hive_table')" +
".or(" +
"__.has('Asset.name', eq(\"sales_fact\"))," +
"__.has('Asset.name', eq(\"testtable_1\"))," +
"__.has('Asset.name', eq(\"testtable_2\"))" +
").limit(25).toList()");
verify("hive_table where name=\"sales_fact\" and name=\"testtable_1\" and name=\"testtable_2\"",
"g.V().has('__typeName', 'hive_table')" +
".and(" +
"__.has('Asset.name', eq(\"sales_fact\"))," +
"__.has('Asset.name', eq(\"testtable_1\"))," +
"__.has('Asset.name', eq(\"testtable_2\"))" +
").limit(25).toList()");
verify("hive_table where (name=\"sales_fact\" or name=\"testtable_1\") and name=\"testtable_2\"",
"g.V().has('__typeName', 'hive_table')" +
".and(" +
"__.or(" +
"__.has('Asset.name', eq(\"sales_fact\"))," +
"__.has('Asset.name', eq(\"testtable_1\"))" +
")," +
"__.has('Asset.name', eq(\"testtable_2\")))" +
".limit(25).toList()");
verify("hive_table where name=\"sales_fact\" or (name=\"testtable_1\" and name=\"testtable_2\")",
"g.V().has('__typeName', 'hive_table')" +
".or(" +
"__.has('Asset.name', eq(\"sales_fact\"))," +
"__.and(" +
"__.has('Asset.name', eq(\"testtable_1\"))," +
"__.has('Asset.name', eq(\"testtable_2\")))" +
")" +
".limit(25).toList()");
verify("hive_table where name=\"sales_fact\" or name=\"testtable_1\" and name=\"testtable_2\"",
"g.V().has('__typeName', 'hive_table')" +
".and(" +
"__.or(" +
"__.has('Asset.name', eq(\"sales_fact\"))," +
"__.has('Asset.name', eq(\"testtable_1\"))" +
")," +
"__.has('Asset.name', eq(\"testtable_2\")))" +
".limit(25).toList()");
verify("hive_table where (name=\"sales_fact\" and owner=\"Joe\") OR (name=\"sales_fact_daily_mv\" and owner=\"Joe BI\")",
"g.V().has('__typeName', 'hive_table')" +
".or(" +
"__.and(" +
"__.has('Asset.name', eq(\"sales_fact\"))," +
"__.has('Asset.owner', eq(\"Joe\"))" +
")," +
"__.and(" +
"__.has('Asset.name', eq(\"sales_fact_daily_mv\"))," +
"__.has('Asset.owner', eq(\"Joe BI\"))" +
"))" +
".limit(25).toList()");
verify("hive_table where owner=\"hdfs\" or ((name=\"testtable_1\" or name=\"testtable_2\") and createTime < \"2017-12-12T02:35:58.440Z\")",
"g.V().has('__typeName', 'hive_table').or(__.has('Asset.owner', eq(\"hdfs\")),__.and(__.or(__.has('Asset.name', eq(\"testtable_1\")),__.has('Asset.name', eq(\"testtable_2\"))),__.has('hive_table.createTime', lt('1513046158440')))).limit(25).toList()");
verify("hive_table where hive_table.name='Reporting' and hive_table.createTime < '2017-12-12T02:35:58.440Z'",
"g.V().has('__typeName', 'hive_table').and(__.has('Asset.name', eq('Reporting')),__.has('hive_table.createTime', lt('1513046158440'))).limit(25).toList()");
verify("hive_table where db.name='Sales' and db.clusterName='cl1'",
"g.V().has('__typeName', 'hive_table').and(__.out('__hive_table.db').has('Asset.name', eq('Sales')).dedup().in('__hive_table.db'),__.out('__hive_table.db').has('hive_db.clusterName', eq('cl1')).dedup().in('__hive_table.db')).limit(25).toList()");
}
private void verify(String dsl, String expectedGremlin) {
AtlasDSLParser.QueryContext queryContext = getParsedQuery(dsl);
String actualGremlin = getGremlinQuery(queryContext);
assertEquals(actualGremlin, expectedGremlin);
}
private String getExpected(String select, String main) {
return String.format("%s f(%s)", select, main);
}
private AtlasDSLParser.QueryContext getParsedQuery(String query) {
AtlasDSLParser.QueryContext queryContext = null;
InputStream stream = new ByteArrayInputStream(query.getBytes());
AtlasDSLLexer lexer = null;
try {
lexer = new AtlasDSLLexer(CharStreams.fromStream(stream));
} catch (IOException e) {
assertTrue(false);
}
TokenStream inputTokenStream = new CommonTokenStream(lexer);
AtlasDSLParser parser = new AtlasDSLParser(inputTokenStream);
queryContext = parser.query();
assertNotNull(queryContext);
assertNull(queryContext.exception);
return queryContext;
}
private String getGremlinQuery(AtlasDSLParser.QueryContext queryContext) {
GremlinQueryComposer gremlinQueryComposer = new GremlinQueryComposer(typeRegistry, new AtlasDSL.QueryMetadata(queryContext));
DSLVisitor qv = new DSLVisitor(gremlinQueryComposer);
qv.visit(queryContext);
String s = gremlinQueryComposer.get();
assertTrue(StringUtils.isNotEmpty(s));
return s;
}
private void assertSearchResult(AtlasSearchResult searchResult, int expected) { private void assertSearchResult(AtlasSearchResult searchResult, int expected) {
assertNotNull(searchResult); assertNotNull(searchResult);
if(expected == 0) { if(expected == 0) {
...@@ -379,4 +782,49 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -379,4 +782,49 @@ public class DSLQueriesTest extends BasicTestSetup {
assertEquals(searchResult.getAttributes().getValues().size(), expected); assertEquals(searchResult.getAttributes().getValues().size(), expected);
} }
} }
private void assertSearchResult(AtlasSearchResult searchResult, FieldValueValidator expected) {
assertNotNull(searchResult);
assertNull(searchResult.getEntities());
assertEquals(searchResult.getAttributes().getName().size(), expected.getFieldNamesCount());
for (int i = 0; i < searchResult.getAttributes().getName().size(); i++) {
String s = searchResult.getAttributes().getName().get(i);
assertEquals(s, expected.fieldNames[i]);
}
assertEquals(searchResult.getAttributes().getValues().size(), expected.values.size());
}
private class FieldValueValidator {
class ResultObject {
Map<String, Object> fieldValues = new HashMap<>();
public void setFieldValue(String string, Object object) {
fieldValues.put(string, object);
}
}
private String[] fieldNames;
private List<ResultObject> values = new ArrayList<>();
public FieldValueValidator withFieldNames(String... fieldNames) {
this.fieldNames = fieldNames;
return this;
}
public FieldValueValidator withExpectedValues(Object... values) {
ResultObject obj = new ResultObject();
for (int i = 0; i < fieldNames.length; i++) {
obj.setFieldValue(fieldNames[i], values[i]);
}
this.values.add(obj);
return this;
}
public int getFieldNamesCount() {
return (fieldNames != null) ? fieldNames.length : 0;
}
}
} }
repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
...@@ -56,9 +56,6 @@ public class GremlinQueryComposerTest { ...@@ -56,9 +56,6 @@ public class GremlinQueryComposerTest {
verify("Table isa Dimension", expected); verify("Table isa Dimension", expected);
verify("Table is Dimension", expected); verify("Table is Dimension", expected);
verify("Table where Table is Dimension", expected); verify("Table where Table is Dimension", expected);
// Not supported since it requires two singleSrcQuery, one for isa clause other for where clause
// verify("Table isa Dimension where name = 'sales'",
// "g.V().has('__typeName', 'Table').has('__traitNames', within('Dimension')).has('Table.name', eq('sales')).limit(25).toList()");
} }
@Test @Test
...@@ -82,7 +79,8 @@ public class GremlinQueryComposerTest { ...@@ -82,7 +79,8 @@ public class GremlinQueryComposerTest {
@Test @Test
public void DBasDSelect() { public void DBasDSelect() {
String expected = "def f(r){ return [['d.name','d.owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }; f(g.V().has('__typeName', 'DB').as('d')"; String expected = "def f(r){ t=[['d.name','d.owner']]; r.each({t.add([it.value('DB.name'),it.value('DB.owner')])}); t.unique(); }; " +
"f(g.V().has('__typeName', 'DB').as('d')";
verify("DB as d select d.name, d.owner", expected + ".limit(25).toList())"); verify("DB as d select d.name, d.owner", expected + ".limit(25).toList())");
verify("DB as d select d.name, d.owner limit 10", expected + ".limit(10).toList())"); verify("DB as d select d.name, d.owner limit 10", expected + ".limit(10).toList())");
} }
...@@ -91,7 +89,7 @@ public class GremlinQueryComposerTest { ...@@ -91,7 +89,7 @@ public class GremlinQueryComposerTest {
public void tableSelectColumns() { public void tableSelectColumns() {
String exMain = "g.V().has('__typeName', 'Table').out('__Table.columns').limit(10).toList()"; String exMain = "g.V().has('__typeName', 'Table').out('__Table.columns').limit(10).toList()";
String exSel = "def f(r){ r }"; String exSel = "def f(r){ r }";
String exSel1 = "def f(r){ return [['db.name']].plus(r.collect({[it.value('DB.name')]})).unique(); }"; String exSel1 = "def f(r){ t=[['db.name']]; r.each({t.add([it.value('DB.name')])}); t.unique(); }";
verify("Table select columns limit 10", getExpected(exSel, exMain)); verify("Table select columns limit 10", getExpected(exSel, exMain));
String exMain2 = "g.V().has('__typeName', 'Table').out('__Table.db').limit(25).toList()"; String exMain2 = "g.V().has('__typeName', 'Table').out('__Table.db').limit(25).toList()";
...@@ -102,15 +100,28 @@ public class GremlinQueryComposerTest { ...@@ -102,15 +100,28 @@ public class GremlinQueryComposerTest {
} }
@Test(enabled = false) @Test
public void DBTableFrom() { public void valueArray() {
verify("Table, db", "g.V().has('__typeName', 'Table').out('__DB.Table').limit(25).toList()"); verify("DB where owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'DB').has('DB.owner', within('hdfs','anon')).limit(25).toList()");
}
@Test
public void groupByMin() {
verify("from DB groupby (owner) select min(name) orderby name limit 2",
"def f(l){ t=[['min(name)']]; l.get(0).each({k,r -> L:{ def min=r.min({it.value('DB.name')}).value('DB.name'); t.add([min]); } }); t; }; " +
"f(g.V().has('__typeName', 'DB').order().by('DB.name').group().by('DB.owner').limit(2).toList())");
}
@Test
public void groupByOrderBy() {
verify("Table groupby(owner) select name, owner, clusterName orderby name",
"def f(l){ t=[['name','owner','clusterName']]; l.get(0).each({k,r -> L:{ r.each({t.add([it.value('Table.name'),it.value('Table.owner'),it.value('Table.clusterName')])}) } }); t.unique(); }; " +
"f(g.V().has('__typeName', 'Table').order().by('Table.name').group().by('Table.owner').limit(25).toList())");
} }
@Test @Test
public void DBAsDSelectLimit() { public void DBAsDSelectLimit() {
verify("from DB limit 5", "g.V().has('__typeName', 'DB').limit(5).toList()"); verify("from DB limit 5", "g.V().has('__typeName', 'DB').limit(5).toList()");
verify("from DB limit 5 offset 2", "g.V().has('__typeName', 'DB').range(2, 2 + 5).limit(25).toList()"); verify("from DB limit 5 offset 2", "g.V().has('__typeName', 'DB').range(2, 2 + 5).toList()");
} }
@Test @Test
...@@ -122,12 +133,12 @@ public class GremlinQueryComposerTest { ...@@ -122,12 +133,12 @@ public class GremlinQueryComposerTest {
verify("DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()"); verify("DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()");
String exSel = "def f(r){ return [['d.name','d.owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }"; String exSel = "def f(r){ t=[['d.name','d.owner']]; r.each({t.add([it.value('DB.name'),it.value('DB.owner')])}); t.unique(); }";
String exMain = "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(25).toList()"; String exMain = "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(25).toList()";
verify("DB as d select d.name, d.owner orderby (d.owner) limit 25", getExpected(exSel, exMain)); verify("DB as d select d.name, d.owner orderby (d.owner) limit 25", getExpected(exSel, exMain));
String exMain2 = "g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.createTime', gt('1388563200000'))).order().by('Table.createTime').limit(25).toList()"; String exMain2 = "g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.createTime', gt('1388563200000'))).order().by('Table.createTime').limit(25).toList()";
String exSel2 = "def f(r){ return [['_col_0','_col_1']].plus(r.collect({[it.value('Table.name'),it.value('Table.createTime')]})).unique(); }"; String exSel2 = "def f(r){ t=[['_col_0','_col_1']]; r.each({t.add([it.value('Table.name'),it.value('Table.createTime')])}); t.unique(); }";
verify("Table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 orderby _col_1", verify("Table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 orderby _col_1",
getExpected(exSel2, exMain2)); getExpected(exSel2, exMain2));
} }
...@@ -139,7 +150,7 @@ public class GremlinQueryComposerTest { ...@@ -139,7 +150,7 @@ public class GremlinQueryComposerTest {
@Test @Test
public void fromDBSelect() { public void fromDBSelect() {
String expected = "def f(r){ return [['DB.name','DB.owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }; f(g.V().has('__typeName', 'DB').limit(25).toList())"; String expected = "def f(r){ t=[['DB.name','DB.owner']]; r.each({t.add([it.value('DB.name'),it.value('DB.owner')])}); t.unique(); }; f(g.V().has('__typeName', 'DB').limit(25).toList())";
verify("from DB select DB.name, DB.owner", expected); verify("from DB select DB.name, DB.owner", expected);
} }
...@@ -151,7 +162,7 @@ public class GremlinQueryComposerTest { ...@@ -151,7 +162,7 @@ public class GremlinQueryComposerTest {
@Test @Test
public void whereClauseTextContains() { public void whereClauseTextContains() {
String exMain = "g.V().has('__typeName', 'DB').has('DB.name', eq(\"Reporting\")).limit(25).toList()"; String exMain = "g.V().has('__typeName', 'DB').has('DB.name', eq(\"Reporting\")).limit(25).toList()";
String exSel = "def f(r){ return [['name','owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }"; String exSel = "def f(r){ t=[['name','owner']]; r.each({t.add([it.value('DB.name'),it.value('DB.owner')])}); t.unique(); }";
verify("from DB where name = \"Reporting\" select name, owner", getExpected(exSel, exMain)); verify("from DB where name = \"Reporting\" select name, owner", getExpected(exSel, exMain));
verify("from DB where (name = \"Reporting\") select name, owner", getExpected(exSel, exMain)); verify("from DB where (name = \"Reporting\") select name, owner", getExpected(exSel, exMain));
verify("Table where Asset.name like \"Tab*\"", verify("Table where Asset.name like \"Tab*\"",
...@@ -162,14 +173,14 @@ public class GremlinQueryComposerTest { ...@@ -162,14 +173,14 @@ public class GremlinQueryComposerTest {
@Test @Test
public void whereClauseWithAsTextContains() { public void whereClauseWithAsTextContains() {
String exSel = "def f(r){ return [['t.name','t.owner']].plus(r.collect({[it.value('Table.name'),it.value('Table.owner')]})).unique(); }"; String exSel = "def f(r){ t=[['t.name','t.owner']]; r.each({t.add([it.value('Table.name'),it.value('Table.owner')])}); t.unique(); }";
String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.name', eq(\"testtable_1\")).limit(25).toList()"; String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.name', eq(\"testtable_1\")).limit(25).toList()";
verify("Table as t where t.name = \"testtable_1\" select t.name, t.owner)", getExpected(exSel, exMain)); verify("Table as t where t.name = \"testtable_1\" select t.name, t.owner)", getExpected(exSel, exMain));
} }
@Test @Test
public void whereClauseWithDateCompare() { public void whereClauseWithDateCompare() {
String exSel = "def f(r){ return [['t.name','t.owner']].plus(r.collect({[it.value('Table.name'),it.value('Table.owner')]})).unique(); }"; String exSel = "def f(r){ t=[['t.name','t.owner']]; r.each({t.add([it.value('Table.name'),it.value('Table.owner')])}); t.unique(); }";
String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.createdTime', eq('1513046158440')).limit(25).toList()"; String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.createdTime', eq('1513046158440')).limit(25).toList()";
verify("Table as t where t.createdTime = \"2017-12-12T02:35:58.440Z\" select t.name, t.owner)", getExpected(exSel, exMain)); verify("Table as t where t.createdTime = \"2017-12-12T02:35:58.440Z\" select t.name, t.owner)", getExpected(exSel, exMain));
} }
...@@ -177,17 +188,39 @@ public class GremlinQueryComposerTest { ...@@ -177,17 +188,39 @@ public class GremlinQueryComposerTest {
@Test @Test
public void subType() { public void subType() {
String exMain = "g.V().has('__typeName', within('Asset','Table')).limit(25).toList()"; String exMain = "g.V().has('__typeName', within('Asset','Table')).limit(25).toList()";
String exSel = "def f(r){ return [['name','owner']].plus(r.collect({[it.value('Asset.name'),it.value('Asset.owner')]})).unique(); }"; String exSel = "def f(r){ t=[['name','owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); }";
verify("Asset select name, owner", getExpected(exSel, exMain)); verify("Asset select name, owner", getExpected(exSel, exMain));
} }
@Test @Test
public void TraitWithSpace() { public void countMinMax() {
verify("from DB groupby (owner) select count()",
"def f(l){ t=[['count()']]; l.get(0).each({k,r -> L:{ def count=r.size(); t.add([count]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList())");
verify("from DB groupby (owner) select max(name)",
"def f(l){ t=[['max(name)']]; l.get(0).each({k,r -> L:{ def max=r.max({it.value('DB.name')}).value('DB.name'); t.add([max]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList())");
verify("from DB groupby (owner) select min(name)",
"def f(l){ t=[['min(name)']]; l.get(0).each({k,r -> L:{ def min=r.min({it.value('DB.name')}).value('DB.name'); t.add([min]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList())");
verify("from Table select sum(createTime)",
"def f(r){ t=[['sum(createTime)']]; def sum=r.sum({it.value('Table.createTime')}); t.add([sum]); t;}; f(g.V().has('__typeName', 'Table').limit(25).toList())");
}
@Test
public void traitWithSpace() {
verify("`Log Data`", "g.V().has('__typeName', 'Log Data').limit(25).toList()"); verify("`Log Data`", "g.V().has('__typeName', 'Log Data').limit(25).toList()");
} }
@Test @Test
public void whereClauseWithBooleanCondition() {
String queryFormat = "Table as t where name ='Reporting' or t.isFile = %s";
String expectedFormat = "g.V().has('__typeName', 'Table').as('t').or(__.has('Table.name', eq('Reporting')),__.has('Table.isFile', eq(%s))).limit(25).toList()";
verify(String.format(queryFormat, "true"), String.format(expectedFormat, "true"));
verify(String.format(queryFormat, "false"), String.format(expectedFormat, "false"));
verify(String.format(queryFormat, "True"), String.format(expectedFormat, "True"));
verify(String.format(queryFormat, "FALSE"), String.format(expectedFormat, "FALSE"));
}
@Test
public void nestedQueries() { public void nestedQueries() {
verify("Table where name=\"sales_fact\" or name=\"testtable_1\"", verify("Table where name=\"sales_fact\" or name=\"testtable_1\"",
"g.V().has('__typeName', 'Table').or(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(25).toList()"); "g.V().has('__typeName', 'Table').or(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(25).toList()");
...@@ -254,6 +287,11 @@ public class GremlinQueryComposerTest { ...@@ -254,6 +287,11 @@ public class GremlinQueryComposerTest {
"g.V().has('__typeName', 'Table').and(__.out('__Table.db').has('DB.name', eq('Sales')).dedup().in('__Table.db'),__.out('__Table.db').has('DB.clusterName', eq('cl1')).dedup().in('__Table.db')).limit(25).toList()"); "g.V().has('__typeName', 'Table').and(__.out('__Table.db').has('DB.name', eq('Sales')).dedup().in('__Table.db'),__.out('__Table.db').has('DB.clusterName', eq('cl1')).dedup().in('__Table.db')).limit(25).toList()");
} }
@Test
public void invalidQueries() {
verify("hdfs_path like h1", "");
}
private void verify(String dsl, String expectedGremlin) { private void verify(String dsl, String expectedGremlin) {
AtlasDSLParser.QueryContext queryContext = getParsedQuery(dsl); AtlasDSLParser.QueryContext queryContext = getParsedQuery(dsl);
String actualGremlin = getGremlinQuery(queryContext); String actualGremlin = getGremlinQuery(queryContext);
...@@ -289,13 +327,13 @@ public class GremlinQueryComposerTest { ...@@ -289,13 +327,13 @@ public class GremlinQueryComposerTest {
AtlasTypeRegistry registry = mock(AtlasTypeRegistry.class); AtlasTypeRegistry registry = mock(AtlasTypeRegistry.class);
org.apache.atlas.query.Lookup lookup = new TestLookup(errorList, registry); org.apache.atlas.query.Lookup lookup = new TestLookup(errorList, registry);
GremlinQueryComposer.Context context = new GremlinQueryComposer.Context(errorList, lookup); GremlinQueryComposer.Context context = new GremlinQueryComposer.Context(errorList, lookup);
AtlasDSL.QueryMetadata queryMetadata = new AtlasDSL.QueryMetadata(queryContext);
GremlinQueryComposer gremlinQueryComposer = new GremlinQueryComposer(lookup, context); GremlinQueryComposer gremlinQueryComposer = new GremlinQueryComposer(lookup, context, queryMetadata);
DSLVisitor qv = new DSLVisitor(gremlinQueryComposer); DSLVisitor qv = new DSLVisitor(gremlinQueryComposer);
qv.visit(queryContext); qv.visit(queryContext);
String s = gremlinQueryComposer.get(); String s = gremlinQueryComposer.get();
assertTrue(StringUtils.isNotEmpty(s));
return s; return s;
} }
......
repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import com.google.inject.Inject;
import org.apache.atlas.TestModules;
import org.apache.atlas.discovery.EntityDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.junit.Assert;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@Guice(modules = TestModules.TestOnlyModule.class)
public class EntityDiscoveryServiceTest {
@Inject
AtlasTypeRegistry typeRegistry;
@Inject
private AtlasTypeDefStore typeDefStore;
@Inject
private AtlasGraph atlasGraph;
@Inject
EntityDiscoveryService entityDiscoveryService;
@Test
public void dslTest() throws AtlasBaseException {
//String dslQuery = "DB where name = \"Reporting\"";
String dslQuery = "hive_table where Asset.name = \"testtable_x_0\"";
AtlasSearchResult result = entityDiscoveryService.searchUsingDslQuery(dslQuery, 20 , 0);
Assert.assertNotNull(result);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment