Commit c35f82ca by Ashutosh Mestry Committed by apoorvnaik

ATLAS-2419: DSL Semantic Validation. Fix for limit & range.

parent 1fee4a5b
......@@ -103,7 +103,23 @@ public enum AtlasErrorCode {
SAVED_SEARCH_CHANGE_USER(400, "ATLAS-400-00-056", "saved-search {0} can not be moved from user {1} to {2}"),
INVALID_QUERY_PARAM_LENGTH(400, "ATLAS-400-00-057" , "Length of query param {0} exceeds the limit"),
INVALID_QUERY_LENGTH(400, "ATLAS-400-00-058" , "Invalid query length, update {0} to change the limit" ),
// DSL related error codes
INVALID_DSL_QUERY(400, "ATLAS-400-00-059" , "Invalid DSL query: {0} | Reason: {1}. Please refer to Atlas DSL grammar for more information" ),
INVALID_DSL_GROUPBY(400, "ATLAS-400-00-05A", "DSL Semantic Error - GroupBy attribute {0} is non-primitive"),
INVALID_DSL_UNKNOWN_TYPE(400, "ATLAS-400-00-05B", "DSL Semantic Error - {0} type not found"),
INVALID_DSL_UNKNOWN_CLASSIFICATION(400, "ATLAS-400-00-05C", "DSL Semantic Error - {0} classification not found"),
INVALID_DSL_UNKNOWN_ATTR_TYPE(400, "ATLAS-400-00-05D", "DSL Semantic Error - {0} attribute not found for type {1}"),
INVALID_DSL_ORDERBY(400, "ATLAS-400-00-05E", "DSL Semantic Error - OrderBy attribute {0} is non-primitive"),
INVALID_DSL_FROM(400, "ATLAS-400-00-05F", "DSL Semantic Error - From source {0} is not a valid Entity/Classification type"),
INVALID_DSL_SELECT_REFERRED_ATTR(400, "ATLAS-400-00-060", "DSL Semantic Error - Select clause has multiple referred attributes {0}"),
INVALID_DSL_SELECT_INVALID_AGG(400, "ATLAS-400-00-061", "DSL Semantic Error - Select clause has aggregation on referred attributes {0}"),
INVALID_DSL_SELECT_ATTR_MIXING(400, "ATLAS-400-00-062", "DSL Semantic Error - Select clause has simple and referred attributes"),
INVALID_DSL_HAS_ATTRIBUTE(400, "ATLAS-400-00-063", "DSL Semantic Error - No attribute {0} exists for type {1}"),
INVALID_DSL_QUALIFIED_NAME(400, "ATLAS-400-00-064", "DSL Semantic Error - Qualified name for {0} failed!"),
INVALID_DSL_QUALIFIED_NAME2(400, "ATLAS-400-00-065", "DSL Semantic Error - Qualified name for {0} failed for type {1}. Cause: {2}"),
INVALID_DSL_DUPLICATE_ALIAS(400, "ATLAS-400-00-066", "DSL Semantic Error - Duplicate alias found: '{0}' for type '{1}' already present."),
INVALID_DSL_INVALID_DATE(400, "ATLAS-400-00-067", "DSL Semantic Error - Date format: {0}."),
INVALID_DSL_HAS_PROPERTY(400, "ATLAS-400-00-068", "DSL Semantic Error - Property needs to be a primitive type: {0}"),
// All Not found enums go here
TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"),
......
......@@ -104,6 +104,8 @@ public class AtlasDSL {
}
public static class Translator {
private static final Logger LOG = LoggerFactory.getLogger(Translator.class);
private final AtlasDSLParser.QueryContext queryContext;
private final AtlasTypeRegistry typeRegistry;
private final int offset;
......@@ -123,33 +125,21 @@ public class AtlasDSL {
GremlinQueryComposer gremlinQueryComposer = new GremlinQueryComposer(typeRegistry, queryMetadata, limit, offset);
DSLVisitor dslVisitor = new DSLVisitor(gremlinQueryComposer);
try {
queryContext.accept(dslVisitor);
queryContext.accept(dslVisitor);
processErrorList(gremlinQueryComposer, null);
processErrorList(gremlinQueryComposer);
return new GremlinQuery(gremlinQueryComposer.get(), queryMetadata.hasSelect());
} catch (Exception e) {
processErrorList(gremlinQueryComposer, e);
}
String gremlinQuery = gremlinQueryComposer.get();
return null;
return new GremlinQuery(gremlinQuery, queryMetadata.hasSelect());
}
private void processErrorList(GremlinQueryComposer gremlinQueryComposer, Exception e) throws AtlasBaseException {
private void processErrorList(GremlinQueryComposer gremlinQueryComposer) throws AtlasBaseException {
final String errorMessage;
if (CollectionUtils.isNotEmpty(gremlinQueryComposer.getErrorList())) {
errorMessage = StringUtils.join(gremlinQueryComposer.getErrorList(), ", ");
} else {
errorMessage = e != null ? (e.getMessage() != null ? e.getMessage() : e.toString()) : null;
}
if (errorMessage != null) {
if (e != null) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, e, this.query, errorMessage);
}
LOG.warn("DSL Errors: {}", errorMessage);
throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, this.query, errorMessage);
}
}
......
......@@ -26,15 +26,12 @@ import org.slf4j.LoggerFactory;
import java.util.*;
import static org.apache.atlas.query.antlr4.AtlasDSLParser.RULE_whereClause;
public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
private static final Logger LOG = LoggerFactory.getLogger(DSLVisitor.class);
private static final String AND = "AND";
private static final String OR = "OR";
private Set<Integer> visitedRuleIndexes = new HashSet<>();
private final GremlinQueryComposer gremlinQueryComposer;
public DSLVisitor(GremlinQueryComposer gremlinQueryComposer) {
......@@ -42,44 +39,6 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
@Override
public Void visitSpaceDelimitedQueries(SpaceDelimitedQueriesContext ctx) {
addVisitedRule(ctx.getRuleIndex());
return super.visitSpaceDelimitedQueries(ctx);
}
@Override
public Void visitCommaDelimitedQueries(CommaDelimitedQueriesContext ctx) {
addVisitedRule(ctx.getRuleIndex());
return super.visitCommaDelimitedQueries(ctx);
}
@Override
public Void visitIsClause(IsClauseContext ctx) {
if (LOG.isDebugEnabled()) {
LOG.debug("=> DSLVisitor.visitIsClause({})", ctx);
}
if(!hasVisitedRule(RULE_whereClause)) {
gremlinQueryComposer.addFromIsA(ctx.arithE().getText(), ctx.identifier().getText());
}
return super.visitIsClause(ctx);
}
@Override
public Void visitHasClause(HasClauseContext ctx) {
if (LOG.isDebugEnabled()) {
LOG.debug("=> DSLVisitor.visitHasClause({})", ctx);
}
if(!hasVisitedRule(RULE_whereClause)) {
gremlinQueryComposer.addFromProperty(ctx.arithE().getText(), ctx.identifier().getText());
}
return super.visitHasClause(ctx);
}
@Override
public Void visitLimitOffset(LimitOffsetContext ctx) {
if (LOG.isDebugEnabled()) {
LOG.debug("=> DSLVisitor.visitLimitOffset({})", ctx);
......@@ -159,7 +118,6 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
LOG.debug("=> DSLVisitor.visitWhereClause({})", ctx);
}
addVisitedRule(ctx.getRuleIndex());
ExprContext expr = ctx.expr();
processExpr(expr, gremlinQueryComposer);
return super.visitWhereClause(ctx);
......@@ -171,7 +129,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
LOG.debug("=> DSLVisitor.visitFromExpression({})", ctx);
}
FromSrcContext fromSrc = ctx.fromSrc();
FromSrcContext fromSrc = ctx.fromSrc();
AliasExprContext aliasExpr = fromSrc.aliasExpr();
if (aliasExpr != null) {
......@@ -188,11 +146,13 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
@Override
public Void visitSingleQrySrc(SingleQrySrcContext ctx) {
if (!hasVisitedRule(RULE_whereClause)) {
if (ctx.fromExpression() == null) {
if (ctx.expr() != null && gremlinQueryComposer.hasFromClause()) {
processExpr(ctx.expr(), gremlinQueryComposer);
}
if (ctx.fromExpression() == null) {
if (ctx.expr() != null && !gremlinQueryComposer.hasFromClause()) {
inferFromClause(ctx);
}
if (ctx.expr() != null && gremlinQueryComposer.hasFromClause()) {
processExpr(ctx.expr(), gremlinQueryComposer);
}
}
......@@ -210,6 +170,43 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
return super.visitGroupByExpression(ctx);
}
private Void visitIsClause(GremlinQueryComposer gqc, IsClauseContext ctx) {
if (LOG.isDebugEnabled()) {
LOG.debug("=> DSLVisitor.visitIsClause({})", ctx);
}
gqc.addIsA(ctx.arithE().getText(), ctx.identifier().getText());
return super.visitIsClause(ctx);
}
private void visitHasClause(GremlinQueryComposer gqc, HasClauseContext ctx) {
if (LOG.isDebugEnabled()) {
LOG.debug("=> DSLVisitor.visitHasClause({})", ctx);
}
gqc.addFromProperty(ctx.arithE().getText(), ctx.identifier().getText());
super.visitHasClause(ctx);
}
private void inferFromClause(SingleQrySrcContext ctx) {
if (ctx.fromExpression() != null) {
return;
}
if (ctx.expr() != null && gremlinQueryComposer.hasFromClause()) {
return;
}
if (ctx.expr().compE() != null && ctx.expr().compE().isClause() != null && ctx.expr().compE().isClause().arithE() != null) {
gremlinQueryComposer.addFrom(ctx.expr().compE().isClause().arithE().getText());
return;
}
if (ctx.expr().compE() != null && ctx.expr().compE().hasClause() != null && ctx.expr().compE().hasClause().arithE() != null) {
gremlinQueryComposer.addFrom(ctx.expr().compE().hasClause().arithE().getText());
}
}
private void processExpr(final ExprContext expr, GremlinQueryComposer gremlinQueryComposer) {
if (CollectionUtils.isNotEmpty(expr.exprRight())) {
processExprRight(expr, gremlinQueryComposer);
......@@ -281,11 +278,11 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
if (comparisonClause != null) {
String lhs = comparisonClause.arithE(0).getText();
String op, rhs;
String lhs = comparisonClause.arithE(0).getText();
String op, rhs;
AtomEContext atomECtx = comparisonClause.arithE(1).multiE().atomE();
if (atomECtx.literal() == null ||
(atomECtx.literal() != null && atomECtx.literal().valueArray() == null)) {
(atomECtx.literal() != null && atomECtx.literal().valueArray() == null)) {
op = comparisonClause.operator().getText().toUpperCase();
rhs = comparisonClause.arithE(1).getText();
} else {
......@@ -300,31 +297,23 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
if (compE != null && compE.isClause() != null) {
gremlinQueryComposer.addFromIsA(compE.isClause().arithE().getText(), compE.isClause().identifier().getText());
visitIsClause(gremlinQueryComposer, compE.isClause());
}
if (compE != null && compE.hasClause() != null) {
gremlinQueryComposer.addFromProperty(compE.hasClause().arithE().getText(), compE.hasClause().identifier().getText());
visitHasClause(gremlinQueryComposer, compE.hasClause());
}
}
private String getInClause(AtomEContext atomEContext) {
StringBuilder sb = new StringBuilder();
StringBuilder sb = new StringBuilder();
ValueArrayContext valueArrayContext = atomEContext.literal().valueArray();
int startIdx = 1;
int endIdx = valueArrayContext.children.size() - 1;
int startIdx = 1;
int endIdx = valueArrayContext.children.size() - 1;
for (int i = startIdx; i < endIdx; i++) {
sb.append(valueArrayContext.getChild(i));
}
return sb.toString();
}
private void addVisitedRule(int ruleIndex) {
visitedRuleIndexes.add(ruleIndex);
}
private boolean hasVisitedRule(int ruleIndex) {
return visitedRuleIndexes.contains(ruleIndex);
}
}
}
\ No newline at end of file
......@@ -36,11 +36,11 @@ enum GremlinClause {
AND("and(%s)"),
NESTED_START("__"),
NESTED_HAS_OPERATOR("has('%s', %s(%s))"),
LIMIT("limit(%s)"),
LIMIT("limit(local, %s).limit(%s)"),
ORDER_BY("order().by('%s')"),
ORDER_BY_DESC("order().by('%s', decr)"),
OUT("out('%s')"),
RANGE("range(%s, %s + %s)"),
RANGE("range(local, %s, %s + %s).range(%s, %s + %s)"),
SELECT("select('%s')"),
TO_LIST("toList()"),
TEXT_CONTAINS("has('%s', org.janusgraph.core.attribute.Text.textRegex(%s))"),
......
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.query;
import org.apache.atlas.type.AtlasEntityType;
import java.util.LinkedList;
import java.util.List;
class GremlinClauseList {
private final List<GremlinQueryComposer.GremlinClauseValue> list;
GremlinClauseList() {
this.list = new LinkedList<>();
}
public void add(GremlinQueryComposer.GremlinClauseValue g) {
list.add(g);
}
public void add(int idx, GremlinQueryComposer.GremlinClauseValue g) {
list.add(idx, g);
}
public void add(GremlinQueryComposer.GremlinClauseValue g, AtlasEntityType t) {
add(g);
}
public void add(int idx, GremlinQueryComposer.GremlinClauseValue g, AtlasEntityType t) {
add(idx, g);
}
public void add(GremlinClause clause, String... args) {
list.add(new GremlinQueryComposer.GremlinClauseValue(clause, clause.get(args)));
}
public void add(int i, GremlinClause clause, String... args) {
list.add(i, new GremlinQueryComposer.GremlinClauseValue(clause, clause.get(args)));
}
public GremlinQueryComposer.GremlinClauseValue getAt(int i) {
return list.get(i);
}
public String getValue(int i) {
return list.get(i).getValue();
}
public GremlinQueryComposer.GremlinClauseValue get(int i) {
return list.get(i);
}
public int size() {
return list.size();
}
public int contains(GremlinClause clause) {
for (int i = 0; i < list.size(); i++) {
if (list.get(i).getClause() == clause)
return i;
}
return -1;
}
public boolean isEmpty() {
return list.size() == 0 || containsGVLimit();
}
private boolean containsGVLimit() {
return list.size() == 3 &&
list.get(0).getClause() == GremlinClause.G &&
list.get(1).getClause() == GremlinClause.V &&
list.get(2).getClause() == GremlinClause.LIMIT;
}
public void clear() {
list.clear();
}
public GremlinQueryComposer.GremlinClauseValue remove(int index) {
GremlinQueryComposer.GremlinClauseValue gcv = get(index);
list.remove(index);
return gcv;
}
}
......@@ -18,7 +18,9 @@
package org.apache.atlas.query;
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.type.AtlasEntityType;
......@@ -88,24 +90,24 @@ public class GremlinQueryComposer {
LOG.debug("addFrom(typeName={})", typeName);
}
IdentifierHelper.Advice ta = getAdvice(typeName);
IdentifierHelper.IdentifierMetadata ta = getIdMetadata(typeName);
if(context.shouldRegister(ta.get())) {
context.registerActive(ta.get());
IdentifierHelper.Advice ia = getAdvice(ta.get());
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(ta.get());
if (ia.isTrait()) {
add(GremlinClause.TRAIT, ia.get());
add(GremlinClause.TRAIT, ia);
} else {
if (ia.hasSubtypes()) {
add(GremlinClause.HAS_TYPE_WITHIN, ia.getSubTypes());
} else {
add(GremlinClause.HAS_TYPE, ia.get());
add(GremlinClause.HAS_TYPE, ia);
}
}
} else {
IdentifierHelper.Advice ia = getAdvice(ta.get());
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(ta.get());
introduceType(ia);
}
}
......@@ -119,16 +121,16 @@ public class GremlinQueryComposer {
addFrom(typeName);
}
add(GremlinClause.HAS_PROPERTY,
IdentifierHelper.getQualifiedName(lookup, context, attribute));
add(GremlinClause.HAS_PROPERTY, getIdMetadata(attribute));
}
public void addFromIsA(String typeName, String traitName) {
public void addIsA(String typeName, String traitName) {
if (!isNestedQuery) {
addFrom(typeName);
}
add(GremlinClause.TRAIT, traitName);
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(traitName);
add(GremlinClause.TRAIT, ia);
}
public void addWhere(String lhs, String operator, String rhs) {
......@@ -136,64 +138,51 @@ public class GremlinQueryComposer {
LOG.debug("addWhere(lhs={}, operator={}, rhs={})", lhs, operator, rhs);
}
String currentType = context.getActiveTypeName();
SearchParameters.Operator op = SearchParameters.Operator.fromString(operator);
IdentifierHelper.Advice org = null;
IdentifierHelper.Advice lhsI = getAdvice(lhs);
String currentType = context.getActiveTypeName();
IdentifierHelper.IdentifierMetadata org = null;
IdentifierHelper.IdentifierMetadata lhsI = getIdMetadata(lhs);
if (!lhsI.isPrimitive()) {
introduceType(lhsI);
org = lhsI;
lhsI = getAdvice(lhs);
lhsI = getIdMetadata(lhs);
}
if (StringUtils.isEmpty(lhsI.getQualifiedName())) {
if (LOG.isDebugEnabled()) {
LOG.debug("unknown identifier '" + lhsI.getRaw() + "'");
}
context.getErrorList().add("unknown identifier '" + lhsI.getRaw() + "'");
} else {
if (lhsI.isDate()) {
rhs = parseDate(rhs);
}
rhs = addQuotesIfNecessary(rhs);
if (op == SearchParameters.Operator.LIKE) {
add(GremlinClause.TEXT_CONTAINS, lhsI.getQualifiedName(), getFixedRegEx(rhs));
} else if (op == SearchParameters.Operator.IN) {
add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), "within", rhs);
} else {
add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), op.getSymbols()[1], rhs);
}
if (!context.validator.isValidQualifiedName(lhsI.getQualifiedName(), lhsI.getRaw())) {
return;
}
if (org != null && org.getIntroduceType()) {
add(GremlinClause.DEDUP);
add(GremlinClause.IN, org.getEdgeLabel());
context.registerActive(currentType);
}
if (lhsI.isDate()) {
rhs = parseDate(rhs);
}
}
private String getQualifiedName(IdentifierHelper.Advice ia) {
String s = ia.getQualifiedName();
if(StringUtils.isEmpty(s)) {
s = ia.getRaw();
getErrorList().add(String.format("Error: %s is invalid", ia.getRaw()));
SearchParameters.Operator op = SearchParameters.Operator.fromString(operator);
rhs = addQuotesIfNecessary(rhs);
if (op == SearchParameters.Operator.LIKE) {
add(GremlinClause.TEXT_CONTAINS, lhsI.getQualifiedName(), IdentifierHelper.getFixedRegEx(rhs));
} else if (op == SearchParameters.Operator.IN) {
add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), "within", rhs);
} else {
add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), op.getSymbols()[1], rhs);
}
return s;
if (org != null && org.isReferredType()) {
add(GremlinClause.DEDUP);
add(GremlinClause.IN, org.getEdgeLabel());
context.registerActive(currentType);
}
}
private String getFixedRegEx(String rhs) {
return rhs.replace("*", ".*").replace('?', '.');
private String getQualifiedName(IdentifierHelper.IdentifierMetadata ia) {
return context.validator.isValidQualifiedName(ia.getQualifiedName(), ia.getRaw()) ?
ia.getQualifiedName() : ia.getRaw();
}
public void addAndClauses(List<String> clauses) {
queryClauses.add(GremlinClause.AND, String.join(",", clauses));
add(GremlinClause.AND, String.join(",", clauses));
}
public void addOrClauses(List<String> clauses) {
queryClauses.add(GremlinClause.OR, String.join(",", clauses));
add(GremlinClause.OR, String.join(",", clauses));
}
public void addSelect(SelectClauseComposer selectClauseComposer) {
......@@ -214,7 +203,12 @@ public class GremlinQueryComposer {
}
for (int i = 0; i < scc.getItems().length; i++) {
IdentifierHelper.Advice ia = getAdvice(scc.getItem(i));
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(scc.getItem(i));
if(scc.isAggregatorWithArgument(i) && !ia.isPrimitive()) {
context.check(false, AtlasErrorCode.INVALID_DSL_SELECT_INVALID_AGG, ia.getQualifiedName());
return;
}
if (!scc.getItem(i).equals(scc.getLabel(i))) {
context.addAlias(scc.getLabel(i), getQualifiedName(ia));
......@@ -225,25 +219,29 @@ public class GremlinQueryComposer {
}
scc.isSelectNoop = hasNoopCondition(ia);
if(scc.isSelectNoop) {
if (scc.isSelectNoop) {
return;
}
if (introduceType(ia)) {
scc.incrementTypesIntroduced();
scc.isSelectNoop = !ia.hasParts();
if(ia.hasParts()) {
scc.assign(i, getQualifiedName(getAdvice(ia.get())), GremlinClause.INLINE_GET_PROPERTY);
if (ia.hasParts()) {
scc.assign(i, getQualifiedName(getIdMetadata(ia.get())), GremlinClause.INLINE_GET_PROPERTY);
}
} else {
scc.assign(i, getQualifiedName(ia), GremlinClause.INLINE_GET_PROPERTY);
scc.incrementPrimitiveType();
}
}
context.validator.check(!scc.hasMultipleReferredTypes(),
AtlasErrorCode.INVALID_DSL_SELECT_REFERRED_ATTR, Integer.toString(scc.getIntroducedTypesCount()));
context.validator.check(!scc.hasMixedAttributes(), AtlasErrorCode.INVALID_DSL_SELECT_ATTR_MIXING);
}
private boolean hasNoopCondition(IdentifierHelper.Advice ia) {
return ia.isPrimitive() == false &&
ia.isAttribute() == false &&
context.hasAlias(ia.getRaw());
private boolean hasNoopCondition(IdentifierHelper.IdentifierMetadata ia) {
return !ia.isPrimitive() && !ia.isAttribute() && context.hasAlias(ia.getRaw());
}
public GremlinQueryComposer createNestedProcessor() {
......@@ -284,7 +282,7 @@ public class GremlinQueryComposer {
}
if (offset.equalsIgnoreCase("0")) {
add(GremlinClause.LIMIT, limit);
add(GremlinClause.LIMIT, limit, limit);
} else {
addRangeClause(offset, limit);
}
......@@ -342,7 +340,7 @@ public class GremlinQueryComposer {
LOG.debug("addOrderBy(name={}, isDesc={})", name, isDesc);
}
IdentifierHelper.Advice ia = getAdvice(name);
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(name);
if (queryMetadata.hasSelect() && queryMetadata.hasGroupBy()) {
addSelectTransformation(this.context.selectClauseComposer, getQualifiedName(ia), isDesc);
} else if (queryMetadata.hasGroupBy()) {
......@@ -370,7 +368,7 @@ public class GremlinQueryComposer {
GremlinClause.SELECT_FN;
}
if (StringUtils.isEmpty(orderByQualifiedAttrName)) {
queryClauses.add(0, fn,
add(0, fn,
selectClauseComposer.getLabelHeader(),
selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString(): EMPTY_STRING,
selectClauseComposer.getItemsString(), EMPTY_STRING);
......@@ -381,14 +379,15 @@ public class GremlinQueryComposer {
sortClause = isDesc ? GremlinClause.INLINE_SORT_DESC : GremlinClause.INLINE_SORT_ASC;
}
String idxStr = String.valueOf(itemIdx);
queryClauses.add(0, fn,
add(0, fn,
selectClauseComposer.getLabelHeader(),
selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString(): EMPTY_STRING,
selectClauseComposer.getItemsString(),
sortClause.get(idxStr, idxStr)
);
}
queryClauses.add(GremlinClause.INLINE_TRANSFORM_CALL);
add(GremlinClause.INLINE_TRANSFORM_CALL);
}
private String addQuotesIfNecessary(String rhs) {
......@@ -410,7 +409,7 @@ public class GremlinQueryComposer {
try {
return DSL_DATE_FORMAT.get().parse(s).getTime();
} catch (ParseException ex) {
context.errorList.add(ex.getMessage());
context.validator.check(ex, AtlasErrorCode.INVALID_DSL_INVALID_DATE);
}
return -1;
......@@ -453,36 +452,28 @@ public class GremlinQueryComposer {
}
}
private boolean introduceType(IdentifierHelper.Advice ia) {
if (ia.getIntroduceType()) {
private boolean introduceType(IdentifierHelper.IdentifierMetadata ia) {
if (ia.isReferredType()) {
add(GremlinClause.OUT, ia.getEdgeLabel());
context.registerActive(ia);
}
return ia.getIntroduceType();
return ia.isReferredType();
}
private IdentifierHelper.Advice getAdvice(String actualTypeName) {
private IdentifierHelper.IdentifierMetadata getIdMetadata(String actualTypeName) {
return IdentifierHelper.create(context, lookup, actualTypeName);
}
private void add(GremlinClause clause, String... args) {
queryClauses.add(new GremlinClauseValue(clause, clause.get(args)));
}
private void add(int idx, GremlinClause clause, String... args) {
queryClauses.add(idx, new GremlinClauseValue(clause, clause.get(args)));
}
private void addRangeClause(String startIndex, String endIndex) {
if (LOG.isDebugEnabled()) {
LOG.debug("addRangeClause(startIndex={}, endIndex={})", startIndex, endIndex);
}
if (queryMetadata.hasSelect()) {
add(queryClauses.size() - 1, GremlinClause.RANGE, startIndex, startIndex, endIndex);
add(queryClauses.size() - 1, GremlinClause.RANGE, startIndex, startIndex, endIndex, startIndex, startIndex, endIndex);
} else {
add(GremlinClause.RANGE, startIndex, startIndex, endIndex);
add(GremlinClause.RANGE, startIndex, startIndex, endIndex, startIndex, startIndex, endIndex);
}
}
......@@ -491,8 +482,8 @@ public class GremlinQueryComposer {
LOG.debug("addOrderByClause(name={})", name, descr);
}
IdentifierHelper.Advice ia = getAdvice(name);
add((!descr) ? GremlinClause.ORDER_BY : GremlinClause.ORDER_BY_DESC, getQualifiedName(ia));
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(name);
add((!descr) ? GremlinClause.ORDER_BY : GremlinClause.ORDER_BY_DESC, ia);
}
private void addGroupByClause(String name) {
......@@ -500,8 +491,8 @@ public class GremlinQueryComposer {
LOG.debug("addGroupByClause(name={})", name);
}
IdentifierHelper.Advice ia = getAdvice(name);
add(GremlinClause.GROUP_BY, getQualifiedName(ia));
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(name);
add(GremlinClause.GROUP_BY, ia);
}
public boolean hasFromClause() {
......@@ -509,7 +500,23 @@ public class GremlinQueryComposer {
queryClauses.contains(GremlinClause.HAS_TYPE_WITHIN) != -1;
}
private static class GremlinClauseValue {
private void add(GremlinClause clause, IdentifierHelper.IdentifierMetadata ia) {
if(context != null && !context.validator.isValid(context, clause, ia)) {
return;
}
add(clause, (ia.getQualifiedName() == null ? ia.get() : ia.getQualifiedName()));
}
private void add(GremlinClause clause, String... args) {
queryClauses.add(new GremlinClauseValue(clause, clause.get(args)));
}
private void add(int idx, GremlinClause clause, String... args) {
queryClauses.add(idx, new GremlinClauseValue(clause, clause.get(args)));
}
static class GremlinClauseValue {
private final GremlinClause clause;
private final String value;
......@@ -527,124 +534,42 @@ public class GremlinQueryComposer {
}
}
private static class GremlinClauseList {
private final List<GremlinClauseValue> list;
private GremlinClauseList() {
this.list = new LinkedList<>();
}
public void add(GremlinClauseValue g) {
list.add(g);
}
public void add(int idx, GremlinClauseValue g) {
list.add(idx, g);
}
public void add(GremlinClauseValue g, AtlasEntityType t) {
add(g);
}
public void add(int idx, GremlinClauseValue g, AtlasEntityType t) {
add(idx, g);
}
public void add(GremlinClause clause, String... args) {
list.add(new GremlinClauseValue(clause, clause.get(args)));
}
public void add(int i, GremlinClause clause, String... args) {
list.add(i, new GremlinClauseValue(clause, clause.get(args)));
}
public GremlinClauseValue getAt(int i) {
return list.get(i);
}
public String getValue(int i) {
return list.get(i).value;
}
public GremlinClauseValue get(int i) {
return list.get(i);
}
public int size() {
return list.size();
}
public int contains(GremlinClause clause) {
for (int i = 0; i < list.size(); i++) {
if (list.get(i).getClause() == clause)
return i;
}
return -1;
}
public boolean isEmpty() {
return list.size() == 0 || containsGVLimit();
}
private boolean containsGVLimit() {
return list.size() == 3 &&
list.get(0).clause == GremlinClause.G &&
list.get(1).clause == GremlinClause.V &&
list.get(2).clause == GremlinClause.LIMIT;
}
public void clear() {
list.clear();
}
public GremlinClauseValue remove(int index) {
GremlinClauseValue gcv = get(index);
list.remove(index);
return gcv;
}
}
@VisibleForTesting
static class Context {
private static final AtlasStructType UNKNOWN_TYPE = new AtlasStructType(new AtlasStructDef());
private final Lookup lookup;
private final List<String> errorList = new ArrayList<>();
private final Map<String, String> aliasMap = new HashMap<>();
private AtlasType activeType;
private SelectClauseComposer selectClauseComposer;
private ClauseValidator validator;
public Context(Lookup lookup) {
this.lookup = lookup;
validator = new ClauseValidator(lookup);
}
public void registerActive(String typeName) {
if(shouldRegister(typeName)) {
try {
activeType = lookup.getType(typeName);
aliasMap.put(typeName, typeName);
} catch (AtlasBaseException e) {
errorList.add(e.getMessage());
validator.check(e, AtlasErrorCode.INVALID_DSL_UNKNOWN_TYPE, typeName);
activeType = UNKNOWN_TYPE;
}
}
}
public void registerActive(IdentifierHelper.Advice advice) {
if (StringUtils.isNotEmpty(advice.getTypeName())) {
registerActive(advice.getTypeName());
public void registerActive(IdentifierHelper.IdentifierMetadata identifierMetadata) {
if (validator.check(StringUtils.isNotEmpty(identifierMetadata.getTypeName()),
AtlasErrorCode.INVALID_DSL_UNKNOWN_TYPE, identifierMetadata.getRaw())) {
registerActive(identifierMetadata.getTypeName());
} else {
errorList.add("unknown identifier '" + advice.getRaw() + "'");
activeType = UNKNOWN_TYPE;
}
}
public AtlasType getActiveType() {
return activeType;
}
public AtlasEntityType getActiveEntityType() {
return (activeType instanceof AtlasEntityType) ?
(AtlasEntityType) activeType :
......@@ -655,6 +580,10 @@ public class GremlinQueryComposer {
return activeType.getTypeName();
}
public AtlasType getActiveType() {
return activeType;
}
public boolean shouldRegister(String typeName) {
return activeType == null ||
(activeType != null && !StringUtils.equals(getActiveTypeName(), typeName)) &&
......@@ -683,7 +612,7 @@ public class GremlinQueryComposer {
public void addAlias(String alias, String typeName) {
if(aliasMap.containsKey(alias)) {
errorList.add(String.format("Duplicate alias found: %s for type %s already present.", alias, getActiveEntityType()));
check(false, AtlasErrorCode.INVALID_DSL_DUPLICATE_ALIAS, alias, getActiveTypeName());
return;
}
......@@ -691,7 +620,81 @@ public class GremlinQueryComposer {
}
public List<String> getErrorList() {
return validator.getErrorList();
}
public boolean error(AtlasBaseException e, AtlasErrorCode ec, String t, String name) {
return validator.check(e, ec, t, name);
}
public boolean check(boolean condition, AtlasErrorCode vm, String... args) {
return validator.check(condition, vm, args);
}
}
private static class ClauseValidator {
private final Lookup lookup;
List<String> errorList = new ArrayList<>();
public ClauseValidator(Lookup lookup) {
this.lookup = lookup;
}
public boolean isValid(Context ctx, GremlinClause clause, IdentifierHelper.IdentifierMetadata ia) {
switch (clause) {
case TRAIT:
return check(ia.isTrait(), AtlasErrorCode.INVALID_DSL_UNKNOWN_CLASSIFICATION, ia.getRaw());
case HAS_TYPE:
TypeCategory typeCategory = ctx.getActiveType().getTypeCategory();
return check(StringUtils.isNotEmpty(ia.getTypeName()) &&
typeCategory == TypeCategory.CLASSIFICATION || typeCategory == TypeCategory.ENTITY,
AtlasErrorCode.INVALID_DSL_UNKNOWN_TYPE, ia.getRaw());
case HAS_PROPERTY:
return check(ia.isPrimitive(), AtlasErrorCode.INVALID_DSL_HAS_PROPERTY, ia.getRaw());
case ORDER_BY:
return check(ia.isPrimitive(), AtlasErrorCode.INVALID_DSL_ORDERBY, ia.getRaw());
case GROUP_BY:
return check(ia.isPrimitive(), AtlasErrorCode.INVALID_DSL_SELECT_INVALID_AGG, ia.getRaw());
default:
return (getErrorList().size() == 0);
}
}
public boolean check(Exception ex, AtlasErrorCode vm, String... args) {
String[] extraArgs = getExtraSlotArgs(args, ex.getMessage());
return check(false, vm, extraArgs);
}
private String[] getExtraSlotArgs(String[] args, String s) {
String[] argsPlus1 = new String[args.length + 1];
System.arraycopy(args, 0, argsPlus1, 0, args.length);
argsPlus1[args.length] = s;
return argsPlus1;
}
public boolean check(boolean condition, AtlasErrorCode vm, String... args) {
if(!condition) {
addError(vm, args);
}
return condition;
}
public void addError(AtlasErrorCode ec, String... messages) {
errorList.add(ec.getFormattedErrorMessage(messages));
}
public List<String> getErrorList() {
return errorList;
}
public boolean isValidQualifiedName(String qualifiedName, String raw) {
return check(StringUtils.isNotEmpty(qualifiedName), AtlasErrorCode.INVALID_DSL_QUALIFIED_NAME, raw);
}
}
}
......@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......@@ -18,6 +18,7 @@
package org.apache.atlas.query;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.commons.lang.StringUtils;
......@@ -46,10 +47,10 @@ public class IdentifierHelper {
return ret;
}
public static Advice create(GremlinQueryComposer.Context context,
org.apache.atlas.query.Lookup lookup,
String identifier) {
Advice ia = new Advice(identifier);
public static IdentifierMetadata create(GremlinQueryComposer.Context context,
org.apache.atlas.query.Lookup lookup,
String identifier) {
IdentifierMetadata ia = new IdentifierMetadata(identifier);
ia.update(lookup, context);
return ia;
}
......@@ -65,7 +66,7 @@ public class IdentifierHelper {
try {
return lookup.getQualifiedName(context, name);
} catch (AtlasBaseException e) {
context.getErrorList().add(String.format("Error for %s.%s: %s", context.getActiveTypeName(), name, e.getMessage()));
context.error(e, AtlasErrorCode.INVALID_DSL_QUALIFIED_NAME, context.getActiveTypeName(), name);
}
return "";
......@@ -100,24 +101,29 @@ public class IdentifierHelper {
return rhs.equalsIgnoreCase("true") || rhs.equalsIgnoreCase("false");
}
public static class Advice {
private String raw;
private String actual;
public static String getFixedRegEx(String s) {
return s.replace("*", ".*").replace('?', '.');
}
public static class IdentifierMetadata {
private String raw;
private String actual;
private String[] parts;
private String typeName;
private String attributeName;
private boolean isPrimitive;
private String edgeLabel;
private boolean introduceType;
private boolean hasSubtypes;
private String subTypes;
private boolean isTrait;
private boolean newContext;
private boolean isAttribute;
private String qualifiedName;
private boolean isDate;
public Advice(String s) {
private String typeName;
private String attributeName;
private boolean isPrimitive;
private String edgeLabel;
private boolean introduceType;
private boolean hasSubtypes;
private String subTypes;
private boolean isTrait;
private boolean newContext;
private boolean isAttribute;
private String qualifiedName;
private boolean isDate;
public IdentifierMetadata(String s) {
this.raw = removeQuotes(s);
this.actual = IdentifierHelper.get(raw);
}
......@@ -132,7 +138,7 @@ public class IdentifierHelper {
updateParts();
updateTypeInfo(lookup, context);
isTrait = lookup.isTraitType(context);
setIsTrait(context, lookup, attributeName);
updateEdgeInfo(lookup, context);
introduceType = !isPrimitive() && !context.hasAlias(parts[0]);
updateSubTypes(lookup, context);
......@@ -142,41 +148,45 @@ public class IdentifierHelper {
}
}
private void setIsTrait(GremlinQueryComposer.Context ctx, Lookup lookup, String s) {
isTrait = lookup.isTraitType(s);
}
private void updateSubTypes(org.apache.atlas.query.Lookup lookup, GremlinQueryComposer.Context context) {
if(isTrait) {
if (isTrait) {
return;
}
hasSubtypes = lookup.doesTypeHaveSubTypes(context);
if(hasSubtypes) {
if (hasSubtypes) {
subTypes = lookup.getTypeAndSubTypes(context);
}
}
private void updateEdgeInfo(org.apache.atlas.query.Lookup lookup, GremlinQueryComposer.Context context) {
if(isPrimitive == false && isTrait == false) {
if (!isPrimitive && !isTrait && typeName != attributeName) {
edgeLabel = lookup.getRelationshipEdgeLabel(context, attributeName);
typeName = lookup.getTypeFromEdge(context, attributeName);
}
}
private void updateTypeInfo(org.apache.atlas.query.Lookup lookup, GremlinQueryComposer.Context context) {
if(parts.length == 1) {
if (parts.length == 1) {
typeName = context.hasAlias(parts[0]) ?
context.getTypeNameFromAlias(parts[0]) :
context.getActiveTypeName();
context.getTypeNameFromAlias(parts[0]) :
context.getActiveTypeName();
qualifiedName = getDefaultQualifiedNameForSinglePartName(context, parts[0]);
attributeName = parts[0];
}
if(parts.length == 2) {
if (parts.length == 2) {
boolean isAttrOfActiveType = lookup.hasAttribute(context, parts[0]);
if(isAttrOfActiveType) {
if (isAttrOfActiveType) {
attributeName = parts[0];
} else {
typeName = context.hasAlias(parts[0]) ?
context.getTypeNameFromAlias(parts[0]) :
parts[0];
context.getTypeNameFromAlias(parts[0]) :
parts[0];
attributeName = parts[1];
}
......@@ -190,7 +200,7 @@ public class IdentifierHelper {
private String getDefaultQualifiedNameForSinglePartName(GremlinQueryComposer.Context context, String s) {
String qn = context.getTypeNameFromAlias(s);
if(StringUtils.isEmpty(qn) && SelectClauseComposer.isKeyword(s)) {
if (StringUtils.isEmpty(qn) && SelectClauseComposer.isKeyword(s)) {
return s;
}
......@@ -198,22 +208,17 @@ public class IdentifierHelper {
}
private void setQualifiedName(Lookup lookup, GremlinQueryComposer.Context context, boolean isAttribute, String attrName) {
if(isAttribute) {
if (isAttribute) {
qualifiedName = getQualifiedName(lookup, context, attrName);
}
}
private String getQualifiedName(Lookup lookup, GremlinQueryComposer.Context context, String name) {
try {
return lookup.getQualifiedName(context, name);
} catch (AtlasBaseException e) {
context.getErrorList().add(String.format("Error for %s.%s: %s", context.getActiveTypeName(), name, e.getMessage()));
return "";
}
return IdentifierHelper.getQualifiedName(lookup, context, name);
}
private void setIsDate(Lookup lookup, GremlinQueryComposer.Context context, boolean isPrimitive, String attrName) {
if(isPrimitive) {
if (isPrimitive) {
isDate = lookup.isDate(context, attrName);
}
}
......@@ -246,7 +251,7 @@ public class IdentifierHelper {
return typeName;
}
public boolean getIntroduceType() {
public boolean isReferredType() {
return introduceType;
}
......@@ -277,5 +282,6 @@ public class IdentifierHelper {
public String getRaw() {
return raw;
}
}
}
......@@ -21,9 +21,6 @@ package org.apache.atlas.query;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.type.AtlasType;
import java.util.Collection;
import java.util.List;
public interface Lookup {
AtlasType getType(String typeName) throws AtlasBaseException;
......@@ -39,7 +36,7 @@ public interface Lookup {
String getTypeAndSubTypes(GremlinQueryComposer.Context context);
boolean isTraitType(GremlinQueryComposer.Context context);
boolean isTraitType(String s);
String getTypeFromEdge(GremlinQueryComposer.Context context, String item);
......
......@@ -20,20 +20,27 @@ package org.apache.atlas.query;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.type.*;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
class RegistryBasedLookup implements Lookup {
private final List<String> errorList;
private static final Set<String> SYSTEM_ATTRIBUTES = new HashSet<>(
Arrays.asList(Constants.GUID_PROPERTY_KEY,
Constants.MODIFIED_BY_KEY,
Constants.CREATED_BY_KEY,
Constants.STATE_PROPERTY_KEY,
Constants.TIMESTAMP_PROPERTY_KEY,
Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY));
private final AtlasTypeRegistry typeRegistry;
public RegistryBasedLookup(AtlasTypeRegistry typeRegistry) {
this.errorList = new ArrayList<>();
this.typeRegistry = typeRegistry;
}
......@@ -49,7 +56,15 @@ class RegistryBasedLookup implements Lookup {
return "";
}
return et.getQualifiedAttributeName(name);
if(isSystemAttribute(name)) {
return name;
} else {
return et.getQualifiedAttributeName(name);
}
}
private boolean isSystemAttribute(String s) {
return SYSTEM_ATTRIBUTES.contains(s);
}
@Override
......@@ -59,6 +74,10 @@ class RegistryBasedLookup implements Lookup {
return false;
}
if(isSystemAttribute(attributeName)) {
return true;
}
AtlasType at = et.getAttributeType(attributeName);
if(at == null) {
return false;
......@@ -97,7 +116,8 @@ class RegistryBasedLookup implements Lookup {
@Override
public boolean hasAttribute(GremlinQueryComposer.Context context, String typeName) {
return (context.getActiveEntityType() != null) && context.getActiveEntityType().getAttribute(typeName) != null;
return (context.getActiveEntityType() != null) &&
(isSystemAttribute(typeName) || context.getActiveEntityType().getAttribute(typeName) != null);
}
@Override
......@@ -123,9 +143,19 @@ class RegistryBasedLookup implements Lookup {
}
@Override
public boolean isTraitType(GremlinQueryComposer.Context context) {
return (context.getActiveType() != null &&
context.getActiveType().getTypeCategory() == TypeCategory.CLASSIFICATION);
public boolean isTraitType(String typeName) {
AtlasType t = null;
try {
t = typeRegistry.getType(typeName);
} catch (AtlasBaseException e) {
return false;
}
return isTraitType(t);
}
private boolean isTraitType(AtlasType t) {
return (t != null && t.getTypeCategory() == TypeCategory.CLASSIFICATION);
}
@Override
......
......@@ -41,9 +41,18 @@ class SelectClauseComposer {
private int maxIdx = -1;
private int minIdx = -1;
private int aggCount = 0;
private int introducedTypesCount = 0;
private int primitiveTypeCount = 0;
public SelectClauseComposer() {}
public static boolean isKeyword(String s) {
return COUNT_STR.equals(s) ||
MIN_STR.equals(s) ||
MAX_STR.equals(s) ||
SUM_STR.equals(s);
}
public String[] getItems() {
return items;
}
......@@ -73,13 +82,6 @@ class SelectClauseComposer {
return ret;
}
public static boolean isKeyword(String s) {
return COUNT_STR.equals(s) ||
MIN_STR.equals(s) ||
MAX_STR.equals(s) ||
SUM_STR.equals(s);
}
public String[] getAttributes() {
return attributes;
}
......@@ -164,7 +166,7 @@ class SelectClauseComposer {
return assign(items[i], inline.get(s, clause.get(p1, p1)));
}
private int getCountIdx() {
public int getCountIdx() {
return countIdx;
}
......@@ -173,7 +175,7 @@ class SelectClauseComposer {
aggCount++;
}
private int getSumIdx() {
public int getSumIdx() {
return sumIdx;
}
......@@ -182,7 +184,7 @@ class SelectClauseComposer {
aggCount++;
}
private int getMaxIdx() {
public int getMaxIdx() {
return maxIdx;
}
......@@ -191,7 +193,7 @@ class SelectClauseComposer {
aggCount++;
}
private int getMinIdx() {
public int getMinIdx() {
return minIdx;
}
......@@ -207,4 +209,32 @@ class SelectClauseComposer {
.forEach(joiner::add);
return joiner.toString();
}
public boolean isAggregatorWithArgument(int i) {
return i == getMaxIdx() || i == getMinIdx() || i == getSumIdx();
}
public void incrementTypesIntroduced() {
introducedTypesCount++;
}
public int getIntroducedTypesCount() {
return introducedTypesCount;
}
public void incrementPrimitiveType() {
primitiveTypeCount++;
}
public boolean hasMultipleReferredTypes() {
return getIntroducedTypesCount() > 1;
}
public boolean hasMixedAttributes() {
return getIntroducedTypesCount() > 0 && getPrimitiveTypeCount() > 0;
}
private int getPrimitiveTypeCount() {
return primitiveTypeCount;
}
}
......@@ -23,10 +23,13 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.commons.collections.CollectionUtils;
import org.testng.annotations.*;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -39,6 +42,7 @@ import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
public class DSLQueriesTest extends BasicTestSetup {
private final int DEFAULT_LIMIT = 25;
@Inject
private EntityDiscoveryService discoveryService;
......@@ -61,7 +65,7 @@ public class DSLQueriesTest extends BasicTestSetup {
{"Person where (birthday >= \"1975-01-01T02:35:58.440Z\" )", 2},
{"Person where (birthday <= \"1950-01-01T02:35:58.440Z\" )", 0},
{"Person where (birthday = \"1975-01-01T02:35:58.440Z\" )", 0},
{"Person where (birthday != \"1975-01-01T02:35:58.440Z\" )", 0},
{"Person where (birthday != \"1975-01-01T02:35:58.440Z\" )", 4},
{"Person where (hasPets = true)", 2},
{"Person where (hasPets = false)", 2},
......@@ -74,7 +78,7 @@ public class DSLQueriesTest extends BasicTestSetup {
{"Person where (numberOfCars < 2)", 3},
{"Person where (numberOfCars <= 2)", 4},
{"Person where (numberOfCars = 2)", 1},
{"Person where (numberOfCars != 2)", 0},
{"Person where (numberOfCars != 2)", 3},
{"Person where (houseNumber > 0)", 2},
{"Person where (houseNumber > 17)", 1},
......@@ -82,7 +86,7 @@ public class DSLQueriesTest extends BasicTestSetup {
{"Person where (houseNumber < 153)", 3},
{"Person where (houseNumber <= 153)", 4},
{"Person where (houseNumber = 17)", 1},
{"Person where (houseNumber != 17)", 0},
{"Person where (houseNumber != 17)", 3},
{"Person where (carMileage > 0)", 2},
{"Person where (carMileage > 13)", 1},
......@@ -90,7 +94,7 @@ public class DSLQueriesTest extends BasicTestSetup {
{"Person where (carMileage < 13364)", 3},
{"Person where (carMileage <= 13364)", 4},
{"Person where (carMileage = 13)", 1},
{"Person where (carMileage != 13)", 0},
{"Person where (carMileage != 13)", 3},
{"Person where (age > 36)", 1},
{"Person where (age > 49)", 1},
......@@ -98,17 +102,17 @@ public class DSLQueriesTest extends BasicTestSetup {
{"Person where (age < 50)", 3},
{"Person where (age <= 35)", 2},
{"Person where (age = 35)", 0},
{"Person where (age != 35)", 0}
{"Person where (age != 35)", 4}
};
}
@Test(dataProvider = "comparisonQueriesProvider")
public void comparison(String query, int expected) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
assertSearchResult(searchResult, expected);
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, DEFAULT_LIMIT, 0);
assertSearchResult(searchResult, expected, query);
AtlasSearchResult searchResult2 = discoveryService.searchUsingDslQuery(query.replace("where", " "), 25, 0);
assertSearchResult(searchResult2, expected);
AtlasSearchResult searchResult2 = discoveryService.searchUsingDslQuery(query.replace("where", " "), DEFAULT_LIMIT, 0);
assertSearchResult(searchResult2, expected, query);
}
@DataProvider(name = "basicProvider")
......@@ -149,13 +153,29 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "basicProvider")
public void basic(String query, int expected) throws AtlasBaseException {
queryAssert(query, expected);
queryAssert(query.replace("where", " "), expected);
queryAssert(query, expected, DEFAULT_LIMIT, 0);
queryAssert(query.replace("where", " "), expected, DEFAULT_LIMIT, 0);
}
@DataProvider(name = "systemAttributesProvider")
private Object[][] systemAttributesQueries() {
return new Object[][]{
{"hive_db has __state", 3},
{"hive_db where hive_db has __state", 3},
{"hive_db as d where d.__state = 'ACTIVE'", 3},
{"hive_db select __guid", 3},
{"hive_db where __state = 'ACTIVE' select name, __guid, __state", 3},
};
}
@Test(dataProvider = "systemAttributesProvider")
public void systemAttributes(String query, int expected) throws AtlasBaseException {
queryAssert(query, expected, DEFAULT_LIMIT, 0);
}
private void queryAssert(String query, int expected) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
assertSearchResult(searchResult, expected);
private void queryAssert(String query, final int expected, final int limit, final int offset) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, limit, offset);
assertSearchResult(searchResult, expected, query);
}
@DataProvider(name = "limitProvider")
......@@ -172,12 +192,12 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "limitProvider")
public void limit(String query, int expected, int limit, int offset) throws AtlasBaseException {
queryAssert(query, expected);
queryAssert(query.replace("where", " "), expected);
queryAssert(query, expected, limit, offset);
queryAssert(query.replace("where", " "), expected, limit, offset);
}
@DataProvider(name = "syntaxVerifierProvider")
private Object[][] syntaxVerifierQueries() {
@DataProvider(name = "syntaxProvider")
private Object[][] syntaxQueries() {
return new Object[][]{
{"hive_column limit 10 ", 10},
{"hive_column select hive_column.qualifiedName limit 10 ", 10},
......@@ -264,10 +284,10 @@ public class DSLQueriesTest extends BasicTestSetup {
};
}
@Test(dataProvider = "syntaxVerifierProvider")
@Test(dataProvider = "syntaxProvider")
public void syntax(String query, int expected) throws AtlasBaseException {
queryAssert(query, expected);
queryAssert(query.replace("where", " "), expected);
queryAssert(query, expected, DEFAULT_LIMIT, 0);
queryAssert(query.replace("where", " "), expected, DEFAULT_LIMIT, 0);
}
@DataProvider(name = "orderByProvider")
......@@ -346,8 +366,8 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "orderByProvider")
public void orderBy(String query, int expected, String orderBy, boolean ascending) throws AtlasBaseException {
queryAssert(query, expected);
queryAssert(query.replace("where", " "), expected);
queryAssert(query, expected, DEFAULT_LIMIT, 0);
queryAssert(query.replace("where", " "), expected, DEFAULT_LIMIT, 0);
}
@DataProvider(name = "likeQueriesProvider")
......@@ -365,8 +385,8 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "likeQueriesProvider")
public void likeQueries(String query, int expected) throws AtlasBaseException {
queryAssert(query, expected);
queryAssert(query.replace("where", " "), expected);
queryAssert(query, expected, DEFAULT_LIMIT, 0);
queryAssert(query.replace("where", " "), expected, DEFAULT_LIMIT, 0);
}
@DataProvider(name = "minMaxCountProvider")
......@@ -468,16 +488,16 @@ public class DSLQueriesTest extends BasicTestSetup {
new FieldValueValidator()
.withFieldNames("'count'", "'sum'")
.withExpectedValues(4, 86) },
// { "from hive_db groupby (owner) select min(name) orderby name limit 2 ",
// new FieldValueValidator()
// .withFieldNames("min(name)")
// .withExpectedValues("Logging")
// .withExpectedValues("Reporting") },
// { "from hive_db groupby (owner) select min(name) orderby name desc limit 2 ",
// new FieldValueValidator()
// .withFieldNames("min(name)")
// .withExpectedValues("Reporting")
// .withExpectedValues("Sales") }
{ "from hive_db groupby (owner) select min(name) orderby name limit 2 ",
new FieldValueValidator()
.withFieldNames("min(name)")
.withExpectedValues("Logging")
.withExpectedValues("Reporting") },
{ "from hive_db groupby (owner) select min(name) orderby name desc limit 2 ",
new FieldValueValidator()
.withFieldNames("min(name)")
.withExpectedValues("Reporting")
.withExpectedValues("Sales") }
};
}
......@@ -490,21 +510,32 @@ public class DSLQueriesTest extends BasicTestSetup {
@DataProvider(name = "errorQueriesProvider")
private Object[][] errorQueries() {
return new Object[][]{
{"`isa`"},
{"PIII"},
{"DBBB as d select d"},
{"`isa`"}, // Tag doesn't exist in the test data
{"PIII"}, // same as above
{"DBBB as d select d"}, // same as above
{"hive_db has db"}, // same as above
{"hive_table where (name = \"sales_fact\" and createTime >= \"2014-12-11\" ) select name as _col_0, createTime as _col_1 orderby name limit 0 offset 1"},
{"hive_table as t, sd, hive_column as c where t.name=\"sales_fact\" select c.name as colName, c.dataType as colType"}
{"hive_table as t, sd, hive_column as c where t.name=\"sales_fact\" select c.name as colName, c.dataType as colType"},
{"hive_table isa hive_db"}, // isa should be a trait/classification
{"hive_table isa FooTag"}, // FooTag doesn't exist
{"hive_table groupby(db.name)"}, // GroupBy on referred attribute is not supported
{"hive_table orderby(db.name)"}, // OrderBy on referred attribute is not supported
{"hive_table select db, columns"}, // Can't select multiple referred attributes/entity
{"hive_table select min(db.name), columns"}, // Can't do aggregation on referred attribute
{"hive_table select db.name, columns"}, // Can't select more than one referred attribute
{"hive_table select owner, columns"}, // Can't select a mix of immediate attribute and referred entity
{"hive_table select owner, db.name"}, // Same as above
{"hive_order"}, // From src should be an Entity or Classification
};
}
@Test(dataProvider = "errorQueriesProvider", expectedExceptions = { AtlasBaseException.class })
public void errorQueries(String query) throws AtlasBaseException {
discoveryService.searchUsingDslQuery(query, 25, 0);
discoveryService.searchUsingDslQuery(query, DEFAULT_LIMIT, 0);
}
private void queryAssert(String query, FieldValueValidator fv) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, DEFAULT_LIMIT, 0);
assertSearchResult(searchResult, fv);
}
......@@ -521,17 +552,17 @@ public class DSLQueriesTest extends BasicTestSetup {
assertEquals(searchResult.getAttributes().getValues().size(), expected.values.size());
}
private void assertSearchResult(AtlasSearchResult searchResult, int expected) {
private void assertSearchResult(AtlasSearchResult searchResult, int expected, String query) {
assertNotNull(searchResult);
if(expected == 0) {
assertTrue(searchResult.getAttributes() == null || CollectionUtils.isEmpty(searchResult.getAttributes().getValues()));
assertNull(searchResult.getEntities());
assertNull(searchResult.getEntities(), query);
} else if(searchResult.getEntities() != null) {
assertEquals(searchResult.getEntities().size(), expected);
assertEquals(searchResult.getEntities().size(), expected, query);
} else {
assertNotNull(searchResult.getAttributes());
assertNotNull(searchResult.getAttributes().getValues());
assertEquals(searchResult.getAttributes().getValues().size(), expected);
assertEquals(searchResult.getAttributes().getValues().size(), expected, query);
}
}
......
......@@ -17,7 +17,9 @@
*/
package org.apache.atlas.query;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.query.antlr4.AtlasDSLParser;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
......@@ -26,26 +28,21 @@ import org.apache.commons.lang.StringUtils;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.List;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
public class GremlinQueryComposerTest {
private List<String> errorList = new ArrayList<>();
@Test
public void classification() {
String expected = "g.V().has('__traitNames', within('PII')).limit(25).toList()";
String expected = "g.V().has('__traitNames', within('PII')).limit(local, 25).limit(25).toList()";
verify("PII", expected);
}
@Test()
public void dimension() {
String expected = "g.V().has('__typeName', 'Table').has('__traitNames', within('Dimension')).limit(25).toList()";
String expected = "g.V().has('__typeName', 'Table').has('__traitNames', within('Dimension')).limit(local, 25).limit(25).toList()";
verify("Table isa Dimension", expected);
verify("Table is Dimension", expected);
verify("Table where Table is Dimension", expected);
......@@ -53,58 +50,59 @@ public class GremlinQueryComposerTest {
@Test
public void fromDB() {
verify("from DB", "g.V().has('__typeName', 'DB').limit(25).toList()");
verify("from DB limit 10", "g.V().has('__typeName', 'DB').limit(10).toList()");
verify("DB limit 10", "g.V().has('__typeName', 'DB').limit(10).toList()");
String expected10 = "g.V().has('__typeName', 'DB').limit(local, 10).limit(10).toList()";
verify("from DB", "g.V().has('__typeName', 'DB').limit(local, 25).limit(25).toList()");
verify("from DB limit 10", expected10);
verify("DB limit 10", expected10);
}
@Test
public void DBHasName() {
String expected = "g.V().has('__typeName', 'DB').has('DB.name').limit(25).toList()";
String expected = "g.V().has('__typeName', 'DB').has('DB.name').limit(local, 25).limit(25).toList()";
verify("DB has name", expected);
verify("DB where DB has name", expected);
}
@Test
public void DBasD() {
verify("DB as d", "g.V().has('__typeName', 'DB').as('d').limit(25).toList()");
verify("DB as d", "g.V().has('__typeName', 'DB').as('d').limit(local, 25).limit(25).toList()");
}
@Test
public void DBasDSelect() {
String expected = "def f(r){ t=[['d.name','d.owner']]; r.each({t.add([it.value('DB.name'),it.value('DB.owner')])}); t.unique(); }; " +
"f(g.V().has('__typeName', 'DB').as('d')";
verify("DB as d select d.name, d.owner", expected + ".limit(25).toList())");
verify("DB as d select d.name, d.owner limit 10", expected + ".limit(10).toList())");
verify("DB as d select d","def f(r){ r }; f(g.V().has('__typeName', 'DB').as('d').limit(25).toList())");
verify("DB as d select d.name, d.owner", expected + ".limit(local, 25).limit(25).toList())");
verify("DB as d select d.name, d.owner limit 10", expected + ".limit(local, 10).limit(10).toList())");
verify("DB as d select d","def f(r){ r }; f(g.V().has('__typeName', 'DB').as('d').limit(local, 25).limit(25).toList())");
}
@Test
public void tableSelectColumns() {
String exMain = "g.V().has('__typeName', 'Table').out('__Table.columns').limit(10).toList()";
String exMain = "g.V().has('__typeName', 'Table').out('__Table.columns').limit(local, 10).limit(10).toList()";
String exSel = "def f(r){ r }";
String exSel1 = "def f(r){ t=[['db.name']]; r.each({t.add([it.value('DB.name')])}); t.unique(); }";
verify("Table select columns limit 10", getExpected(exSel, exMain));
String exMain2 = "g.V().has('__typeName', 'Table').out('__Table.db').limit(25).toList()";
String exMain2 = "g.V().has('__typeName', 'Table').out('__Table.db').limit(local, 25).limit(25).toList()";
verify("Table select db", getExpected(exSel, exMain2));
String exMain3 = "g.V().has('__typeName', 'Table').out('__Table.db').limit(25).toList()";
String exMain3 = "g.V().has('__typeName', 'Table').out('__Table.db').limit(local, 25).limit(25).toList()";
verify("Table select db.name", getExpected(exSel1, exMain3));
}
@Test
public void valueArray() {
verify("DB where owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'DB').has('DB.owner', within('hdfs','anon')).limit(25).toList()");
verify("DB owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'DB').has('DB.owner', within('hdfs','anon')).limit(25).toList()");
verify("hive_db as d owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'hive_db').as('d').has('hive_db.owner', within('hdfs','anon')).limit(25).toList()");
verify("DB where owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'DB').has('DB.owner', within('hdfs','anon')).limit(local, 25).limit(25).toList()");
verify("DB owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'DB').has('DB.owner', within('hdfs','anon')).limit(local, 25).limit(25).toList()");
verify("hive_db as d owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'hive_db').as('d').has('hive_db.owner', within('hdfs','anon')).limit(local, 25).limit(25).toList()");
}
@Test
public void groupByMin() {
verify("from DB groupby (owner) select min(name) orderby name limit 2",
"def f(l){ t=[['min(name)']]; l.get(0).each({k,r -> L:{ def min=r.min({it.value('DB.name')}).value('DB.name'); t.add([min]); } }); t; }; " +
"f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(2).toList())");
"f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(local, 2).limit(2).toList())");
}
@Test
......@@ -113,29 +111,28 @@ public class GremlinQueryComposerTest {
"def f(l){ h=[['name','owner','clusterName']]; t=[]; " +
"l.get(0).each({k,r -> L:{ r.each({t.add([it.value('Table.name'),it.value('Table.owner'),it.value('Table.clusterName')])}) } }); " +
"h.plus(t.unique().sort{a,b -> a[0] <=> b[0]}); }; " +
"f(g.V().has('__typeName', 'Table').group().by('Table.owner').limit(25).toList())");
"f(g.V().has('__typeName', 'Table').group().by('Table.owner').limit(local, 25).limit(25).toList())");
}
@Test
public void DBAsDSelectLimit() {
verify("from DB limit 5", "g.V().has('__typeName', 'DB').limit(5).toList()");
verify("from DB limit 5 offset 2", "g.V().has('__typeName', 'DB').range(2, 2 + 5).toList()");
verify("from DB limit 5", "g.V().has('__typeName', 'DB').limit(local, 5).limit(5).toList()");
verify("from DB limit 5 offset 2", "g.V().has('__typeName', 'DB').range(local, 2, 2 + 5).range(2, 2 + 5).toList()");
}
@Test
public void DBOrderBy() {
String expected = "g.V().has('__typeName', 'DB').order().by('DB.name').limit(25).toList()";
String expected = "g.V().has('__typeName', 'DB').order().by('DB.name').limit(local, 25).limit(25).toList()";
verify("DB orderby name", expected);
verify("from DB orderby name", expected);
verify("from DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()");
verify("DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()");
verify("from DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(local, 3).limit(3).toList()");
verify("DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(local, 3).limit(3).toList()");
String exSel = "def f(r){ t=[['d.name','d.owner']]; r.each({t.add([it.value('DB.name'),it.value('DB.owner')])}); t.unique(); }";
String exMain = "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(25).toList()";
String exMain = "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(local, 25).limit(25).toList()";
verify("DB as d select d.name, d.owner orderby (d.owner) limit 25", getExpected(exSel, exMain));
String exMain2 = "g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.createTime', gt('1418265300000'))).order().by('Table.createTime').limit(25).toList()";
String exMain2 = "g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.createTime', gt('1418265300000'))).order().by('Table.createTime').limit(local, 25).limit(25).toList()";
String exSel2 = "def f(r){ t=[['_col_0','_col_1']]; r.each({t.add([it.value('Table.name'),it.value('Table.createTime')])}); t.unique(); }";
verify("Table where (name = \"sales_fact\" and createTime > \"2014-12-11T02:35:0.0Z\" ) select name as _col_0, createTime as _col_1 orderby _col_1",
getExpected(exSel2, exMain2));
......@@ -143,49 +140,49 @@ public class GremlinQueryComposerTest {
@Test
public void fromDBOrderByNameDesc() {
verify("from DB orderby name DESC", "g.V().has('__typeName', 'DB').order().by('DB.name', decr).limit(25).toList()");
verify("from DB orderby name DESC", "g.V().has('__typeName', 'DB').order().by('DB.name', decr).limit(local, 25).limit(25).toList()");
}
@Test
public void fromDBSelect() {
String expected = "def f(r){ t=[['DB.name','DB.owner']]; r.each({t.add([it.value('DB.name'),it.value('DB.owner')])}); t.unique(); }; f(g.V().has('__typeName', 'DB').limit(25).toList())";
String expected = "def f(r){ t=[['DB.name','DB.owner']]; r.each({t.add([it.value('DB.name'),it.value('DB.owner')])}); t.unique(); }; f(g.V().has('__typeName', 'DB').limit(local, 25).limit(25).toList())";
verify("from DB select DB.name, DB.owner", expected);
}
@Test
public void fromDBGroupBy() {
verify("from DB groupby (DB.owner)", "g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList()");
verify("from DB groupby (DB.owner)", "g.V().has('__typeName', 'DB').group().by('DB.owner').limit(local, 25).limit(25).toList()");
}
@Test
public void whereClauseTextContains() {
String exMain = "g.V().has('__typeName', 'DB').has('DB.name', eq(\"Reporting\")).limit(25).toList()";
String exMain = "g.V().has('__typeName', 'DB').has('DB.name', eq(\"Reporting\")).limit(local, 25).limit(25).toList()";
String exSel = "def f(r){ t=[['name','owner']]; r.each({t.add([it.value('DB.name'),it.value('DB.owner')])}); t.unique(); }";
verify("from DB where name = \"Reporting\" select name, owner", getExpected(exSel, exMain));
verify("from DB where (name = \"Reporting\") select name, owner", getExpected(exSel, exMain));
verify("Table where Asset.name like \"Tab*\"",
"g.V().has('__typeName', 'Table').has('Table.name', org.janusgraph.core.attribute.Text.textRegex(\"Tab.*\")).limit(25).toList()");
"g.V().has('__typeName', 'Table').has('Table.name', org.janusgraph.core.attribute.Text.textRegex(\"Tab.*\")).limit(local, 25).limit(25).toList()");
verify("from Table where (db.name = \"Reporting\")",
"g.V().has('__typeName', 'Table').out('__Table.db').has('DB.name', eq(\"Reporting\")).dedup().in('__Table.db').limit(25).toList()");
"g.V().has('__typeName', 'Table').out('__Table.db').has('DB.name', eq(\"Reporting\")).dedup().in('__Table.db').limit(local, 25).limit(25).toList()");
}
@Test
public void whereClauseWithAsTextContains() {
String exSel = "def f(r){ t=[['t.name','t.owner']]; r.each({t.add([it.value('Table.name'),it.value('Table.owner')])}); t.unique(); }";
String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.name', eq(\"testtable_1\")).limit(25).toList()";
String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.name', eq(\"testtable_1\")).limit(local, 25).limit(25).toList()";
verify("Table as t where t.name = \"testtable_1\" select t.name, t.owner", getExpected(exSel, exMain));
}
@Test
public void whereClauseWithDateCompare() {
String exSel = "def f(r){ t=[['t.name','t.owner']]; r.each({t.add([it.value('Table.name'),it.value('Table.owner')])}); t.unique(); }";
String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.createTime', eq('1513046158440')).limit(25).toList()";
String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.createTime', eq('1513046158440')).limit(local, 25).limit(25).toList()";
verify("Table as t where t.createTime = \"2017-12-12T02:35:58.440Z\" select t.name, t.owner", getExpected(exSel, exMain));
}
@Test
public void subType() {
String exMain = "g.V().has('__typeName', within('Asset','Table')).limit(25).toList()";
String exMain = "g.V().has('__typeName', within('Asset','Table')).limit(local, 25).limit(25).toList()";
String exSel = "def f(r){ t=[['name','owner']]; r.each({t.add([it.value('Asset.name'),it.value('Asset.owner')])}); t.unique(); }";
verify("Asset select name, owner", getExpected(exSel, exMain));
......@@ -194,24 +191,24 @@ public class GremlinQueryComposerTest {
@Test
public void countMinMax() {
verify("from DB groupby (owner) select count()",
"def f(l){ t=[['count()']]; l.get(0).each({k,r -> L:{ def count=r.size(); t.add([count]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList())");
"def f(l){ t=[['count()']]; l.get(0).each({k,r -> L:{ def count=r.size(); t.add([count]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(local, 25).limit(25).toList())");
verify("from DB groupby (owner) select max(name)",
"def f(l){ t=[['max(name)']]; l.get(0).each({k,r -> L:{ def max=r.max({it.value('DB.name')}).value('DB.name'); t.add([max]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList())");
"def f(l){ t=[['max(name)']]; l.get(0).each({k,r -> L:{ def max=r.max({it.value('DB.name')}).value('DB.name'); t.add([max]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(local, 25).limit(25).toList())");
verify("from DB groupby (owner) select min(name)",
"def f(l){ t=[['min(name)']]; l.get(0).each({k,r -> L:{ def min=r.min({it.value('DB.name')}).value('DB.name'); t.add([min]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList())");
"def f(l){ t=[['min(name)']]; l.get(0).each({k,r -> L:{ def min=r.min({it.value('DB.name')}).value('DB.name'); t.add([min]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(local, 25).limit(25).toList())");
verify("from Table select sum(createTime)",
"def f(r){ t=[['sum(createTime)']]; def sum=r.sum({it.value('Table.createTime')}); t.add([sum]); t;}; f(g.V().has('__typeName', 'Table').limit(25).toList())");
"def f(r){ t=[['sum(createTime)']]; def sum=r.sum({it.value('Table.createTime')}); t.add([sum]); t;}; f(g.V().has('__typeName', 'Table').limit(local, 25).limit(25).toList())");
}
@Test
public void traitWithSpace() {
verify("`Log Data`", "g.V().has('__typeName', 'Log Data').limit(25).toList()");
verify("`Log Data`", "g.V().has('__typeName', 'Log Data').limit(local, 25).limit(25).toList()");
}
@Test
public void whereClauseWithBooleanCondition() {
String queryFormat = "Table as t where name ='Reporting' or t.isFile = %s";
String expectedFormat = "g.V().has('__typeName', 'Table').as('t').or(__.has('Table.name', eq('Reporting')),__.has('Table.isFile', eq(%s))).limit(25).toList()";
String expectedFormat = "g.V().has('__typeName', 'Table').as('t').or(__.has('Table.name', eq('Reporting')),__.has('Table.isFile', eq(%s))).limit(local, 25).limit(25).toList()";
verify(String.format(queryFormat, "true"), String.format(expectedFormat, "true"));
verify(String.format(queryFormat, "false"), String.format(expectedFormat, "false"));
verify(String.format(queryFormat, "True"), String.format(expectedFormat, "True"));
......@@ -222,23 +219,23 @@ public class GremlinQueryComposerTest {
private Object[][] nestedQueriesSource() {
return new Object[][]{
{"Table where name=\"sales_fact\" or name=\"testtable_1\"",
"g.V().has('__typeName', 'Table').or(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(25).toList()"},
"g.V().has('__typeName', 'Table').or(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(local, 25).limit(25).toList()"},
{"Table where name=\"sales_fact\" and name=\"testtable_1\"",
"g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(25).toList()"},
"g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(local, 25).limit(25).toList()"},
{"Table where name=\"sales_fact\" or name=\"testtable_1\" or name=\"testtable_2\"",
"g.V().has('__typeName', 'Table')" +
".or(" +
"__.has('Table.name', eq(\"sales_fact\"))," +
"__.has('Table.name', eq(\"testtable_1\"))," +
"__.has('Table.name', eq(\"testtable_2\"))" +
").limit(25).toList()"},
").limit(local, 25).limit(25).toList()"},
{"Table where name=\"sales_fact\" and name=\"testtable_1\" and name=\"testtable_2\"",
"g.V().has('__typeName', 'Table')" +
".and(" +
"__.has('Table.name', eq(\"sales_fact\"))," +
"__.has('Table.name', eq(\"testtable_1\"))," +
"__.has('Table.name', eq(\"testtable_2\"))" +
").limit(25).toList()"},
").limit(local, 25).limit(25).toList()"},
{"Table where (name=\"sales_fact\" or name=\"testtable_1\") and name=\"testtable_2\"",
"g.V().has('__typeName', 'Table')" +
".and(" +
......@@ -247,7 +244,7 @@ public class GremlinQueryComposerTest {
"__.has('Table.name', eq(\"testtable_1\"))" +
")," +
"__.has('Table.name', eq(\"testtable_2\")))" +
".limit(25).toList()"},
".limit(local, 25).limit(25).toList()"},
{"Table where name=\"sales_fact\" or (name=\"testtable_1\" and name=\"testtable_2\")",
"g.V().has('__typeName', 'Table')" +
".or(" +
......@@ -256,7 +253,7 @@ public class GremlinQueryComposerTest {
"__.has('Table.name', eq(\"testtable_1\"))," +
"__.has('Table.name', eq(\"testtable_2\")))" +
")" +
".limit(25).toList()"},
".limit(local, 25).limit(25).toList()"},
{"Table where name=\"sales_fact\" or name=\"testtable_1\" and name=\"testtable_2\"",
"g.V().has('__typeName', 'Table')" +
".and(" +
......@@ -265,7 +262,7 @@ public class GremlinQueryComposerTest {
"__.has('Table.name', eq(\"testtable_1\"))" +
")," +
"__.has('Table.name', eq(\"testtable_2\")))" +
".limit(25).toList()"},
".limit(local, 25).limit(25).toList()"},
{"Table where (name=\"sales_fact\" and owner=\"Joe\") OR (name=\"sales_fact_daily_mv\" and owner=\"Joe BI\")",
"g.V().has('__typeName', 'Table')" +
".or(" +
......@@ -277,13 +274,13 @@ public class GremlinQueryComposerTest {
"__.has('Table.name', eq(\"sales_fact_daily_mv\"))," +
"__.has('Table.owner', eq(\"Joe BI\"))" +
"))" +
".limit(25).toList()"},
".limit(local, 25).limit(25).toList()"},
{"Table where owner=\"hdfs\" or ((name=\"testtable_1\" or name=\"testtable_2\") and createTime < \"2017-12-12T02:35:58.440Z\")",
"g.V().has('__typeName', 'Table').or(__.has('Table.owner', eq(\"hdfs\")),__.and(__.or(__.has('Table.name', eq(\"testtable_1\")),__.has('Table.name', eq(\"testtable_2\"))),__.has('Table.createTime', lt('1513046158440')))).limit(25).toList()"},
"g.V().has('__typeName', 'Table').or(__.has('Table.owner', eq(\"hdfs\")),__.and(__.or(__.has('Table.name', eq(\"testtable_1\")),__.has('Table.name', eq(\"testtable_2\"))),__.has('Table.createTime', lt('1513046158440')))).limit(local, 25).limit(25).toList()"},
{"hive_db where hive_db.name='Reporting' and hive_db.createTime < '2017-12-12T02:35:58.440Z'",
"g.V().has('__typeName', 'hive_db').and(__.has('hive_db.name', eq('Reporting')),__.has('hive_db.createTime', lt('1513046158440'))).limit(25).toList()"},
"g.V().has('__typeName', 'hive_db').and(__.has('hive_db.name', eq('Reporting')),__.has('hive_db.createTime', lt('1513046158440'))).limit(local, 25).limit(25).toList()"},
{"Table where db.name='Sales' and db.clusterName='cl1'",
"g.V().has('__typeName', 'Table').and(__.out('__Table.db').has('DB.name', eq('Sales')).dedup().in('__Table.db'),__.out('__Table.db').has('DB.clusterName', eq('cl1')).dedup().in('__Table.db')).limit(25).toList()"},
"g.V().has('__typeName', 'Table').and(__.out('__Table.db').has('DB.name', eq('Sales')).dedup().in('__Table.db'),__.out('__Table.db').has('DB.clusterName', eq('cl1')).dedup().in('__Table.db')).limit(local, 25).limit(25).toList()"},
};
}
......@@ -296,25 +293,57 @@ public class GremlinQueryComposerTest {
@Test
public void keywordsInWhereClause() {
verify("Table as t where t has name and t isa Dimension",
"g.V().has('__typeName', 'Table').as('t').and(__.has('Table.name'),__.has('__traitNames', within('Dimension'))).limit(25).toList()");
"g.V().has('__typeName', 'Table').as('t').and(__.has('Table.name'),__.has('__traitNames', within('Dimension'))).limit(local, 25).limit(25).toList()");
verify("Table as t where t has name and t.name = 'sales_fact'",
"g.V().has('__typeName', 'Table').as('t').and(__.has('Table.name'),__.has('Table.name', eq('sales_fact'))).limit(25).toList()");
"g.V().has('__typeName', 'Table').as('t').and(__.has('Table.name'),__.has('Table.name', eq('sales_fact'))).limit(local, 25).limit(25).toList()");
verify("Table as t where t is Dimension and t.name = 'sales_fact'",
"g.V().has('__typeName', 'Table').as('t').and(__.has('__traitNames', within('Dimension')),__.has('Table.name', eq('sales_fact'))).limit(25).toList()");
verify("Table isa 'Dimension' and t.name = 'sales_fact'", "g.V().has('__typeName', 'Table').has('__traitNames', within(''Dimension'')).limit(25).toList()");
"g.V().has('__typeName', 'Table').as('t').and(__.has('__traitNames', within('Dimension')),__.has('Table.name', eq('sales_fact'))).limit(local, 25).limit(25).toList()");
verify("Table isa 'Dimension' and name = 'sales_fact'", "g.V().has('__typeName', 'Table').and(__.has('__traitNames', within('Dimension')),__.has('Table.name', eq('sales_fact'))).limit(local, 25).limit(25).toList()");
verify("Table has name and name = 'sales_fact'", "g.V().has('__typeName', 'Table').and(__.has('Table.name'),__.has('Table.name', eq('sales_fact'))).limit(local, 25).limit(25).toList()");
verify("Table is 'Dimension' and Table has owner and name = 'sales_fact'", "g.V().has('__typeName', 'Table').and(__.has('__traitNames', within('Dimension')),__.has('Table.owner'),__.has('Table.name', eq('sales_fact'))).limit(local, 25).limit(25).toList()");
verify("Table has name and Table has owner and name = 'sales_fact'", "g.V().has('__typeName', 'Table').and(__.has('Table.name'),__.has('Table.owner'),__.has('Table.name', eq('sales_fact'))).limit(local, 25).limit(25).toList()");
}
@Test
public void systemAttributes() {
verify("Table has __state", "g.V().has('__typeName', 'Table').has('__state').limit(local, 25).limit(25).toList()");
verify("Table select __guid", "def f(r){ t=[['__guid']]; r.each({t.add([it.value('__guid')])}); t.unique(); }; f(g.V().has('__typeName', 'Table').limit(local, 25).limit(25).toList())");
verify("Table as t where t.__state = 'ACTIVE'", "g.V().has('__typeName', 'Table').as('t').has('__state', eq('ACTIVE')).limit(local, 25).limit(25).toList()");
}
@Test
public void invalidQueries() {
verify("hdfs_path like h1", "");
// verify("hdfs_path select xxx", "");
}
private void verify(String dsl, String expectedGremlin) {
@Test
public void invalidQueries2() {
verify("PIII", 3);
verify("TableTTT where name = 'abcd'", 1);
verify("Table isa Table", 1);
verify("Table has xxx", 1);
verify("Table where createType = '2010-03-30'", 1);
verify("Table has db", 1);
verify("Table groupby(db) select name", 1);
verify("Table groupby(name) select name, max(db)", 1);
verify("Table select db, columns", 2);
verify("Table select db, owner, columns", 3);
}
private void verify(String dsl, String expectedGremlin, int expectedNumberOfErrors) {
AtlasDSLParser.QueryContext queryContext = getParsedQuery(dsl);
String actualGremlin = getGremlinQuery(queryContext);
assertEquals(actualGremlin, expectedGremlin);
String actualGremlin = getGremlinQuery(dsl, queryContext, expectedNumberOfErrors);
if(expectedNumberOfErrors == 0) {
assertEquals(actualGremlin, expectedGremlin, dsl);
}
}
private void verify(String dsl, int expectedNumberOfErrors) {
verify(dsl, "", expectedNumberOfErrors);
}
private void verify(String dsl, String expectedGremlin) {
verify(dsl, expectedGremlin, 0);
}
private String getExpected(String select, String main) {
......@@ -332,9 +361,9 @@ public class GremlinQueryComposerTest {
return queryContext;
}
private String getGremlinQuery(AtlasDSLParser.QueryContext queryContext) {
private String getGremlinQuery(String dsl, AtlasDSLParser.QueryContext queryContext, int expectedNumberOfErrors) {
AtlasTypeRegistry registry = mock(AtlasTypeRegistry.class);
org.apache.atlas.query.Lookup lookup = new TestLookup(errorList, registry);
org.apache.atlas.query.Lookup lookup = new TestLookup(registry);
GremlinQueryComposer.Context context = new GremlinQueryComposer.Context(lookup);
AtlasDSL.QueryMetadata queryMetadata = new AtlasDSL.QueryMetadata(queryContext);
......@@ -343,35 +372,42 @@ public class GremlinQueryComposerTest {
qv.visit(queryContext);
String s = gremlinQueryComposer.get();
assertEquals(gremlinQueryComposer.getErrorList().size(), 0);
int actualNumberOfErrors = gremlinQueryComposer.getErrorList().size();
assertEquals(actualNumberOfErrors, expectedNumberOfErrors, dsl);
return s;
}
private static class TestLookup implements org.apache.atlas.query.Lookup {
List<String> errorList;
AtlasTypeRegistry registry;
public TestLookup(List<String> errorList, AtlasTypeRegistry typeRegistry) {
this.errorList = errorList;
public TestLookup(AtlasTypeRegistry typeRegistry) {
this.registry = typeRegistry;
}
@Override
public AtlasType getType(String typeName) {
AtlasType type = null;
public AtlasType getType(String typeName) throws AtlasBaseException {
AtlasType type;
if(typeName.equals("PII") || typeName.equals("Dimension")) {
type = mock(AtlasType.class);
when(type.getTypeCategory()).thenReturn(TypeCategory.CLASSIFICATION);
} else {
type = mock(AtlasEntityType.class);
when(type.getTypeCategory()).thenReturn(TypeCategory.ENTITY);
}
if(typeName.equals("PIII")) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND);
}
when(type.getTypeName()).thenReturn(typeName);
return type;
}
@Override
public String getQualifiedName(GremlinQueryComposer.Context context, String name) throws AtlasBaseException {
if(name.startsWith("__")) {
return name.equals("__state") || name.equals("__guid") ? name : "";
}
if(!hasAttribute(context, name)) {
throw new AtlasBaseException("Invalid attribute");
}
......@@ -390,7 +426,9 @@ public class GremlinQueryComposerTest {
return attributeName.equals("name") ||
attributeName.equals("owner") ||
attributeName.equals("createTime") ||
attributeName.equals("clusterName");
attributeName.equals("clusterName") ||
attributeName.equals("__guid") ||
attributeName.equals("__state");
}
@Override
......@@ -412,6 +450,8 @@ public class GremlinQueryComposerTest {
(context.getActiveTypeName().equals("Table") && attributeName.equals("owner")) ||
(context.getActiveTypeName().equals("Table") && attributeName.equals("clusterName")) ||
(context.getActiveTypeName().equals("Table") && attributeName.equals("isFile")) ||
(context.getActiveTypeName().equals("Table") && attributeName.equals("__guid")) ||
(context.getActiveTypeName().equals("Table") && attributeName.equals("__state")) ||
(context.getActiveTypeName().equals("hive_db") && attributeName.equals("name")) ||
(context.getActiveTypeName().equals("hive_db") && attributeName.equals("owner")) ||
(context.getActiveTypeName().equals("hive_db") && attributeName.equals("createTime")) ||
......@@ -434,8 +474,8 @@ public class GremlinQueryComposerTest {
}
@Override
public boolean isTraitType(GremlinQueryComposer.Context context) {
return context.getActiveTypeName().equals("PII") || context.getActiveTypeName().equals("Dimension");
public boolean isTraitType(String typeName) {
return typeName.equals("PII") || typeName.equals("Dimension");
}
@Override
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment