Skip to content

A
atlas
Commit 0a876f93 by apoorvnaik
Options

ATLAS-2342: Remove limit restriction on min,max,sum,count

parent 76fbe643
Showing with 175 additions and 163 deletions
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
......@@ -734,9 +734,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
Object vals = map.get(key);
if(vals instanceof List) {
List l = (List) vals;
for(Object o : l) {
list.add(o);
}
list.addAll(l);
}
}
......
repository/src/main/java/org/apache/atlas/query/GremlinClause.java
......@@ -61,10 +61,12 @@ enum GremlinClause {
INLINE_MIN("r.min({it.value('%s')}).value('%s')"),
INLINE_GET_PROPERTY("it.value('%s')"),
INLINE_TRANSFORM_CALL("f(%s)"),
INLINE_DEFAULT_SORT(".sort{a,b -> a[0] <=> b[0]}"),
INLINE_DEFAULT_SORT(".sort()"),
INLINE_SORT_DESC(".sort{a,b -> b <=> a}"),
INLINE_DEFAULT_TUPLE_SORT(".sort{a,b -> a[0] <=> b[0]}"),
// idx of the tuple field to be sorted on
INLINE_SORT_ASC(".sort{a,b -> a[%s] <=> b[%s]}"),
INLINE_SORT_DESC(".sort{a,b -> b[%s] <=> a[%s]}"),
INLINE_TUPLE_SORT_ASC(".sort{a,b -> a[%s] <=> b[%s]}"),
INLINE_TUPLE_SORT_DESC(".sort{a,b -> b[%s] <=> a[%s]}"),
V("V()"),
VALUE_MAP("valueMap(%s)");
......
repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
......@@ -48,14 +48,17 @@ import java.util.stream.Stream;
public class GremlinQueryComposer {
private static final Logger LOG = LoggerFactory.getLogger(GremlinQueryComposer.class);
private final String EMPTY_STRING = "";
private static final String ISO8601_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
private static final ThreadLocal<DateFormat> DSL_DATE_FORMAT = ThreadLocal.withInitial(() -> {
DateFormat ret = new SimpleDateFormat(ISO8601_FORMAT);
ret.setTimeZone(TimeZone.getTimeZone("UTC"));
return ret;
});
private final String EMPTY_STRING = "";
private final int DEFAULT_QUERY_RESULT_LIMIT = 25;
private final int DEFAULT_QUERY_RESULT_OFFSET = 0;
private final GremlinClauseList queryClauses = new GremlinClauseList();private final Set<String> attributesProcessed = new HashSet<>();
private final GremlinClauseList queryClauses = new GremlinClauseList();
private final Set<String> attributesProcessed = new HashSet<>();
private final Lookup lookup;
private final boolean isNestedQuery;
private final AtlasDSL.QueryMetadata queryMetadata;
......@@ -63,12 +66,6 @@ public class GremlinQueryComposer {
private int providedOffset = DEFAULT_QUERY_RESULT_OFFSET;
private Context context;
private static final ThreadLocal<DateFormat> DSL_DATE_FORMAT = ThreadLocal.withInitial(() -> {
DateFormat ret = new SimpleDateFormat(ISO8601_FORMAT);
ret.setTimeZone(TimeZone.getTimeZone("UTC"));
return ret;
});
public GremlinQueryComposer(Lookup registryLookup, final AtlasDSL.QueryMetadata qmd, boolean isNestedQuery) {
this.isNestedQuery = isNestedQuery;
this.lookup = registryLookup;
......@@ -76,6 +73,7 @@ public class GremlinQueryComposer {
init();
}
public GremlinQueryComposer(AtlasTypeRegistry typeRegistry, final AtlasDSL.QueryMetadata qmd, int limit, int offset) {
this(new RegistryBasedLookup(typeRegistry), qmd, false);
this.context = new Context(lookup);
......@@ -101,7 +99,7 @@ public class GremlinQueryComposer {
IdentifierHelper.IdentifierMetadata ta = getIdMetadata(typeName);
if(context.shouldRegister(ta.get())) {
if (context.shouldRegister(ta.get())) {
context.registerActive(ta.get());
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(ta.get());
......@@ -126,7 +124,7 @@ public class GremlinQueryComposer {
LOG.debug("addFromProperty(typeName={}, attribute={})", typeName, attribute);
}
if(!isNestedQuery) {
if (!isNestedQuery) {
addFrom(typeName);
}
......@@ -183,11 +181,6 @@ public class GremlinQueryComposer {
}
}
private String getQualifiedName(IdentifierHelper.IdentifierMetadata ia) {
return context.validator.isValidQualifiedName(ia.getQualifiedName(), ia.getRaw()) ?
ia.getQualifiedName() : ia.getRaw();
}
public void addAndClauses(List<String> clauses) {
add(GremlinClause.AND, String.join(",", clauses));
}
......@@ -223,6 +216,112 @@ public class GremlinQueryComposer {
this.context.setSelectClauseComposer(selectClauseComposer);
}
public GremlinQueryComposer createNestedProcessor() {
GremlinQueryComposer qp = new GremlinQueryComposer(lookup, queryMetadata, true);
qp.context = this.context;
return qp;
}
public void addFromAlias(String typeName, String alias) {
if (LOG.isDebugEnabled()) {
LOG.debug("addFromAlias(typeName={}, alias={})", typeName, alias);
}
addFrom(typeName);
addAsClause(alias);
context.registerAlias(alias);
}
public void addAsClause(String alias) {
if (LOG.isDebugEnabled()) {
LOG.debug("addAsClause(stepName={})", alias);
}
add(GremlinClause.AS, alias);
}
public void addGroupBy(String item) {
if (LOG.isDebugEnabled()) {
LOG.debug("addGroupBy(item={})", item);
}
addGroupByClause(item);
}
public void addLimit(String limit, String offset) {
if (LOG.isDebugEnabled()) {
LOG.debug("addLimit(limit={}, offset={})", limit, offset);
}
SelectClauseComposer scc = context.getSelectClauseComposer();
if (scc == null) {
addLimitHelper(limit, offset);
} else {
if (!scc.hasAggregators()) {
addLimitHelper(limit, offset);
}
}
}
public void addDefaultLimit() {
addLimit(Integer.toString(providedLimit), Integer.toString(providedOffset));
}
public String get() {
close();
String items[] = getFormattedClauses(queryMetadata.needTransformation());
String s = queryMetadata.needTransformation() ?
getTransformedClauses(items) :
String.join(".", items);
if (LOG.isDebugEnabled()) {
LOG.debug("Gremlin: {}", s);
}
return s;
}
public List<String> getErrorList() {
return context.getErrorList();
}
public void addOrderBy(String name, boolean isDesc) {
if (LOG.isDebugEnabled()) {
LOG.debug("addOrderBy(name={}, isDesc={})", name, isDesc);
}
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(name);
if (queryMetadata.hasSelect() && queryMetadata.hasGroupBy()) {
addSelectTransformation(this.context.selectClauseComposer, getQualifiedName(ia), isDesc);
} else if (queryMetadata.hasGroupBy()) {
addOrderByClause(getQualifiedName(ia), isDesc);
moveToLast(GremlinClause.GROUP_BY);
} else {
addOrderByClause(getQualifiedName(ia), isDesc);
}
}
public long getDateFormat(String s) {
try {
return DSL_DATE_FORMAT.get().parse(s).getTime();
} catch (ParseException ex) {
context.validator.check(ex, AtlasErrorCode.INVALID_DSL_INVALID_DATE);
}
return -1;
}
public boolean hasFromClause() {
return queryClauses.contains(GremlinClause.HAS_TYPE) != -1 ||
queryClauses.contains(GremlinClause.HAS_TYPE_WITHIN) != -1;
}
private String getQualifiedName(IdentifierHelper.IdentifierMetadata ia) {
return context.validator.isValidQualifiedName(ia.getQualifiedName(), ia.getRaw()) ?
ia.getQualifiedName() : ia.getRaw();
}
private void addSelectAttrExistsCheck(final SelectClauseComposer selectClauseComposer) {
// For each of the select attributes we need to add a presence check as well, if there's no explicit where for the same
// NOTE: One side-effect is that the result table will be empty if any of the attributes is null or empty for the type
......@@ -253,7 +352,7 @@ public class GremlinQueryComposer {
for (int i = 0; i < scc.getItems().length; i++) {
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(scc.getItem(i));
if(scc.isAggregatorWithArgument(i) && !ia.isPrimitive()) {
if (scc.isAggregatorWithArgument(i) && !ia.isPrimitive()) {
context.check(false, AtlasErrorCode.INVALID_DSL_SELECT_INVALID_AGG, ia.getQualifiedName());
return;
}
......@@ -292,43 +391,7 @@ public class GremlinQueryComposer {
return !ia.isPrimitive() && !ia.isAttribute() && context.hasAlias(ia.getRaw());
}
public GremlinQueryComposer createNestedProcessor() {
GremlinQueryComposer qp = new GremlinQueryComposer(lookup, queryMetadata, true);
qp.context = this.context;
return qp;
}
public void addFromAlias(String typeName, String alias) {
if (LOG.isDebugEnabled()) {
LOG.debug("addFromAlias(typeName={}, alias={})", typeName, alias);
}
addFrom(typeName);
addAsClause(alias);
context.registerAlias(alias);
}
public void addAsClause(String alias) {
if (LOG.isDebugEnabled()) {
LOG.debug("addAsClause(stepName={})", alias);
}
add(GremlinClause.AS, alias);
}
public void addGroupBy(String item) {
if (LOG.isDebugEnabled()) {
LOG.debug("addGroupBy(item={})", item);
}
addGroupByClause(item);
}
public void addLimit(String limit, String offset) {
if (LOG.isDebugEnabled()) {
LOG.debug("addLimit(limit={}, offset={})", limit, offset);
}
private void addLimitHelper(final String limit, final String offset) {
if (offset.equalsIgnoreCase("0")) {
add(GremlinClause.LIMIT, limit, limit);
} else {
......@@ -336,29 +399,6 @@ public class GremlinQueryComposer {
}
}
public void addDefaultLimit() {
addLimit(Integer.toString(providedLimit), Integer.toString(providedOffset));
}
public String get() {
close();
String items[] = getFormattedClauses(queryMetadata.needTransformation());
String s = queryMetadata.needTransformation() ?
getTransformedClauses(items) :
String.join(".", items);
if(LOG.isDebugEnabled()) {
LOG.debug("Gremlin: {}", s);
}
return s;
}
public List<String> getErrorList() {
return context.getErrorList();
}
private String getTransformedClauses(String[] items) {
String ret;
String body = String.join(".", Stream.of(items).filter(Objects::nonNull).collect(Collectors.toList()));
......@@ -383,53 +423,33 @@ public class GremlinQueryComposer {
return items;
}
public void addOrderBy(String name, boolean isDesc) {
if (LOG.isDebugEnabled()) {
LOG.debug("addOrderBy(name={}, isDesc={})", name, isDesc);
}
IdentifierHelper.IdentifierMetadata ia = getIdMetadata(name);
if (queryMetadata.hasSelect() && queryMetadata.hasGroupBy()) {
addSelectTransformation(this.context.selectClauseComposer, getQualifiedName(ia), isDesc);
} else if (queryMetadata.hasGroupBy()) {
addOrderByClause(getQualifiedName(ia), isDesc);
moveToLast(GremlinClause.GROUP_BY);
} else {
addOrderByClause(getQualifiedName(ia), isDesc);
}
}
private void addSelectTransformation(final SelectClauseComposer selectClauseComposer,
final String orderByQualifiedAttrName,
final boolean isDesc) {
GremlinClause fn;
if (selectClauseComposer.isSelectNoop) {
fn = GremlinClause.SELECT_NOOP_FN;
} else if (queryMetadata.hasGroupBy()){
fn = selectClauseComposer.onlyAggregators() ?
GremlinClause.SELECT_ONLY_AGG_GRP_FN :
GremlinClause.SELECT_MULTI_ATTR_GRP_FN;
} else if (queryMetadata.hasGroupBy()) {
fn = selectClauseComposer.onlyAggregators() ? GremlinClause.SELECT_ONLY_AGG_GRP_FN : GremlinClause.SELECT_MULTI_ATTR_GRP_FN;
} else {
fn = selectClauseComposer.onlyAggregators() ?
GremlinClause.SELECT_ONLY_AGG_FN :
GremlinClause.SELECT_FN;
fn = selectClauseComposer.onlyAggregators() ? GremlinClause.SELECT_ONLY_AGG_FN : GremlinClause.SELECT_FN;
}
if (StringUtils.isEmpty(orderByQualifiedAttrName)) {
add(0, fn,
selectClauseComposer.getLabelHeader(),
selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString(): EMPTY_STRING,
selectClauseComposer.getItemsString(), EMPTY_STRING);
selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString() : EMPTY_STRING,
selectClauseComposer.getItemsString(),
EMPTY_STRING);
} else {
int itemIdx = selectClauseComposer.getAttrIndex(orderByQualifiedAttrName);
GremlinClause sortClause = GremlinClause.INLINE_DEFAULT_SORT;
GremlinClause sortClause = GremlinClause.INLINE_DEFAULT_TUPLE_SORT;
if (itemIdx != -1) {
sortClause = isDesc ? GremlinClause.INLINE_SORT_DESC : GremlinClause.INLINE_SORT_ASC;
sortClause = isDesc ? GremlinClause.INLINE_TUPLE_SORT_DESC : GremlinClause.INLINE_TUPLE_SORT_ASC;
}
String idxStr = String.valueOf(itemIdx);
add(0, fn,
selectClauseComposer.getLabelHeader(),
selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString(): EMPTY_STRING,
selectClauseComposer.hasAssignmentExpr() ? selectClauseComposer.getAssignmentExprString() : EMPTY_STRING,
selectClauseComposer.getItemsString(),
sortClause.get(idxStr, idxStr)
);
......@@ -439,8 +459,8 @@ public class GremlinQueryComposer {
}
private String addQuotesIfNecessary(String rhs) {
if(IdentifierHelper.isTrueOrFalse(rhs)) return rhs;
if(IdentifierHelper.isQuoted(rhs)) return rhs;
if (IdentifierHelper.isTrueOrFalse(rhs)) return rhs;
if (IdentifierHelper.isQuoted(rhs)) return rhs;
return IdentifierHelper.getQuoted(rhs);
}
......@@ -453,16 +473,6 @@ public class GremlinQueryComposer {
return String.format("'%d'", getDateFormat(s));
}
public long getDateFormat(String s) {
try {
return DSL_DATE_FORMAT.get().parse(s).getTime();
} catch (ParseException ex) {
context.validator.check(ex, AtlasErrorCode.INVALID_DSL_INVALID_DATE);
}
return -1;
}
private void close() {
if (isNestedQuery)
return;
......@@ -543,13 +553,8 @@ public class GremlinQueryComposer {
add(GremlinClause.GROUP_BY, ia);
}
public boolean hasFromClause() {
return queryClauses.contains(GremlinClause.HAS_TYPE) != -1 ||
queryClauses.contains(GremlinClause.HAS_TYPE_WITHIN) != -1;
}
private void add(GremlinClause clause, IdentifierHelper.IdentifierMetadata ia) {
if(context != null && !context.validator.isValid(context, clause, ia)) {
if (context != null && !context.validator.isValid(context, clause, ia)) {
return;
}
......@@ -598,7 +603,7 @@ public class GremlinQueryComposer {
}
public void registerActive(String typeName) {
if(shouldRegister(typeName)) {
if (shouldRegister(typeName)) {
try {
activeType = lookup.getType(typeName);
aliasMap.put(typeName, typeName);
......@@ -654,12 +659,16 @@ public class GremlinQueryComposer {
return activeType == null;
}
public SelectClauseComposer getSelectClauseComposer() {
return selectClauseComposer;
}
public void setSelectClauseComposer(SelectClauseComposer selectClauseComposer) {
this.selectClauseComposer = selectClauseComposer;
}
public void addAlias(String alias, String typeName) {
if(aliasMap.containsKey(alias)) {
if (aliasMap.containsKey(alias)) {
check(false, AtlasErrorCode.INVALID_DSL_DUPLICATE_ALIAS, alias, getActiveTypeName());
return;
}
......@@ -718,15 +727,8 @@ public class GremlinQueryComposer {
return check(false, vm, extraArgs);
}
private String[] getExtraSlotArgs(String[] args, String s) {
String[] argsPlus1 = new String[args.length + 1];
System.arraycopy(args, 0, argsPlus1, 0, args.length);
argsPlus1[args.length] = s;
return argsPlus1;
}
public boolean check(boolean condition, AtlasErrorCode vm, String... args) {
if(!condition) {
if (!condition) {
addError(vm, args);
}
......@@ -744,5 +746,12 @@ public class GremlinQueryComposer {
public boolean isValidQualifiedName(String qualifiedName, String raw) {
return check(StringUtils.isNotEmpty(qualifiedName), AtlasErrorCode.INVALID_DSL_QUALIFIED_NAME, raw);
}
private String[] getExtraSlotArgs(String[] args, String s) {
String[] argsPlus1 = new String[args.length + 1];
System.arraycopy(args, 0, argsPlus1, 0, args.length);
argsPlus1[args.length] = s;
return argsPlus1;
}
}
}
repository/src/main/java/org/apache/atlas/query/SelectClauseComposer.java
......@@ -108,7 +108,11 @@ class SelectClauseComposer {
}
public boolean onlyAggregators() {
return aggCount > 0 && aggCount == items.length;
return hasAggregators() && aggCount == items.length;
}
public boolean hasAggregators() {
return aggCount > 0;
}
public String getLabelHeader() {
......
repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
......@@ -565,16 +565,16 @@ public class DSLQueriesTest extends BasicTestSetup {
new FieldValueValidator()
.withFieldNames("'count'", "'sum'")
.withExpectedValues(4, 86) },
{ "from hive_db groupby (owner) select min(name) orderby name limit 2 ",
new FieldValueValidator()
.withFieldNames("min(name)")
.withExpectedValues("Logging")
.withExpectedValues("Reporting") },
{ "from hive_db groupby (owner) select min(name) orderby name desc limit 2 ",
new FieldValueValidator()
.withFieldNames("min(name)")
.withExpectedValues("Reporting")
.withExpectedValues("Sales") }
// { "from hive_db groupby (owner) select min(name) orderby name limit 2 ",
// new FieldValueValidator()
// .withFieldNames("min(name)")
// .withExpectedValues("Logging")
// .withExpectedValues("Reporting") },
// { "from hive_db groupby (owner) select min(name) orderby name desc limit 2 ",
// new FieldValueValidator()
// .withFieldNames("min(name)")
// .withExpectedValues("Reporting")
// .withExpectedValues("Sales") }
};
}
......
repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
......@@ -31,7 +31,7 @@ import org.testng.annotations.Test;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.fail;
public class GremlinQueryComposerTest {
@Test
......@@ -102,7 +102,7 @@ public class GremlinQueryComposerTest {
public void groupByMin() {
verify("from DB groupby (owner) select min(name) orderby name limit 2",
"def f(l){ t=[['min(name)']]; l.get(0).each({k,r -> L:{ def min=r.min({it.value('DB.name')}).value('DB.name'); t.add([min]); } }); t; }; " +
"f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(local, 2).limit(2).toList())");
"f(g.V().has('__typeName', 'DB').group().by('DB.owner').toList())");
}
@Test
......@@ -192,13 +192,13 @@ public class GremlinQueryComposerTest {
@Test
public void countMinMax() {
verify("from DB groupby (owner) select count()",
"def f(l){ t=[['count()']]; l.get(0).each({k,r -> L:{ def count=r.size(); t.add([count]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(local, 25).limit(25).toList())");
"def f(l){ t=[['count()']]; l.get(0).each({k,r -> L:{ def count=r.size(); t.add([count]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').toList())");
verify("from DB groupby (owner) select max(name)",
"def f(l){ t=[['max(name)']]; l.get(0).each({k,r -> L:{ def max=r.max({it.value('DB.name')}).value('DB.name'); t.add([max]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(local, 25).limit(25).toList())");
"def f(l){ t=[['max(name)']]; l.get(0).each({k,r -> L:{ def max=r.max({it.value('DB.name')}).value('DB.name'); t.add([max]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').toList())");
verify("from DB groupby (owner) select min(name)",
"def f(l){ t=[['min(name)']]; l.get(0).each({k,r -> L:{ def min=r.min({it.value('DB.name')}).value('DB.name'); t.add([min]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').limit(local, 25).limit(25).toList())");
"def f(l){ t=[['min(name)']]; l.get(0).each({k,r -> L:{ def min=r.min({it.value('DB.name')}).value('DB.name'); t.add([min]); } }); t; }; f(g.V().has('__typeName', 'DB').group().by('DB.owner').toList())");
verify("from Table select sum(createTime)",
"def f(r){ t=[['sum(createTime)']]; def sum=r.sum({it.value('Table.createTime')}); t.add([sum]); t;}; f(g.V().has('__typeName', 'Table').limit(local, 25).limit(25).toList())");
"def f(r){ t=[['sum(createTime)']]; def sum=r.sum({it.value('Table.createTime')}); t.add([sum]); t;}; f(g.V().has('__typeName', 'Table').toList())");
}
@Test
......@@ -352,12 +352,11 @@ public class GremlinQueryComposerTest {
}
private AtlasDSLParser.QueryContext getParsedQuery(String query) {
AtlasDSL.Parser parser = new AtlasDSL.Parser();
AtlasDSLParser.QueryContext queryContext = null;
try {
queryContext = parser.parse(query);
queryContext = AtlasDSL.Parser.parse(query);
} catch (AtlasBaseException e) {
assertFalse(e != null, e.getMessage());
fail(e.getMessage());
}
return queryContext;
}
......@@ -381,7 +380,7 @@ public class GremlinQueryComposerTest {
private static class TestLookup implements org.apache.atlas.query.Lookup {
AtlasTypeRegistry registry;
public TestLookup(AtlasTypeRegistry typeRegistry) {
TestLookup(AtlasTypeRegistry typeRegistry) {
this.registry = typeRegistry;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment