Skip to content

A
atlas
Commit 5384a742 by Madhan Neethiraj
Options

ATLAS-2229: fix for unit test failures in DSL tests; also fixed incorrect…

ATLAS-2229: fix for unit test failures in DSL tests; also fixed incorrect handling of orderby/groupby when limit/offset is present
parent 2a2b07e9
Showing with 63 additions and 46 deletions
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
...@@ -344,7 +344,7 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE ...@@ -344,7 +344,7 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
return result; return result;
} catch (ScriptException e) { } catch (ScriptException e) {
throw new AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, gremlinQuery); throw new AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, e, gremlinQuery);
} finally { } finally {
releaseGremlinScriptEngine(scriptEngine); releaseGremlinScriptEngine(scriptEngine);
} }
......
repository/src/main/java/org/apache/atlas/query/QueryProcessor.java
...@@ -281,8 +281,7 @@ public class QueryProcessor { ...@@ -281,8 +281,7 @@ public class QueryProcessor {
LOG.debug("addGroupBy(item={})", item); LOG.debug("addGroupBy(item={})", item);
} }
add(GremlinClause.GROUP); addGroupByClause(item);
addByClause(item, false);
hasGrpBy = true; hasGrpBy = true;
} }
...@@ -347,8 +346,7 @@ public class QueryProcessor { ...@@ -347,8 +346,7 @@ public class QueryProcessor {
LOG.debug("addOrderBy(name={}, isDesc={})", name, isDesc); LOG.debug("addOrderBy(name={}, isDesc={})", name, isDesc);
} }
add(GremlinClause.ORDER); addOrderByClause(name, isDesc);
addByClause(name, isDesc);
} }
private void updatePosition(GremlinClause clause) { private void updatePosition(GremlinClause clause) {
...@@ -407,22 +405,29 @@ public class QueryProcessor { ...@@ -407,22 +405,29 @@ public class QueryProcessor {
} }
} }
private void addByClause(String name, boolean descr) { private void addOrderByClause(String name, boolean descr) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("addByClause(name={})", name, descr); LOG.debug("addOrderByClause(name={})", name, descr);
} }
IdentifierHelper.Advice ia = getAdvice(name); IdentifierHelper.Advice ia = getAdvice(name);
add((!descr) ? GremlinClause.BY : GremlinClause.BY_DESC, ia.getQualifiedName()); add((!descr) ? GremlinClause.ORDER_BY : GremlinClause.ORDER_BY_DESC, ia.getQualifiedName());
}
private void addGroupByClause(String name) {
if (LOG.isDebugEnabled()) {
LOG.debug("addGroupByClause(name={})", name);
}
IdentifierHelper.Advice ia = getAdvice(name);
add(GremlinClause.GROUP_BY, ia.getQualifiedName());
} }
private enum GremlinClause { private enum GremlinClause {
AS("as('%s')"), AS("as('%s')"),
BY("by('%s')"),
BY_DESC("by('%s', decr)"),
DEDUP("dedup()"), DEDUP("dedup()"),
G("g"), G("g"),
GROUP("group()"), GROUP_BY("group().by('%')"),
HAS("has('%s', %s)"), HAS("has('%s', %s)"),
HAS_OPERATOR("has('%s', %s(%s))"), HAS_OPERATOR("has('%s', %s(%s))"),
HAS_PROPERTY("has('%s')"), HAS_PROPERTY("has('%s')"),
...@@ -436,7 +441,8 @@ public class QueryProcessor { ...@@ -436,7 +441,8 @@ public class QueryProcessor {
NESTED_START("__"), NESTED_START("__"),
NESTED_HAS_OPERATOR("has('%s', %s(%s))"), NESTED_HAS_OPERATOR("has('%s', %s(%s))"),
LIMIT("limit(%s)"), LIMIT("limit(%s)"),
ORDER("order()"), ORDER_BY("order().by('%s')"),
ORDER_BY_DESC("order().by('%s', decr)"),
OUT("out('%s')"), OUT("out('%s')"),
RANGE("range(%s, %s + %s)"), RANGE("range(%s, %s + %s)"),
SELECT("select('%s')"), SELECT("select('%s')"),
......
repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
...@@ -148,6 +148,7 @@ public abstract class BasicTestSetup { ...@@ -148,6 +148,7 @@ public abstract class BasicTestSetup {
entities.addAll(salesFactColumns); entities.addAll(salesFactColumns);
AtlasEntity salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact"); AtlasEntity salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
salesFact.setAttribute("createTime", new Date(2018, 01, 01));
entities.add(salesFact); entities.add(salesFact);
List<AtlasEntity> logFactColumns = ImmutableList List<AtlasEntity> logFactColumns = ImmutableList
...@@ -179,7 +180,7 @@ public abstract class BasicTestSetup { ...@@ -179,7 +180,7 @@ public abstract class BasicTestSetup {
AtlasEntity circularLineageTable1 = table("table1", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric"); AtlasEntity circularLineageTable1 = table("table1", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric");
entities.add(circularLineageTable1); entities.add(circularLineageTable1);
AtlasEntity circularLineageTable2 = table("table2", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric"); AtlasEntity circularLineageTable2 = table("table2", "", reportingDB, sd, "Vimal 2", "Managed", salesFactColumns, "Metric");
entities.add(circularLineageTable2); entities.add(circularLineageTable2);
AtlasEntity circularLineage1Process = loadProcess("circularLineage1", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable1), AtlasEntity circularLineage1Process = loadProcess("circularLineage1", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable1),
...@@ -209,7 +210,7 @@ public abstract class BasicTestSetup { ...@@ -209,7 +210,7 @@ public abstract class BasicTestSetup {
entities.addAll(productDimColumns); entities.addAll(productDimColumns);
AtlasEntity productDim = AtlasEntity productDim =
table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns, table("product_dim", "product dimension table", salesDB, sd, "John Doe 2", "Managed", productDimColumns,
"Dimension"); "Dimension");
entities.add(productDim); entities.add(productDim);
...@@ -240,7 +241,7 @@ public abstract class BasicTestSetup { ...@@ -240,7 +241,7 @@ public abstract class BasicTestSetup {
entities.add(loadSalesMonthly); entities.add(loadSalesMonthly);
AtlasEntity loggingFactMonthly = AtlasEntity loggingFactMonthly =
table("logging_fact_monthly_mv", "logging fact monthly materialized view", logDB, sd, "Tim ETL", table("logging_fact_monthly_mv", "logging fact monthly materialized view", logDB, sd, "Tim ETL 2",
"Managed", logFactColumns, "Log Data"); "Managed", logFactColumns, "Log Data");
entities.add(loggingFactMonthly); entities.add(loggingFactMonthly);
...@@ -318,9 +319,12 @@ public abstract class BasicTestSetup { ...@@ -318,9 +319,12 @@ public abstract class BasicTestSetup {
protected AtlasEntity table(String name, String description, AtlasEntity db, AtlasEntity sd, String owner, String tableType, protected AtlasEntity table(String name, String description, AtlasEntity db, AtlasEntity sd, String owner, String tableType,
List<AtlasEntity> columns, String... traitNames) { List<AtlasEntity> columns, String... traitNames) {
String dbName = db.getAttribute(AtlasClient.NAME).toString();
String clusterName = db.getAttribute("clusterName").toString();
AtlasEntity table = new AtlasEntity(HIVE_TABLE_TYPE); AtlasEntity table = new AtlasEntity(HIVE_TABLE_TYPE);
table.setAttribute("name", name); table.setAttribute("name", name);
table.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "qualified:" + name); table.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName + "." + name);
table.setAttribute("description", description); table.setAttribute("description", description);
table.setAttribute("owner", owner); table.setAttribute("owner", owner);
table.setAttribute("tableType", tableType); table.setAttribute("tableType", tableType);
...@@ -335,6 +339,12 @@ public abstract class BasicTestSetup { ...@@ -335,6 +339,12 @@ public abstract class BasicTestSetup {
table.setAttribute("columns", getAtlasObjectIds(columns)); table.setAttribute("columns", getAtlasObjectIds(columns));
table.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList())); table.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList()));
sd.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName + "." + name + "@" + clusterName + "_storage");
for (AtlasEntity column : columns) {
column.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName + "." + name + "." + column.getAttribute(AtlasClient.NAME).toString() + "@" + clusterName);
}
return table; return table;
} }
......
repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
...@@ -22,6 +22,7 @@ import org.apache.atlas.discovery.EntityDiscoveryService; ...@@ -22,6 +22,7 @@ import org.apache.atlas.discovery.EntityDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.runner.LocalSolrRunner; import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.commons.collections.CollectionUtils;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider; import org.testng.annotations.DataProvider;
...@@ -116,11 +117,11 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -116,11 +117,11 @@ public class DSLQueriesTest extends BasicTestSetup {
{"hive_table", 10}, {"hive_table", 10},
{"hive_table isa Dimension", 3}, {"hive_table isa Dimension", 3},
{"hive_column where hive_column isa PII", 4}, {"hive_column where hive_column isa PII", 4},
{"hive_column where hive_column isa PII select hive_column.name", 4}, {"hive_column where hive_column isa PII select hive_column.qualifiedName", 4},
{"hive_column select hive_column.name", 17}, {"hive_column select hive_column.qualifiedName", 17},
{"hive_column select name", 17}, {"hive_column select qualifiedName", 17},
{"hive_column where hive_column.name=\"customer_id\"", 2}, {"hive_column where hive_column.name=\"customer_id\"", 2},
{"from hive_table select hive_table.name", 10}, {"from hive_table select hive_table.qualifiedName", 10},
{"hive_db where (name = \"Reporting\")", 1}, {"hive_db where (name = \"Reporting\")", 1},
{"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1", 1}, {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1", 1},
{"hive_db where hive_db is JdbcAccess", 0}, {"hive_db where hive_db is JdbcAccess", 0},
...@@ -154,8 +155,8 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -154,8 +155,8 @@ public class DSLQueriesTest extends BasicTestSetup {
return new Object[][]{ return new Object[][]{
{"hive_column", 17, 40, 0}, {"hive_column", 17, 40, 0},
{"hive_column limit 10", 10, 50, 0}, {"hive_column limit 10", 10, 50, 0},
{"hive_column select hive_column.name limit 10", 10, 5, 0}, {"hive_column select hive_column.qualifiedName limit 10", 10, 5, 0},
{"hive_column select hive_column.name limit 40 offset 10", 7, 40, 0}, {"hive_column select hive_column.qualifiedName limit 40 offset 10", 7, 40, 0},
{"hive_db where name = 'Reporting' limit 10 offset 0", 1, 40, 0}, {"hive_db where name = 'Reporting' limit 10 offset 0", 1, 40, 0},
{"hive_table where db.name = 'Reporting' limit 10", 4, 1, 0}, {"hive_table where db.name = 'Reporting' limit 10", 4, 1, 0},
}; };
...@@ -171,7 +172,7 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -171,7 +172,7 @@ public class DSLQueriesTest extends BasicTestSetup {
private Object[][] syntaxVerifierQueries() { private Object[][] syntaxVerifierQueries() {
return new Object[][]{ return new Object[][]{
{"hive_column limit 10 ", 10}, {"hive_column limit 10 ", 10},
{"hive_column select hive_column.name limit 10 ", 10}, {"hive_column select hive_column.qualifiedName limit 10 ", 10},
{"from hive_db", 3}, {"from hive_db", 3},
{"from hive_db limit 2", 2}, {"from hive_db limit 2", 2},
{"from hive_db limit 2 offset 0", 2}, {"from hive_db limit 2 offset 0", 2},
...@@ -210,13 +211,13 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -210,13 +211,13 @@ public class DSLQueriesTest extends BasicTestSetup {
{"hive_column where hive_column isa PII limit 5 offset 1", 3}, {"hive_column where hive_column isa PII limit 5 offset 1", 3},
{"hive_column where hive_column isa PII limit 5 offset 5", 0}, {"hive_column where hive_column isa PII limit 5 offset 5", 0},
{"hive_column select hive_column.name", 17}, {"hive_column select hive_column.qualifiedName", 17},
{"hive_column select hive_column.name limit 5", 5}, {"hive_column select hive_column.qualifiedName limit 5", 5},
{"hive_column select hive_column.name limit 5 offset 36", 0}, {"hive_column select hive_column.qualifiedName limit 5 offset 36", 0},
{"hive_column select name", 17}, {"hive_column select qualifiedName", 17},
{"hive_column select name limit 5", 5}, {"hive_column select qualifiedName limit 5", 5},
{"hive_column select name limit 5 offset 36 ", 0}, {"hive_column select qualifiedName limit 5 offset 36 ", 0},
{"hive_column where hive_column.name=\"customer_id\"", 2}, {"hive_column where hive_column.name=\"customer_id\"", 2},
{"hive_column where hive_column.name=\"customer_id\" limit 2", 2}, {"hive_column where hive_column.name=\"customer_id\" limit 2", 2},
...@@ -269,19 +270,19 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -269,19 +270,19 @@ public class DSLQueriesTest extends BasicTestSetup {
private Object[][] orderByQueries() { private Object[][] orderByQueries() {
return new Object[][]{ return new Object[][]{
{"from hive_db as h orderby h.owner limit 3", 3, "owner", true}, {"from hive_db as h orderby h.owner limit 3", 3, "owner", true},
{"hive_column as c select c.name orderby hive_column.name ", 17, "c.name", true}, {"hive_column as c select c.qualifiedName orderby hive_column.qualifiedName ", 17, "c.qualifiedName", true},
{"hive_column as c select c.name orderby hive_column.name limit 5", 5, "c.name", true}, {"hive_column as c select c.qualifiedName orderby hive_column.qualifiedName limit 5", 5, "c.qualifiedName", true},
{"hive_column as c select c.name orderby hive_column.name desc limit 5", 5, "c.name", false}, {"hive_column as c select c.qualifiedName orderby hive_column.qualifiedName desc limit 5", 5, "c.qualifiedName", false},
{"from hive_db orderby hive_db.owner limit 3", 3, "owner", true}, {"from hive_db orderby hive_db.owner limit 3", 3, "owner", true},
{"hive_column select hive_column.name orderby hive_column.name ", 17, "hive_column.name", true}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName ", 17, "hive_column.qualifiedName", true},
{"hive_column select hive_column.name orderby hive_column.name limit 5", 5, "hive_column.name", true}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName limit 5", 5, "hive_column.qualifiedName", true},
{"hive_column select hive_column.name orderby hive_column.name desc limit 5", 5, "hive_column.name", false}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName desc limit 5", 5, "hive_column.qualifiedName", false},
{"from hive_db orderby owner limit 3", 3, "owner", true}, {"from hive_db orderby owner limit 3", 3, "owner", true},
{"hive_column select hive_column.name orderby name ", 17, "hive_column.name", true}, {"hive_column select hive_column.qualifiedName orderby qualifiedName ", 17, "hive_column.qualifiedName", true},
{"hive_column select hive_column.name orderby name limit 5", 5, "hive_column.name", true}, {"hive_column select hive_column.qualifiedName orderby qualifiedName limit 5", 5, "hive_column.qualifiedName", true},
{"hive_column select hive_column.name orderby name desc limit 5", 5, "hive_column.name", false}, {"hive_column select hive_column.qualifiedName orderby qualifiedName desc limit 5", 5, "hive_column.qualifiedName", false},
{"from hive_db orderby hive_db.owner limit 3", 3, "owner", true}, {"from hive_db orderby hive_db.owner limit 3", 3, "owner", true},
{"hive_db where hive_db.name=\"Reporting\" orderby 'owner'", 1, "owner", true}, {"hive_db where hive_db.name=\"Reporting\" orderby 'owner'", 1, "owner", true},
...@@ -300,15 +301,15 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -300,15 +301,15 @@ public class DSLQueriesTest extends BasicTestSetup {
{"hive_table orderby hive_table.owner limit 8 offset 0", 8, "owner", true}, {"hive_table orderby hive_table.owner limit 8 offset 0", 8, "owner", true},
{"hive_table orderby hive_table.owner desc limit 8 offset 0", 8, "owner", false}, {"hive_table orderby hive_table.owner desc limit 8 offset 0", 8, "owner", false},
{"hive_column select hive_column.name orderby hive_column.name ", 17, "hive_column.name", true}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName ", 17, "hive_column.qualifiedName", true},
{"hive_column select hive_column.name orderby hive_column.name limit 5", 5, "hive_column.name", true}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName limit 5", 5, "hive_column.qualifiedName", true},
{"hive_column select hive_column.name orderby hive_column.name desc limit 5", 5, "hive_column.name", false}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName desc limit 5", 5, "hive_column.qualifiedName", false},
{"hive_column select hive_column.name orderby hive_column.name limit 5 offset 2", 5, "hive_column.name", true}, {"hive_column select hive_column.qualifiedName orderby hive_column.qualifiedName limit 5 offset 2", 5, "hive_column.qualifiedName", true},
{"hive_column select name orderby hive_column.name", 17, "name", true}, {"hive_column select qualifiedName orderby hive_column.qualifiedName", 17, "qualifiedName", true},
{"hive_column select name orderby hive_column.name limit 5", 5, "name", true}, {"hive_column select qualifiedName orderby hive_column.qualifiedName limit 5", 5, "qualifiedName", true},
{"hive_column select name orderby hive_column.name desc", 17, "name", false}, {"hive_column select qualifiedName orderby hive_column.qualifiedName desc", 17, "qualifiedName", false},
{"hive_column where hive_column.name=\"customer_id\" orderby hive_column.name", 2, "name", true}, {"hive_column where hive_column.name=\"customer_id\" orderby hive_column.name", 2, "name", true},
{"hive_column where hive_column.name=\"customer_id\" orderby hive_column.name limit 2", 2, "name", true}, {"hive_column where hive_column.name=\"customer_id\" orderby hive_column.name limit 2", 2, "name", true},
...@@ -368,7 +369,7 @@ public class DSLQueriesTest extends BasicTestSetup { ...@@ -368,7 +369,7 @@ public class DSLQueriesTest extends BasicTestSetup {
private void assertSearchResult(AtlasSearchResult searchResult, int expected) { private void assertSearchResult(AtlasSearchResult searchResult, int expected) {
assertNotNull(searchResult); assertNotNull(searchResult);
if(expected == 0) { if(expected == 0) {
assertNull(searchResult.getAttributes()); assertTrue(searchResult.getAttributes() == null || CollectionUtils.isEmpty(searchResult.getAttributes().getValues()));
assertNull(searchResult.getEntities()); assertNull(searchResult.getEntities());
} else if(searchResult.getEntities() != null) { } else if(searchResult.getEntities() != null) {
assertEquals(searchResult.getEntities().size(), expected); assertEquals(searchResult.getEntities().size(), expected);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment