Skip to content

A
atlas
Commit 6b33bcf6 by Shwetha GS
Options

ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)

parent 901aca9d
Showing with 902 additions and 419 deletions
pom.xml
...@@ -338,6 +338,7 @@ ...@@ -338,6 +338,7 @@
<spray.version>1.3.1</spray.version> <spray.version>1.3.1</spray.version>
<guava.version>14.0</guava.version> <guava.version>14.0</guava.version>
<fastutil.version>6.5.16</fastutil.version> <fastutil.version>6.5.16</fastutil.version>
<guice.version>4.0</guice.version>
<PermGen>64m</PermGen> <PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen> <MaxPermGen>512m</MaxPermGen>
...@@ -563,25 +564,25 @@ ...@@ -563,25 +564,25 @@
<dependency> <dependency>
<groupId>com.google.inject</groupId> <groupId>com.google.inject</groupId>
<artifactId>guice</artifactId> <artifactId>guice</artifactId>
<version>4.0</version> <version>${guice.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.inject.extensions</groupId> <groupId>com.google.inject.extensions</groupId>
<artifactId>guice-throwingproviders</artifactId> <artifactId>guice-throwingproviders</artifactId>
<version>4.0</version> <version>${guice.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.inject.extensions</groupId> <groupId>com.google.inject.extensions</groupId>
<artifactId>guice-multibindings</artifactId> <artifactId>guice-multibindings</artifactId>
<version>4.0</version> <version>${guice.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.inject.extensions</groupId> <groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId> <artifactId>guice-servlet</artifactId>
<version>4.0</version> <version>${guice.version}</version>
</dependency> </dependency>
<dependency> <dependency>
...@@ -591,6 +592,13 @@ ...@@ -591,6 +592,13 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.skyscreamer</groupId>
<artifactId>jsonassert</artifactId>
<version>1.2.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId> <groupId>joda-time</groupId>
<artifactId>joda-time</artifactId> <artifactId>joda-time</artifactId>
<version>2.5</version> <version>2.5</version>
......
release-log.txt
...@@ -4,6 +4,7 @@ Apache Atlas Release Notes ...@@ -4,6 +4,7 @@ Apache Atlas Release Notes
--trunk - unreleased --trunk - unreleased
INCOMPATIBLE CHANGES: INCOMPATIBLE CHANGES:
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES: ALL CHANGES:
ATLAS-31 ATLAS build fails with clean repo (suma.shivaprasad via shwethags) ATLAS-31 ATLAS build fails with clean repo (suma.shivaprasad via shwethags)
......
repository/pom.xml
...@@ -55,6 +55,11 @@ ...@@ -55,6 +55,11 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId> <groupId>com.google.inject</groupId>
<artifactId>guice</artifactId> <artifactId>guice</artifactId>
</dependency> </dependency>
...@@ -146,6 +151,11 @@ ...@@ -146,6 +151,11 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.skyscreamer</groupId>
<artifactId>jsonassert</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId> <groupId>org.testng</groupId>
<artifactId>testng</artifactId> <artifactId>testng</artifactId>
</dependency> </dependency>
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
...@@ -857,7 +857,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -857,7 +857,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) { } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
propertyValue = typedInstance.getBigDecimal(attributeInfo.name); propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) { } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
propertyValue = typedInstance.getDate(attributeInfo.name); final Date dateVal = typedInstance.getDate(attributeInfo.name);
//Convert Property value to Long while persisting
propertyValue = dateVal.getTime();
} }
addProperty(instanceVertex, vertexPropertyName, propertyValue); addProperty(instanceVertex, vertexPropertyName, propertyValue);
} }
...@@ -1180,7 +1182,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -1180,7 +1182,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
typedInstance typedInstance
.setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName)); .setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) { } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
typedInstance.setDate(attributeInfo.name, instanceVertex.<Date>getProperty(vertexPropertyName)); final Long dateVal = instanceVertex.<Long>getProperty(vertexPropertyName);
typedInstance.setDate(attributeInfo.name, new Date(dateVal));
} }
} }
} }
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
...@@ -65,7 +65,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -65,7 +65,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
this.titanGraph = graphProvider.get(); this.titanGraph = graphProvider.get();
/* Create the transaction for indexing. /* Create the transaction for indexing.
* Commit/rollback is expected to be called from the caller.
*/ */
management = titanGraph.getManagementSystem(); management = titanGraph.getManagementSystem();
initialize(); initialize();
...@@ -257,7 +256,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -257,7 +256,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
} else if (dataType == DataTypes.BIGDECIMAL_TYPE) { } else if (dataType == DataTypes.BIGDECIMAL_TYPE) {
return BigDecimal.class; return BigDecimal.class;
} else if (dataType == DataTypes.DATE_TYPE) { } else if (dataType == DataTypes.DATE_TYPE) {
return Date.class; //Indexing with date converted to long as of now since Titan is yet to add support for Date type with mixed indexes
return Long.class;
} }
...@@ -333,9 +333,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -333,9 +333,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
} }
private boolean checkIfMixedIndexApplicable(Class propertyClass) { private boolean checkIfMixedIndexApplicable(Class propertyClass) {
//TODO - Check why date types are failing in ES/Solr if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class) {
if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class
|| propertyClass == Date.class) {
return false; return false;
} }
return true; return true;
......
repository/src/main/scala/org/apache/atlas/query/Expressions.scala
...@@ -490,6 +490,8 @@ object Expressions { ...@@ -490,6 +490,8 @@ object Expressions {
def string(rawValue: Any) = literal(DataTypes.STRING_TYPE, rawValue) def string(rawValue: Any) = literal(DataTypes.STRING_TYPE, rawValue)
def date(rawValue: Any) = literal(DataTypes.DATE_TYPE, rawValue)
case class ArithmeticExpression(symbol: String, case class ArithmeticExpression(symbol: String,
left: Expression, left: Expression,
right: Expression) right: Expression)
...@@ -598,7 +600,10 @@ object Expressions { ...@@ -598,7 +600,10 @@ object Expressions {
throw new UnresolvedException(this, throw new UnresolvedException(this,
s"datatype. Can not resolve due to unresolved children") s"datatype. Can not resolve due to unresolved children")
} }
if (left.dataType != DataTypes.STRING_TYPE || right.dataType != DataTypes.STRING_TYPE) {
if(left.dataType == DataTypes.DATE_TYPE) {
DataTypes.DATE_TYPE
} else if (left.dataType != DataTypes.STRING_TYPE || right.dataType != DataTypes.STRING_TYPE) {
TypeUtils.combinedType(left.dataType, right.dataType) TypeUtils.combinedType(left.dataType, right.dataType)
} }
DataTypes.BOOLEAN_TYPE DataTypes.BOOLEAN_TYPE
......
repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
...@@ -18,6 +18,8 @@ ...@@ -18,6 +18,8 @@
package org.apache.atlas.query package org.apache.atlas.query
import java.util.Date
import com.thinkaurelius.titan.core.TitanVertex import com.thinkaurelius.titan.core.TitanVertex
import com.tinkerpop.blueprints.Direction import com.tinkerpop.blueprints.Direction
import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException} import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException}
...@@ -304,6 +306,10 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { ...@@ -304,6 +306,10 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
case x: FloatType => i.setFloat(aInfo.name, v.getProperty[java.lang.Float](fName)) case x: FloatType => i.setFloat(aInfo.name, v.getProperty[java.lang.Float](fName))
case x: DoubleType => i.setDouble(aInfo.name, v.getProperty[java.lang.Double](fName)) case x: DoubleType => i.setDouble(aInfo.name, v.getProperty[java.lang.Double](fName))
case x: StringType => i.setString(aInfo.name, v.getProperty[java.lang.String](fName)) case x: StringType => i.setString(aInfo.name, v.getProperty[java.lang.String](fName))
case x: DateType => {
val dateVal = v.getProperty[java.lang.Long](fName)
i.setDate(aInfo.name, new Date(dateVal))
}
case _ => throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported") case _ => throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
} }
} }
......
repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
...@@ -19,7 +19,9 @@ ...@@ -19,7 +19,9 @@
package org.apache.atlas.query package org.apache.atlas.query
import org.apache.atlas.query.Expressions._ import org.apache.atlas.query.Expressions._
import org.apache.atlas.typesystem.types.{TypeSystem, DataTypes}
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory import org.apache.atlas.typesystem.types.DataTypes.TypeCategory
import org.joda.time.format.ISODateTimeFormat
import scala.collection.mutable import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ArrayBuffer
...@@ -227,12 +229,29 @@ class GremlinTranslator(expr: Expression, ...@@ -227,12 +229,29 @@ class GremlinTranslator(expr: Expression,
} }
} }
case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => { case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => {
val QUOTE = "\"";
val fieldGremlinExpr = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}" val fieldGremlinExpr = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}"
ch match { ch match {
case Some(child) => { case Some(child) => {
s"""${genQuery(child, inSelect)}.has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)""" s"""${genQuery(child, inSelect)}.has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)"""
} }
case None => s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)""" case None => {
if (fInfo.attrInfo.dataType == DataTypes.DATE_TYPE) {
try {
//Accepts both date, datetime formats
val dateStr = l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE)
val dateVal = ISODateTimeFormat.dateOptionalTimeParser().parseDateTime(dateStr).getMillis
s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)},${dateVal})"""
} catch {
case pe: java.text.ParseException =>
throw new GremlinTranslationException(c,
"Date format " + l + " not supported. Should be of the format " + TypeSystem.getInstance().getDateFormat.toPattern);
}
}
else
s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)"""
}
} }
} }
case fil@FilterExpression(child, condExpr) => { case fil@FilterExpression(child, condExpr) => {
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
...@@ -169,7 +169,8 @@ public class GraphBackedDiscoveryServiceTest { ...@@ -169,7 +169,8 @@ public class GraphBackedDiscoveryServiceTest {
@DataProvider(name = "dslQueriesProvider") @DataProvider(name = "dslQueriesProvider")
private Object[][] createDSLQueries() { private Object[][] createDSLQueries() {
return new String[][]{{"from DB"}, {"DB"}, {"DB where DB.name=\"Reporting\""}, {"DB DB.name = \"Reporting\""}, return new String[][]{
{"from DB"}, {"DB"}, {"DB where DB.name=\"Reporting\""}, {"DB DB.name = \"Reporting\""},
{"DB where DB.name=\"Reporting\" select name, owner"}, {"DB has name"}, {"DB, Table"}, {"DB where DB.name=\"Reporting\" select name, owner"}, {"DB has name"}, {"DB, Table"},
{"DB is JdbcAccess"}, {"DB is JdbcAccess"},
/* /*
...@@ -186,6 +187,8 @@ public class GraphBackedDiscoveryServiceTest { ...@@ -186,6 +187,8 @@ public class GraphBackedDiscoveryServiceTest {
{"DB where DB is JdbcAccess"}, {"DB where DB has name"}, {"DB Table"}, {"DB where DB has name"}, {"DB where DB is JdbcAccess"}, {"DB where DB has name"}, {"DB Table"}, {"DB where DB has name"},
{"DB as db1 Table where (db1.name = \"Reporting\")"}, {"DB as db1 Table where (db1.name = \"Reporting\")"},
{"DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "}, {"DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
{"Table where (name = \"sales_fact\" and created > \"2014-01-01\" ) select name as _col_0, created as _col_1 "},
{"Table where (name = \"sales_fact\" and created > \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, created as _col_1 "},
/* /*
todo: does not work todo: does not work
{"DB where (name = \"Reporting\") and ((createTime + 1) > 0)"}, {"DB where (name = \"Reporting\") and ((createTime + 1) > 0)"},
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
...@@ -21,15 +21,20 @@ package org.apache.atlas.repository.graph; ...@@ -21,15 +21,20 @@ package org.apache.atlas.repository.graph;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanGraphQuery;
import com.thinkaurelius.titan.core.TitanIndexQuery; import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.diskstorage.BackendException; import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration; import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration;
import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration; import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.tinkerpop.blueprints.Compare; import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery; import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Predicate;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.GraphTransaction; import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.BaseTest;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
...@@ -55,6 +60,7 @@ import org.testng.annotations.Test; ...@@ -55,6 +60,7 @@ import org.testng.annotations.Test;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date;
import java.util.Map; import java.util.Map;
import java.util.Random; import java.util.Random;
...@@ -157,6 +163,7 @@ public class GraphRepoMapperScaleTest { ...@@ -157,6 +163,7 @@ public class GraphRepoMapperScaleTest {
searchWithOutIndex("hive_table_type.name", "bar-999"); searchWithOutIndex("hive_table_type.name", "bar-999");
searchWithIndex("hive_table_type.name", "bar-999"); searchWithIndex("hive_table_type.name", "bar-999");
searchWithIndex("hive_table_type.created", Compare.GREATER_THAN_EQUAL, BaseTest.TEST_DATE_IN_LONG);
for (int index = 500; index < 600; index++) { for (int index = 500; index < 600; index++) {
searchWithIndex("hive_table_type.name", "bar-" + index); searchWithIndex("hive_table_type.name", "bar-" + index);
...@@ -194,6 +201,21 @@ public class GraphRepoMapperScaleTest { ...@@ -194,6 +201,21 @@ public class GraphRepoMapperScaleTest {
} }
} }
private void searchWithIndex(String key, Predicate searchPredicate, Object value) {
TitanGraph graph = graphProvider.get();
long start = System.currentTimeMillis();
int count = 0;
try {
GraphQuery query = graph.query().has(key, searchPredicate, value);
for (Vertex ignored : query.vertices()) {
count++;
}
} finally {
System.out.println("Search on [" + key + "=" + value + "] returned results: " + count + ", took " + (
System.currentTimeMillis() - start) + " ms");
}
}
private void createHiveTypes() throws Exception { private void createHiveTypes() throws Exception {
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = TypesUtil HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = TypesUtil
.createClassTypeDef(DATABASE_TYPE, ImmutableList.<String>of(), .createClassTypeDef(DATABASE_TYPE, ImmutableList.<String>of(),
...@@ -222,6 +244,7 @@ public class GraphRepoMapperScaleTest { ...@@ -222,6 +244,7 @@ public class GraphRepoMapperScaleTest {
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE), TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE), TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
// enum // enum
new AttributeDefinition("tableType", "table_type", Multiplicity.REQUIRED, false, null), new AttributeDefinition("tableType", "table_type", Multiplicity.REQUIRED, false, null),
// array of strings // array of strings
...@@ -262,6 +285,7 @@ public class GraphRepoMapperScaleTest { ...@@ -262,6 +285,7 @@ public class GraphRepoMapperScaleTest {
tableInstance.set("name", TABLE_NAME + "-" + uberIndex); tableInstance.set("name", TABLE_NAME + "-" + uberIndex);
tableInstance.set("description", "bar table" + "-" + uberIndex); tableInstance.set("description", "bar table" + "-" + uberIndex);
tableInstance.set("type", "managed"); tableInstance.set("type", "managed");
tableInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
tableInstance.set("tableType", 1); // enum tableInstance.set("tableType", 1); // enum
// refer to an existing class // refer to an existing class
......
repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
...@@ -43,7 +43,7 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { ...@@ -43,7 +43,7 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
test("testClass") { test("testClass") {
val r = QueryProcessor.evaluate(_class("DB"), g) val r = QueryProcessor.evaluate(_class("DB"), g)
validateJson(r, "{\n \"query\":\"DB\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") validateJson(r, "{\n \"query\":\"DB\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
} }
test("testName") { test("testName") {
...@@ -53,12 +53,12 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { ...@@ -53,12 +53,12 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
test("testFilter") { test("testFilter") {
var r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))), g) var r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))), g)
validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
} }
test("testFilter2") { test("testFilter2") {
var r = QueryProcessor.evaluate(_class("DB").where(id("DB").field("name").`=`(string("Reporting"))), g) var r = QueryProcessor.evaluate(_class("DB").where(id("DB").field("name").`=`(string("Reporting"))), g)
validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
} }
...@@ -70,17 +70,400 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { ...@@ -70,17 +70,400 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
test("testIsTrait") { test("testIsTrait") {
val r = QueryProcessor.evaluate(_class("Table").where(isTrait("Dimension")), g) val r = QueryProcessor.evaluate(_class("Table").where(isTrait("Dimension")), g)
validateJson(r, "{\n \"query\":\"Table where Table is Dimension\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dataTypeName\":\"StorageDesc\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n }\n ]\n}") validateJson(r, """{
| "query":"Table where Table is Dimension",
| "dataType":{
| "superTypes":[
|
| ],
| "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
| "typeName":"Table",
| "attributeDefinitions":[
| {
| "name":"name",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"db",
| "dataTypeName":"DB",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"sd",
| "dataTypeName":"StorageDesc",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"created",
| "dataTypeName":"date",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| }
| ]
| },
| "rows":[
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"product_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"time_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"customer_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| }
| ]
|}""".stripMargin)
} }
test("testhasField") { test("testhasField") {
val r = QueryProcessor.evaluate(_class("DB").where(hasField("name")), g) val r = QueryProcessor.evaluate(_class("DB").where(hasField("name")), g)
validateJson(r, "{\n \"query\":\"DB where DB has name\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") validateJson(r, """{
| "query":"DB where DB has name",
| "dataType":{
| "superTypes":[
|
| ],
| "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
| "typeName":"DB",
| "attributeDefinitions":[
| {
| "name":"name",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"owner",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"createTime",
| "dataTypeName":"int",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| }
| ]
| },
| "rows":[
| {
| "$typeName$":"DB",
| "$id$":{
| "$typeName$":"DB",
| "version":0
| },
| "owner":"John ETL",
| "name":"Sales",
| "createTime":1000
| },
| {
| "$typeName$":"DB",
| "$id$":{
| "$typeName$":"DB",
| "version":0
| },
| "owner":"Jane BI",
| "name":"Reporting",
| "createTime":1500
| }
| ]
|}""".stripMargin)
} }
test("testFieldReference") { test("testFieldReference") {
val r = QueryProcessor.evaluate(_class("DB").field("Table"), g) val r = QueryProcessor.evaluate(_class("DB").field("Table"), g)
validateJson(r, "{\n \"query\":\"DB Table\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dataTypeName\":\"StorageDesc\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"2048\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"512\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"8960\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"7424\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"12800\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"11264\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_monthly_mv\"\n }\n ]\n}") validateJson(r, """{
| "query":"DB Table",
| "dataType":{
| "superTypes":[ ],
| "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
| "typeName":"Table",
| "attributeDefinitions":[
| {
| "name":"name",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"db",
| "dataTypeName":"DB",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"sd",
| "dataTypeName":"StorageDesc",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"created",
| "dataTypeName":"date",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| }
| ]
| },
| "rows":[
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"product_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"time_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"customer_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_daily_mv"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_monthly_mv"
| }
| ]
|}""".stripMargin);
} }
test("testBackReference") { test("testBackReference") {
...@@ -98,7 +481,7 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { ...@@ -98,7 +481,7 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
test("testComparisonLogical") { test("testComparisonLogical") {
val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting")). val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting")).
and(id("createTime") > int(0))), g) and(id("createTime") > int(0))), g)
validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") and (createTime > 0)\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}") validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") and (createTime > 0)\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
} }
test("testJoinAndSelect1") { test("testJoinAndSelect1") {
......
repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala
...@@ -19,12 +19,14 @@ ...@@ -19,12 +19,14 @@
package org.apache.atlas.query package org.apache.atlas.query
import java.io.File import java.io.File
import java.util.UUID import java.util.{Date, UUID}
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
import javax.script.{Bindings, ScriptEngine, ScriptEngineManager} import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
import com.thinkaurelius.titan.core.TitanGraph import com.thinkaurelius.titan.core.TitanGraph
import com.typesafe.config.ConfigFactory import com.typesafe.config.ConfigFactory
import org.apache.atlas.repository.BaseTest
import org.apache.atlas.typesystem.types.TypeSystem
import org.apache.commons.io.FileUtils import org.apache.commons.io.FileUtils
import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ArrayBuffer
...@@ -54,18 +56,19 @@ object HiveTitanSample { ...@@ -54,18 +56,19 @@ object HiveTitanSample {
this.getClass.getDeclaredFields filter (_.getName != "traits") foreach { f => this.getClass.getDeclaredFields filter (_.getName != "traits") foreach { f =>
f.setAccessible(true) f.setAccessible(true)
var fV = f.get(this) val fV = f.get(this)
fV = fV match { val convertedVal = fV match {
case _: String => s""""$fV"""" case _: String => s""""$fV""""
case d: Date => d.getTime
case _ => fV case _ => fV
} }
fV match { convertedVal match {
case x: Vertex => addEdge(x, s"${this.getClass.getSimpleName}.${f.getName}", edges) case x: Vertex => addEdge(x, s"${this.getClass.getSimpleName}.${f.getName}", edges)
case l: List[_] => l.foreach(x => addEdge(x.asInstanceOf[Vertex], case l: List[_] => l.foreach(x => addEdge(x.asInstanceOf[Vertex],
s"${this.getClass.getSimpleName}.${f.getName}", edges)) s"${this.getClass.getSimpleName}.${f.getName}", edges))
case _ => sb.append( s""", "${f.getName}" : $fV""") case _ => sb.append( s""", "${f.getName}" : $convertedVal""")
sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $fV""") sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $convertedVal""")
} }
} }
...@@ -127,17 +130,23 @@ object HiveTitanSample { ...@@ -127,17 +130,23 @@ object HiveTitanSample {
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
case class Table(name: String, db: DB, sd: StorageDescriptor, case class Table(name: String, db: DB, sd: StorageDescriptor,
created: Date,
traits: Option[List[Trait]] = None, traits: Option[List[Trait]] = None,
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
case class TableDef(name: String, db: DB, inputFormat: String, outputFormat: String, case class TableDef(name: String, db: DB, inputFormat: String, outputFormat: String,
columns: List[(String, String, Option[List[Trait]])], columns: List[(String, String, Option[List[Trait]])],
traits: Option[List[Trait]] = None) { traits: Option[List[Trait]] = None,
created: Option[Date] = None) {
val createdDate : Date = created match {
case Some(x) => x
case None => new Date(BaseTest.TEST_DATE_IN_LONG)
}
val sd = StorageDescriptor(inputFormat, outputFormat) val sd = StorageDescriptor(inputFormat, outputFormat)
val colDefs = columns map { c => val colDefs = columns map { c =>
Column(c._1, c._2, sd, c._3) Column(c._1, c._2, sd, c._3)
} }
val tablDef = Table(name, db, sd, traits) val tablDef = Table(name, db, sd, createdDate, traits)
def toGSon(vertices: ArrayBuffer[String], def toGSon(vertices: ArrayBuffer[String],
edges: ArrayBuffer[String]): Unit = { edges: ArrayBuffer[String]): Unit = {
...@@ -167,6 +176,7 @@ object HiveTitanSample { ...@@ -167,6 +176,7 @@ object HiveTitanSample {
("time_id", "int", None), ("time_id", "int", None),
("product_id", "int", None), ("product_id", "int", None),
("customer_id", "int", None), ("customer_id", "int", None),
("created", "date", None),
("sales", "double", Some(List(Metric()))) ("sales", "double", Some(List(Metric())))
)) ))
val productDim = TableDef("product_dim", val productDim = TableDef("product_dim",
...@@ -269,6 +279,7 @@ object HiveTitanSample { ...@@ -269,6 +279,7 @@ object HiveTitanSample {
FileUtils.writeStringToFile(new File(fileName), toGSon()) FileUtils.writeStringToFile(new File(fileName), toGSon())
} }
val GremlinQueries = List( val GremlinQueries = List(
// 1. List all DBs // 1. List all DBs
"""g.V.has("typeName", "DB")""", """g.V.has("typeName", "DB")""",
......
repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
...@@ -26,7 +26,7 @@ import org.scalatest.junit.JUnitRunner ...@@ -26,7 +26,7 @@ import org.scalatest.junit.JUnitRunner
import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite} import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite}
@RunWith(classOf[JUnitRunner]) @RunWith(classOf[JUnitRunner])
class LineageQueryTest extends FunSuite with BeforeAndAfterAll { class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
var g: TitanGraph = null var g: TitanGraph = null
...@@ -40,140 +40,134 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll { ...@@ -40,140 +40,134 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll {
g.shutdown() g.shutdown()
} }
val STRUCT_NAME_REGEX = (TypeUtils.TEMP_STRUCT_NAME_PREFIX + "\\d+").r
val PREFIX_SPACES_REGEX = ("\\n\\s*").r val PREFIX_SPACES_REGEX = ("\\n\\s*").r
def validateJson(r: GremlinQueryResult, expected: String = null): Unit = {
val rJ = r.toJson
if (expected != null) {
var a = STRUCT_NAME_REGEX.replaceAllIn(rJ, "")
a = PREFIX_SPACES_REGEX.replaceAllIn(a, "")
var b = STRUCT_NAME_REGEX.replaceAllIn(expected, "")
b = PREFIX_SPACES_REGEX.replaceAllIn(b, "")
Assertions.assert(a == b)
} else {
println(rJ)
}
}
test("testInputTables") { test("testInputTables") {
val r = QueryProcessor.evaluate(_class("LoadProcess").field("inputTables"), g) val r = QueryProcessor.evaluate(_class("LoadProcess").field("inputTables"), g)
val x = r.toJson val x = r.toJson
validateJson(r, """{ validateJson(r,"""{
"query":"LoadProcess inputTables", | "query":"LoadProcess inputTables",
"dataType":{ | "dataType":{
"superTypes":[ | "superTypes":[
|
], | ],
"hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType", | "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
"typeName":"Table", | "typeName":"Table",
"attributeDefinitions":[ | "attributeDefinitions":[
{ | {
"name":"name", | "name":"name",
"dataTypeName":"string", | "dataTypeName":"string",
"multiplicity":{ | "multiplicity":{
"lower":0, | "lower":0,
"upper":1, | "upper":1,
"isUnique":false | "isUnique":false
}, | },
"isComposite":false, | "isComposite":false,
"isUnique":false, | "isUnique":false,
"isIndexable":true, | "isIndexable":true,
"reverseAttributeName":null | "reverseAttributeName":null
}, | },
{ | {
"name":"db", | "name":"db",
"dataTypeName":"DB", | "dataTypeName":"DB",
"multiplicity":{ | "multiplicity":{
"lower":1, | "lower":1,
"upper":1, | "upper":1,
"isUnique":false | "isUnique":false
}, | },
"isComposite":false, | "isComposite":false,
"isUnique":false, | "isUnique":false,
"isIndexable":true, | "isIndexable":true,
"reverseAttributeName":null | "reverseAttributeName":null
}, | },
{ | {
"name":"sd", | "name":"sd",
"dataTypeName":"StorageDesc", | "dataTypeName":"StorageDesc",
"multiplicity":{ | "multiplicity":{
"lower":1, | "lower":1,
"upper":1, | "upper":1,
"isUnique":false | "isUnique":false
}, | },
"isComposite":false, | "isComposite":false,
"isUnique":false, | "isUnique":false,
"isIndexable":true, | "isIndexable":true,
"reverseAttributeName":null | "reverseAttributeName":null
} | },
] | {
}, | "name":"created",
"rows":[ | "dataTypeName":"date",
{ | "multiplicity":{
"$typeName$":"Table", | "lower":0,
"$id$":{ | "upper":1,
"id":"2048", | "isUnique":false
"$typeName$":"Table", | },
"version":0 | "isComposite":false,
}, | "isUnique":false,
"sd":{ | "isIndexable":true,
"id":"512", | "reverseAttributeName":null
"$typeName$":"StorageDesc", | }
"version":0 | ]
}, | },
"db":{ | "rows":[
"id":"256", | {
"$typeName$":"DB", | "$typeName$":"Table",
"version":0 | "$id$":{
}, | "$typeName$":"Table",
"name":"sales_fact" | "version":0
}, | },
{ | "created":"2014-12-11T02:35:58.440Z",
"$typeName$":"Table", | "sd":{
"$id$":{ | "$typeName$":"StorageDesc",
"id":"4864", | "version":0
"$typeName$":"Table", | },
"version":0 | "db":{
}, | "$typeName$":"DB",
"sd":{ | "version":0
"id":"3840", | },
"$typeName$":"StorageDesc", | "name":"sales_fact"
"version":0 | },
}, | {
"db":{ | "$typeName$":"Table",
"id":"256", | "$id$":{
"$typeName$":"DB", | "$typeName$":"Table",
"version":0 | "version":0
}, | },
"name":"time_dim", | "created":"2014-12-11T02:35:58.440Z",
"$traits$":{ | "sd":{
"Dimension":{ | "$typeName$":"StorageDesc",
"$typeName$":"Dimension" | "version":0
} | },
} | "db":{
}, | "$typeName$":"DB",
{ | "version":0
"$typeName$":"Table", | },
"$id$":{ | "name":"time_dim",
"id":"8960", | "$traits$":{
"$typeName$":"Table", | "Dimension":{
"version":0 | "$typeName$":"Dimension"
}, | }
"sd":{ | }
"id":"7424", | },
"$typeName$":"StorageDesc", | {
"version":0 | "$typeName$":"Table",
}, | "$id$":{
"db":{ | "$typeName$":"Table",
"id":"7168", | "version":0
"$typeName$":"DB", | },
"version":0 | "created":"2014-12-11T02:35:58.440Z",
}, | "sd":{
"name":"sales_fact_daily_mv" | "$typeName$":"StorageDesc",
} | "version":0
] | },
}""") | "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_daily_mv"
| }
| ]
|}
""".stripMargin)
} }
test("testLoadProcessOut") { test("testLoadProcessOut") {
...@@ -184,153 +178,157 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll { ...@@ -184,153 +178,157 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll {
test("testLineageAll") { test("testLineageAll") {
val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable")), g) val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable")), g)
validateJson(r, """{ validateJson(r, """{
"query":"Table as _loop0 loop (LoadProcess outputTable)", | "query":"Table as _loop0 loop (LoadProcess outputTable)",
"dataType":{ | "dataType":{
"superTypes":[ | "superTypes":[
|
], | ],
"hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType", | "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
"typeName":"Table", | "typeName":"Table",
"attributeDefinitions":[ | "attributeDefinitions":[
{ | {
"name":"name", | "name":"name",
"dataTypeName":"string", | "dataTypeName":"string",
"multiplicity":{ | "multiplicity":{
"lower":0, | "lower":0,
"upper":1, | "upper":1,
"isUnique":false | "isUnique":false
}, | },
"isComposite":false, | "isComposite":false,
"isUnique":false, | "isUnique":false,
"isIndexable":true, | "isIndexable":true,
"reverseAttributeName":null | "reverseAttributeName":null
}, | },
{ | {
"name":"db", | "name":"db",
"dataTypeName":"DB", | "dataTypeName":"DB",
"multiplicity":{ | "multiplicity":{
"lower":1, | "lower":1,
"upper":1, | "upper":1,
"isUnique":false | "isUnique":false
}, | },
"isComposite":false, | "isComposite":false,
"isUnique":false, | "isUnique":false,
"isIndexable":true, | "isIndexable":true,
"reverseAttributeName":null | "reverseAttributeName":null
}, | },
{ | {
"name":"sd", | "name":"sd",
"dataTypeName":"StorageDesc", | "dataTypeName":"StorageDesc",
"multiplicity":{ | "multiplicity":{
"lower":1, | "lower":1,
"upper":1, | "upper":1,
"isUnique":false | "isUnique":false
}, | },
"isComposite":false, | "isComposite":false,
"isUnique":false, | "isUnique":false,
"isIndexable":true, | "isIndexable":true,
"reverseAttributeName":null | "reverseAttributeName":null
} | },
] | {
}, | "name":"created",
"rows":[ | "dataTypeName":"date",
{ | "multiplicity":{
"$typeName$":"Table", | "lower":0,
"$id$":{ | "upper":1,
"id":"8960", | "isUnique":false
"$typeName$":"Table", | },
"version":0 | "isComposite":false,
}, | "isUnique":false,
"sd":{ | "isIndexable":true,
"id":"7424", | "reverseAttributeName":null
"$typeName$":"StorageDesc", | }
"version":0 | ]
}, | },
"db":{ | "rows":[
"id":"7168", | {
"$typeName$":"DB", | "$typeName$":"Table",
"version":0 | "$id$":{
}, | "id":"9216",
"name":"sales_fact_daily_mv" | "$typeName$":"Table",
}, | "version":0
{ | },
"$typeName$":"Table", | "created":"2014-12-11T02:35:58.440Z",
"$id$":{ | "sd":{
"id":"12800", | "$typeName$":"StorageDesc",
"$typeName$":"Table", | "version":0
"version":0 | },
}, | "db":{
"sd":{ | "$typeName$":"DB",
"id":"11264", | "version":0
"$typeName$":"StorageDesc", | },
"version":0 | "name":"sales_fact_daily_mv"
}, | },
"db":{ | {
"id":"7168", | "$typeName$":"Table",
"$typeName$":"DB", | "$id$":{
"version":0 | "$typeName$":"Table",
}, | "version":0
"name":"sales_fact_monthly_mv" | },
}, | "created":"2014-12-11T02:35:58.440Z",
{ | "sd":{
"$typeName$":"Table", | "$typeName$":"StorageDesc",
"$id$":{ | "version":0
"id":"8960", | },
"$typeName$":"Table", | "db":{
"version":0 | "$typeName$":"DB",
}, | "version":0
"sd":{ | },
"id":"7424", | "name":"sales_fact_monthly_mv"
"$typeName$":"StorageDesc", | },
"version":0 | {
}, | "$typeName$":"Table",
"db":{ | "$id$":{
"id":"7168", | "$typeName$":"Table",
"$typeName$":"DB", | "version":0
"version":0 | },
}, | "created":"2014-12-11T02:35:58.440Z",
"name":"sales_fact_daily_mv" | "sd":{
}, | "$typeName$":"StorageDesc",
{ | "version":0
"$typeName$":"Table", | },
"$id$":{ | "db":{
"id":"12800", | "$typeName$":"DB",
"$typeName$":"Table", | "version":0
"version":0 | },
}, | "name":"sales_fact_daily_mv"
"sd":{ | },
"id":"11264", | {
"$typeName$":"StorageDesc", | "$typeName$":"Table",
"version":0 | "$id$":{
}, | "$typeName$":"Table",
"db":{ | "version":0
"id":"7168", | },
"$typeName$":"DB", | "created":"2014-12-11T02:35:58.440Z",
"version":0 | "sd":{
}, | "$typeName$":"StorageDesc",
"name":"sales_fact_monthly_mv" | "version":0
}, | },
{ | "db":{
"$typeName$":"Table", | "$typeName$":"DB",
"$id$":{ | "version":0
"id":"12800", | },
"$typeName$":"Table", | "name":"sales_fact_monthly_mv"
"version":0 | },
}, | {
"sd":{ | "$typeName$":"Table",
"id":"11264", | "$id$":{
"$typeName$":"StorageDesc", | "$typeName$":"Table",
"version":0 | "version":0
}, | },
"db":{ | "created":"2014-12-11T02:35:58.440Z",
"id":"7168", | "sd":{
"$typeName$":"DB", | "$typeName$":"StorageDesc",
"version":0 | "version":0
}, | },
"name":"sales_fact_monthly_mv" | "db":{
} | "$typeName$":"DB",
] | "version":0
}""") | },
| "name":"sales_fact_monthly_mv"
| }
| ]
|}""".stripMargin)
} }
test("testLineageAllSelect") { test("testLineageAllSelect") {
...@@ -396,120 +394,127 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll { ...@@ -396,120 +394,127 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll {
"destTable":"sales_fact_monthly_mv" "destTable":"sales_fact_monthly_mv"
} }
] ]
}""") }""".stripMargin)
} }
test("testLineageFixedDepth") { test("testLineageFixedDepth") {
val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable"), int(1)), g) val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable"), int(1)), g)
validateJson(r, """{ validateJson(r, """{
"query":"Table as _loop0 loop (LoadProcess outputTable) times 1", | "query":"Table as _loop0 loop (LoadProcess outputTable) times 1",
"dataType":{ | "dataType":{
"superTypes":[ | "superTypes":[
|
], | ],
"hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType", | "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
"typeName":"Table", | "typeName":"Table",
"attributeDefinitions":[ | "attributeDefinitions":[
{ | {
"name":"name", | "name":"name",
"dataTypeName":"string", | "dataTypeName":"string",
"multiplicity":{ | "multiplicity":{
"lower":0, | "lower":0,
"upper":1, | "upper":1,
"isUnique":false | "isUnique":false
}, | },
"isComposite":false, | "isComposite":false,
"isUnique":false, | "isUnique":false,
"isIndexable":true, | "isIndexable":true,
"reverseAttributeName":null | "reverseAttributeName":null
}, | },
{ | {
"name":"db", | "name":"db",
"dataTypeName":"DB", | "dataTypeName":"DB",
"multiplicity":{ | "multiplicity":{
"lower":1, | "lower":1,
"upper":1, | "upper":1,
"isUnique":false | "isUnique":false
}, | },
"isComposite":false, | "isComposite":false,
"isUnique":false, | "isUnique":false,
"isIndexable":true, | "isIndexable":true,
"reverseAttributeName":null | "reverseAttributeName":null
}, | },
{ | {
"name":"sd", | "name":"sd",
"dataTypeName":"StorageDesc", | "dataTypeName":"StorageDesc",
"multiplicity":{ | "multiplicity":{
"lower":1, | "lower":1,
"upper":1, | "upper":1,
"isUnique":false | "isUnique":false
}, | },
"isComposite":false, | "isComposite":false,
"isUnique":false, | "isUnique":false,
"isIndexable":true, | "isIndexable":true,
"reverseAttributeName":null | "reverseAttributeName":null
} | },
] | {
}, | "name":"created",
"rows":[ | "dataTypeName":"date",
{ | "multiplicity":{
"$typeName$":"Table", | "lower":0,
"$id$":{ | "upper":1,
"id":"8960", | "isUnique":false
"$typeName$":"Table", | },
"version":0 | "isComposite":false,
}, | "isUnique":false,
"sd":{ | "isIndexable":true,
"id":"7424", | "reverseAttributeName":null
"$typeName$":"StorageDesc", | }
"version":0 | ]
}, | },
"db":{ | "rows":[
"id":"7168", | {
"$typeName$":"DB", | "$typeName$":"Table",
"version":0 | "$id$":{
}, | "$typeName$":"Table",
"name":"sales_fact_daily_mv" | "version":0
}, | },
{ | "created":"2014-12-11T02:35:58.440Z",
"$typeName$":"Table", | "sd":{
"$id$":{ | "$typeName$":"StorageDesc",
"id":"8960", | "version":0
"$typeName$":"Table", | },
"version":0 | "db":{
}, | "$typeName$":"DB",
"sd":{ | "version":0
"id":"7424", | },
"$typeName$":"StorageDesc", | "name":"sales_fact_daily_mv"
"version":0 | },
}, | {
"db":{ | "$typeName$":"Table",
"id":"7168", | "$id$":{
"$typeName$":"DB", | "$typeName$":"Table",
"version":0 | "version":0
}, | },
"name":"sales_fact_daily_mv" | "created":"2014-12-11T02:35:58.440Z",
}, | "sd":{
{ | "$typeName$":"StorageDesc",
"$typeName$":"Table", | "version":0
"$id$":{ | },
"id":"12800", | "db":{
"$typeName$":"Table", | "$typeName$":"DB",
"version":0 | "version":0
}, | },
"sd":{ | "name":"sales_fact_daily_mv"
"id":"11264", | },
"$typeName$":"StorageDesc", | {
"version":0 | "$typeName$":"Table",
}, | "$id$":{
"db":{ | "$typeName$":"Table",
"id":"7168", | "version":0
"$typeName$":"DB", | },
"version":0 | "created":"2014-12-11T02:35:58.440Z",
}, | "sd":{
"name":"sales_fact_monthly_mv" | "$typeName$":"StorageDesc",
} | "version":0
] | },
}""") | "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_monthly_mv"
| }
| ]
|}""".stripMargin)
} }
} }
\ No newline at end of file
repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
...@@ -28,7 +28,10 @@ import com.typesafe.config.{Config, ConfigFactory} ...@@ -28,7 +28,10 @@ import com.typesafe.config.{Config, ConfigFactory}
import org.apache.atlas.typesystem.types._ import org.apache.atlas.typesystem.types._
import org.apache.commons.configuration.{Configuration, ConfigurationException, MapConfiguration} import org.apache.commons.configuration.{Configuration, ConfigurationException, MapConfiguration}
import org.apache.commons.io.FileUtils import org.apache.commons.io.FileUtils
import org.json.JSONObject
import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite} import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite}
import org.skyscreamer.jsonassert.JSONAssert
trait GraphUtils { trait GraphUtils {
...@@ -96,7 +99,8 @@ object QueryTestsUtils extends GraphUtils { ...@@ -96,7 +99,8 @@ object QueryTestsUtils extends GraphUtils {
Array( Array(
attrDef("name", DataTypes.STRING_TYPE), attrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("db", "DB", Multiplicity.REQUIRED, false, null), new AttributeDefinition("db", "DB", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null) new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null),
attrDef("created", DataTypes.DATE_TYPE)
)) ))
def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null, def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null,
...@@ -160,7 +164,9 @@ trait BaseGremlinTest { ...@@ -160,7 +164,9 @@ trait BaseGremlinTest {
if (expected != null) { if (expected != null) {
val a = STRUCT_NAME_REGEX.replaceAllIn(rJ, "") val a = STRUCT_NAME_REGEX.replaceAllIn(rJ, "")
val b = STRUCT_NAME_REGEX.replaceAllIn(expected, "") val b = STRUCT_NAME_REGEX.replaceAllIn(expected, "")
Assertions.assert(a == b) val actualjsonObj = new JSONObject(a)
val expectedjsonObj = new JSONObject(b)
JSONAssert.assertEquals(expectedjsonObj, actualjsonObj, false)
} else { } else {
println(rJ) println(rJ)
} }
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
...@@ -48,7 +48,7 @@ public class TypeSystem { ...@@ -48,7 +48,7 @@ public class TypeSystem {
private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() { private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() {
@Override @Override
public SimpleDateFormat initialValue() { public SimpleDateFormat initialValue() {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX"); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
return dateFormat; return dateFormat;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment