Skip to content

A
atlas
Commit 6b33bcf6 by Shwetha GS
Options

ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)

parent 901aca9d
Showing with 902 additions and 419 deletions
pom.xml
......@@ -338,6 +338,7 @@
<spray.version>1.3.1</spray.version>
<guava.version>14.0</guava.version>
<fastutil.version>6.5.16</fastutil.version>
<guice.version>4.0</guice.version>
<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>
......@@ -563,25 +564,25 @@
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>4.0</version>
<version>${guice.version}</version>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-throwingproviders</artifactId>
<version>4.0</version>
<version>${guice.version}</version>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-multibindings</artifactId>
<version>4.0</version>
<version>${guice.version}</version>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
<version>4.0</version>
<version>${guice.version}</version>
</dependency>
<dependency>
......@@ -591,6 +592,13 @@
</dependency>
<dependency>
<groupId>org.skyscreamer</groupId>
<artifactId>jsonassert</artifactId>
<version>1.2.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.5</version>
......
release-log.txt
......@@ -4,6 +4,7 @@ Apache Atlas Release Notes
--trunk - unreleased
INCOMPATIBLE CHANGES:
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES:
ATLAS-31 ATLAS build fails with clean repo (suma.shivaprasad via shwethags)
......
repository/pom.xml
......@@ -55,6 +55,11 @@
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
......@@ -146,6 +151,11 @@
</dependency>
<dependency>
<groupId>org.skyscreamer</groupId>
<artifactId>jsonassert</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -857,7 +857,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
propertyValue = typedInstance.getDate(attributeInfo.name);
final Date dateVal = typedInstance.getDate(attributeInfo.name);
//Convert Property value to Long while persisting
propertyValue = dateVal.getTime();
}
addProperty(instanceVertex, vertexPropertyName, propertyValue);
}
......@@ -1180,7 +1182,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
typedInstance
.setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
} else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
typedInstance.setDate(attributeInfo.name, instanceVertex.<Date>getProperty(vertexPropertyName));
final Long dateVal = instanceVertex.<Long>getProperty(vertexPropertyName);
typedInstance.setDate(attributeInfo.name, new Date(dateVal));
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -65,7 +65,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
this.titanGraph = graphProvider.get();
/* Create the transaction for indexing.
* Commit/rollback is expected to be called from the caller.
*/
management = titanGraph.getManagementSystem();
initialize();
......@@ -257,7 +256,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
} else if (dataType == DataTypes.BIGDECIMAL_TYPE) {
return BigDecimal.class;
} else if (dataType == DataTypes.DATE_TYPE) {
return Date.class;
//Indexing with date converted to long as of now since Titan is yet to add support for Date type with mixed indexes
return Long.class;
}
......@@ -333,9 +333,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
}
private boolean checkIfMixedIndexApplicable(Class propertyClass) {
//TODO - Check why date types are failing in ES/Solr
if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class
|| propertyClass == Date.class) {
if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class) {
return false;
}
return true;
......
repository/src/main/scala/org/apache/atlas/query/Expressions.scala
......@@ -490,6 +490,8 @@ object Expressions {
def string(rawValue: Any) = literal(DataTypes.STRING_TYPE, rawValue)
def date(rawValue: Any) = literal(DataTypes.DATE_TYPE, rawValue)
case class ArithmeticExpression(symbol: String,
left: Expression,
right: Expression)
......@@ -598,7 +600,10 @@ object Expressions {
throw new UnresolvedException(this,
s"datatype. Can not resolve due to unresolved children")
}
if (left.dataType != DataTypes.STRING_TYPE || right.dataType != DataTypes.STRING_TYPE) {
if(left.dataType == DataTypes.DATE_TYPE) {
DataTypes.DATE_TYPE
} else if (left.dataType != DataTypes.STRING_TYPE || right.dataType != DataTypes.STRING_TYPE) {
TypeUtils.combinedType(left.dataType, right.dataType)
}
DataTypes.BOOLEAN_TYPE
......
repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
......@@ -18,6 +18,8 @@
package org.apache.atlas.query
import java.util.Date
import com.thinkaurelius.titan.core.TitanVertex
import com.tinkerpop.blueprints.Direction
import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException}
......@@ -304,6 +306,10 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
case x: FloatType => i.setFloat(aInfo.name, v.getProperty[java.lang.Float](fName))
case x: DoubleType => i.setDouble(aInfo.name, v.getProperty[java.lang.Double](fName))
case x: StringType => i.setString(aInfo.name, v.getProperty[java.lang.String](fName))
case x: DateType => {
val dateVal = v.getProperty[java.lang.Long](fName)
i.setDate(aInfo.name, new Date(dateVal))
}
case _ => throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
}
}
......
repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
......@@ -19,7 +19,9 @@
package org.apache.atlas.query
import org.apache.atlas.query.Expressions._
import org.apache.atlas.typesystem.types.{TypeSystem, DataTypes}
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory
import org.joda.time.format.ISODateTimeFormat
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
......@@ -227,12 +229,29 @@ class GremlinTranslator(expr: Expression,
}
}
case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => {
val QUOTE = "\"";
val fieldGremlinExpr = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}"
ch match {
case Some(child) => {
s"""${genQuery(child, inSelect)}.has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)"""
}
case None => s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)"""
case None => {
if (fInfo.attrInfo.dataType == DataTypes.DATE_TYPE) {
try {
//Accepts both date, datetime formats
val dateStr = l.toString.stripPrefix(QUOTE).stripSuffix(QUOTE)
val dateVal = ISODateTimeFormat.dateOptionalTimeParser().parseDateTime(dateStr).getMillis
s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)},${dateVal})"""
} catch {
case pe: java.text.ParseException =>
throw new GremlinTranslationException(c,
"Date format " + l + " not supported. Should be of the format " + TypeSystem.getInstance().getDateFormat.toPattern);
}
}
else
s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)"""
}
}
}
case fil@FilterExpression(child, condExpr) => {
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -169,7 +169,8 @@ public class GraphBackedDiscoveryServiceTest {
@DataProvider(name = "dslQueriesProvider")
private Object[][] createDSLQueries() {
return new String[][]{{"from DB"}, {"DB"}, {"DB where DB.name=\"Reporting\""}, {"DB DB.name = \"Reporting\""},
return new String[][]{
{"from DB"}, {"DB"}, {"DB where DB.name=\"Reporting\""}, {"DB DB.name = \"Reporting\""},
{"DB where DB.name=\"Reporting\" select name, owner"}, {"DB has name"}, {"DB, Table"},
{"DB is JdbcAccess"},
/*
......@@ -186,6 +187,8 @@ public class GraphBackedDiscoveryServiceTest {
{"DB where DB is JdbcAccess"}, {"DB where DB has name"}, {"DB Table"}, {"DB where DB has name"},
{"DB as db1 Table where (db1.name = \"Reporting\")"},
{"DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
{"Table where (name = \"sales_fact\" and created > \"2014-01-01\" ) select name as _col_0, created as _col_1 "},
{"Table where (name = \"sales_fact\" and created > \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, created as _col_1 "},
/*
todo: does not work
{"DB where (name = \"Reporting\") and ((createTime + 1) > 0)"},
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
......@@ -21,15 +21,20 @@ package org.apache.atlas.repository.graph;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanGraphQuery;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.diskstorage.BackendException;
import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration;
import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Predicate;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.BaseTest;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
......@@ -55,6 +60,7 @@ import org.testng.annotations.Test;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Map;
import java.util.Random;
......@@ -157,6 +163,7 @@ public class GraphRepoMapperScaleTest {
searchWithOutIndex("hive_table_type.name", "bar-999");
searchWithIndex("hive_table_type.name", "bar-999");
searchWithIndex("hive_table_type.created", Compare.GREATER_THAN_EQUAL, BaseTest.TEST_DATE_IN_LONG);
for (int index = 500; index < 600; index++) {
searchWithIndex("hive_table_type.name", "bar-" + index);
......@@ -190,6 +197,21 @@ public class GraphRepoMapperScaleTest {
}
} finally {
System.out.println("Search on [" + key + "=" + value + "] returned results: " + count + ", took " + (
System.currentTimeMillis() - start) + " ms");
}
}
private void searchWithIndex(String key, Predicate searchPredicate, Object value) {
TitanGraph graph = graphProvider.get();
long start = System.currentTimeMillis();
int count = 0;
try {
GraphQuery query = graph.query().has(key, searchPredicate, value);
for (Vertex ignored : query.vertices()) {
count++;
}
} finally {
System.out.println("Search on [" + key + "=" + value + "] returned results: " + count + ", took " + (
System.currentTimeMillis() - start) + " ms");
}
}
......@@ -222,6 +244,7 @@ public class GraphRepoMapperScaleTest {
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
// enum
new AttributeDefinition("tableType", "table_type", Multiplicity.REQUIRED, false, null),
// array of strings
......@@ -262,6 +285,7 @@ public class GraphRepoMapperScaleTest {
tableInstance.set("name", TABLE_NAME + "-" + uberIndex);
tableInstance.set("description", "bar table" + "-" + uberIndex);
tableInstance.set("type", "managed");
tableInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
tableInstance.set("tableType", 1); // enum
// refer to an existing class
......
repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
......@@ -43,7 +43,7 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
test("testClass") {
val r = QueryProcessor.evaluate(_class("DB"), g)
validateJson(r, "{\n \"query\":\"DB\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
validateJson(r, "{\n \"query\":\"DB\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
}
test("testName") {
......@@ -53,12 +53,12 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
test("testFilter") {
var r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))), g)
validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
}
test("testFilter2") {
var r = QueryProcessor.evaluate(_class("DB").where(id("DB").field("name").`=`(string("Reporting"))), g)
validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
}
......@@ -70,17 +70,400 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
test("testIsTrait") {
val r = QueryProcessor.evaluate(_class("Table").where(isTrait("Dimension")), g)
validateJson(r, "{\n \"query\":\"Table where Table is Dimension\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dataTypeName\":\"StorageDesc\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n }\n ]\n}")
validateJson(r, """{
| "query":"Table where Table is Dimension",
| "dataType":{
| "superTypes":[
|
| ],
| "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
| "typeName":"Table",
| "attributeDefinitions":[
| {
| "name":"name",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"db",
| "dataTypeName":"DB",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"sd",
| "dataTypeName":"StorageDesc",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"created",
| "dataTypeName":"date",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| }
| ]
| },
| "rows":[
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"product_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"time_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"customer_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| }
| ]
|}""".stripMargin)
}
test("testhasField") {
val r = QueryProcessor.evaluate(_class("DB").where(hasField("name")), g)
validateJson(r, "{\n \"query\":\"DB where DB has name\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
validateJson(r, """{
| "query":"DB where DB has name",
| "dataType":{
| "superTypes":[
|
| ],
| "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
| "typeName":"DB",
| "attributeDefinitions":[
| {
| "name":"name",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"owner",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"createTime",
| "dataTypeName":"int",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| }
| ]
| },
| "rows":[
| {
| "$typeName$":"DB",
| "$id$":{
| "$typeName$":"DB",
| "version":0
| },
| "owner":"John ETL",
| "name":"Sales",
| "createTime":1000
| },
| {
| "$typeName$":"DB",
| "$id$":{
| "$typeName$":"DB",
| "version":0
| },
| "owner":"Jane BI",
| "name":"Reporting",
| "createTime":1500
| }
| ]
|}""".stripMargin)
}
test("testFieldReference") {
val r = QueryProcessor.evaluate(_class("DB").field("Table"), g)
validateJson(r, "{\n \"query\":\"DB Table\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dataTypeName\":\"StorageDesc\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"2048\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"512\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"8960\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"7424\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"12800\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"11264\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
validateJson(r, """{
| "query":"DB Table",
| "dataType":{
| "superTypes":[ ],
| "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
| "typeName":"Table",
| "attributeDefinitions":[
| {
| "name":"name",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"db",
| "dataTypeName":"DB",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"sd",
| "dataTypeName":"StorageDesc",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"created",
| "dataTypeName":"date",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| }
| ]
| },
| "rows":[
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"product_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"time_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"customer_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_daily_mv"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_monthly_mv"
| }
| ]
|}""".stripMargin);
}
test("testBackReference") {
......@@ -98,7 +481,7 @@ class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
test("testComparisonLogical") {
val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting")).
and(id("createTime") > int(0))), g)
validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") and (createTime > 0)\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") and (createTime > 0)\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7424\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
}
test("testJoinAndSelect1") {
......
repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala
......@@ -19,12 +19,14 @@
package org.apache.atlas.query
import java.io.File
import java.util.UUID
import java.util.{Date, UUID}
import java.util.concurrent.atomic.AtomicInteger
import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
import com.thinkaurelius.titan.core.TitanGraph
import com.typesafe.config.ConfigFactory
import org.apache.atlas.repository.BaseTest
import org.apache.atlas.typesystem.types.TypeSystem
import org.apache.commons.io.FileUtils
import scala.collection.mutable.ArrayBuffer
......@@ -54,18 +56,19 @@ object HiveTitanSample {
this.getClass.getDeclaredFields filter (_.getName != "traits") foreach { f =>
f.setAccessible(true)
var fV = f.get(this)
fV = fV match {
val fV = f.get(this)
val convertedVal = fV match {
case _: String => s""""$fV""""
case d: Date => d.getTime
case _ => fV
}
fV match {
convertedVal match {
case x: Vertex => addEdge(x, s"${this.getClass.getSimpleName}.${f.getName}", edges)
case l: List[_] => l.foreach(x => addEdge(x.asInstanceOf[Vertex],
s"${this.getClass.getSimpleName}.${f.getName}", edges))
case _ => sb.append( s""", "${f.getName}" : $fV""")
sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $fV""")
case _ => sb.append( s""", "${f.getName}" : $convertedVal""")
sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $convertedVal""")
}
}
......@@ -127,17 +130,23 @@ object HiveTitanSample {
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance
case class Table(name: String, db: DB, sd: StorageDescriptor,
created: Date,
traits: Option[List[Trait]] = None,
_id: String = "" + nextVertexId.incrementAndGet()) extends Instance
case class TableDef(name: String, db: DB, inputFormat: String, outputFormat: String,
columns: List[(String, String, Option[List[Trait]])],
traits: Option[List[Trait]] = None) {
traits: Option[List[Trait]] = None,
created: Option[Date] = None) {
val createdDate : Date = created match {
case Some(x) => x
case None => new Date(BaseTest.TEST_DATE_IN_LONG)
}
val sd = StorageDescriptor(inputFormat, outputFormat)
val colDefs = columns map { c =>
Column(c._1, c._2, sd, c._3)
}
val tablDef = Table(name, db, sd, traits)
val tablDef = Table(name, db, sd, createdDate, traits)
def toGSon(vertices: ArrayBuffer[String],
edges: ArrayBuffer[String]): Unit = {
......@@ -167,6 +176,7 @@ object HiveTitanSample {
("time_id", "int", None),
("product_id", "int", None),
("customer_id", "int", None),
("created", "date", None),
("sales", "double", Some(List(Metric())))
))
val productDim = TableDef("product_dim",
......@@ -269,6 +279,7 @@ object HiveTitanSample {
FileUtils.writeStringToFile(new File(fileName), toGSon())
}
val GremlinQueries = List(
// 1. List all DBs
"""g.V.has("typeName", "DB")""",
......
repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
......@@ -26,7 +26,7 @@ import org.scalatest.junit.JUnitRunner
import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite}
@RunWith(classOf[JUnitRunner])
class LineageQueryTest extends FunSuite with BeforeAndAfterAll {
class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
var g: TitanGraph = null
......@@ -40,140 +40,134 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll {
g.shutdown()
}
val STRUCT_NAME_REGEX = (TypeUtils.TEMP_STRUCT_NAME_PREFIX + "\\d+").r
val PREFIX_SPACES_REGEX = ("\\n\\s*").r
def validateJson(r: GremlinQueryResult, expected: String = null): Unit = {
val rJ = r.toJson
if (expected != null) {
var a = STRUCT_NAME_REGEX.replaceAllIn(rJ, "")
a = PREFIX_SPACES_REGEX.replaceAllIn(a, "")
var b = STRUCT_NAME_REGEX.replaceAllIn(expected, "")
b = PREFIX_SPACES_REGEX.replaceAllIn(b, "")
Assertions.assert(a == b)
} else {
println(rJ)
}
}
test("testInputTables") {
val r = QueryProcessor.evaluate(_class("LoadProcess").field("inputTables"), g)
val x = r.toJson
validateJson(r, """{
"query":"LoadProcess inputTables",
"dataType":{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
"typeName":"Table",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":{
"lower":0,
"upper":1,
"isUnique":false
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"db",
"dataTypeName":"DB",
"multiplicity":{
"lower":1,
"upper":1,
"isUnique":false
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"sd",
"dataTypeName":"StorageDesc",
"multiplicity":{
"lower":1,
"upper":1,
"isUnique":false
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
"rows":[
{
"$typeName$":"Table",
"$id$":{
"id":"2048",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"512",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"256",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact"
},
{
"$typeName$":"Table",
"$id$":{
"id":"4864",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"3840",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"256",
"$typeName$":"DB",
"version":0
},
"name":"time_dim",
"$traits$":{
"Dimension":{
"$typeName$":"Dimension"
}
}
},
{
"$typeName$":"Table",
"$id$":{
"id":"8960",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"7424",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"7168",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact_daily_mv"
}
]
}""")
validateJson(r,"""{
| "query":"LoadProcess inputTables",
| "dataType":{
| "superTypes":[
|
| ],
| "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
| "typeName":"Table",
| "attributeDefinitions":[
| {
| "name":"name",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"db",
| "dataTypeName":"DB",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"sd",
| "dataTypeName":"StorageDesc",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"created",
| "dataTypeName":"date",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| }
| ]
| },
| "rows":[
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"time_dim",
| "$traits$":{
| "Dimension":{
| "$typeName$":"Dimension"
| }
| }
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_daily_mv"
| }
| ]
|}
""".stripMargin)
}
test("testLoadProcessOut") {
......@@ -184,153 +178,157 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll {
test("testLineageAll") {
val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable")), g)
validateJson(r, """{
"query":"Table as _loop0 loop (LoadProcess outputTable)",
"dataType":{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
"typeName":"Table",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":{
"lower":0,
"upper":1,
"isUnique":false
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"db",
"dataTypeName":"DB",
"multiplicity":{
"lower":1,
"upper":1,
"isUnique":false
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"sd",
"dataTypeName":"StorageDesc",
"multiplicity":{
"lower":1,
"upper":1,
"isUnique":false
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
"rows":[
{
"$typeName$":"Table",
"$id$":{
"id":"8960",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"7424",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"7168",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact_daily_mv"
},
{
"$typeName$":"Table",
"$id$":{
"id":"12800",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"11264",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"7168",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact_monthly_mv"
},
{
"$typeName$":"Table",
"$id$":{
"id":"8960",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"7424",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"7168",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact_daily_mv"
},
{
"$typeName$":"Table",
"$id$":{
"id":"12800",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"11264",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"7168",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact_monthly_mv"
},
{
"$typeName$":"Table",
"$id$":{
"id":"12800",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"11264",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"7168",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact_monthly_mv"
}
]
}""")
| "query":"Table as _loop0 loop (LoadProcess outputTable)",
| "dataType":{
| "superTypes":[
|
| ],
| "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
| "typeName":"Table",
| "attributeDefinitions":[
| {
| "name":"name",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"db",
| "dataTypeName":"DB",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"sd",
| "dataTypeName":"StorageDesc",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"created",
| "dataTypeName":"date",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| }
| ]
| },
| "rows":[
| {
| "$typeName$":"Table",
| "$id$":{
| "id":"9216",
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_daily_mv"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_monthly_mv"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_daily_mv"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_monthly_mv"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_monthly_mv"
| }
| ]
|}""".stripMargin)
}
test("testLineageAllSelect") {
......@@ -396,120 +394,127 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll {
"destTable":"sales_fact_monthly_mv"
}
]
}""")
}""".stripMargin)
}
test("testLineageFixedDepth") {
val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable"), int(1)), g)
validateJson(r, """{
"query":"Table as _loop0 loop (LoadProcess outputTable) times 1",
"dataType":{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
"typeName":"Table",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":{
"lower":0,
"upper":1,
"isUnique":false
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"db",
"dataTypeName":"DB",
"multiplicity":{
"lower":1,
"upper":1,
"isUnique":false
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"sd",
"dataTypeName":"StorageDesc",
"multiplicity":{
"lower":1,
"upper":1,
"isUnique":false
},
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
"rows":[
{
"$typeName$":"Table",
"$id$":{
"id":"8960",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"7424",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"7168",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact_daily_mv"
},
{
"$typeName$":"Table",
"$id$":{
"id":"8960",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"7424",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"7168",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact_daily_mv"
},
{
"$typeName$":"Table",
"$id$":{
"id":"12800",
"$typeName$":"Table",
"version":0
},
"sd":{
"id":"11264",
"$typeName$":"StorageDesc",
"version":0
},
"db":{
"id":"7168",
"$typeName$":"DB",
"version":0
},
"name":"sales_fact_monthly_mv"
}
]
}""")
| "query":"Table as _loop0 loop (LoadProcess outputTable) times 1",
| "dataType":{
| "superTypes":[
|
| ],
| "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
| "typeName":"Table",
| "attributeDefinitions":[
| {
| "name":"name",
| "dataTypeName":"string",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"db",
| "dataTypeName":"DB",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"sd",
| "dataTypeName":"StorageDesc",
| "multiplicity":{
| "lower":1,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| },
| {
| "name":"created",
| "dataTypeName":"date",
| "multiplicity":{
| "lower":0,
| "upper":1,
| "isUnique":false
| },
| "isComposite":false,
| "isUnique":false,
| "isIndexable":true,
| "reverseAttributeName":null
| }
| ]
| },
| "rows":[
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_daily_mv"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_daily_mv"
| },
| {
| "$typeName$":"Table",
| "$id$":{
| "$typeName$":"Table",
| "version":0
| },
| "created":"2014-12-11T02:35:58.440Z",
| "sd":{
| "$typeName$":"StorageDesc",
| "version":0
| },
| "db":{
| "$typeName$":"DB",
| "version":0
| },
| "name":"sales_fact_monthly_mv"
| }
| ]
|}""".stripMargin)
}
}
\ No newline at end of file
repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
......@@ -28,7 +28,10 @@ import com.typesafe.config.{Config, ConfigFactory}
import org.apache.atlas.typesystem.types._
import org.apache.commons.configuration.{Configuration, ConfigurationException, MapConfiguration}
import org.apache.commons.io.FileUtils
import org.json.JSONObject
import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite}
import org.skyscreamer.jsonassert.JSONAssert
trait GraphUtils {
......@@ -96,7 +99,8 @@ object QueryTestsUtils extends GraphUtils {
Array(
attrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("db", "DB", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null)
new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null),
attrDef("created", DataTypes.DATE_TYPE)
))
def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null,
......@@ -160,7 +164,9 @@ trait BaseGremlinTest {
if (expected != null) {
val a = STRUCT_NAME_REGEX.replaceAllIn(rJ, "")
val b = STRUCT_NAME_REGEX.replaceAllIn(expected, "")
Assertions.assert(a == b)
val actualjsonObj = new JSONObject(a)
val expectedjsonObj = new JSONObject(b)
JSONAssert.assertEquals(expectedjsonObj, actualjsonObj, false)
} else {
println(rJ)
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -48,7 +48,7 @@ public class TypeSystem {
private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() {
@Override
public SimpleDateFormat initialValue() {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
return dateFormat;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment