Skip to content

A
atlas
Commit bfd5f5ca by Shwetha GS
Options

ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags)

parent 91559578
Showing with 106 additions and 34 deletions
release-log.txt
......@@ -20,6 +20,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags)
ATLAS-692 Create abstraction layer for graph databases (jnhagelb via yhemanth)
ATLAS-689 Migrate Atlas-Storm integration to use Storm 1.0 dependencies. (svimal2106 via yhemanth)
ATLAS-754 InstanceSerialization does not serialize Reference in the values array of Reference.(harishjp via sumasai)
......
repository/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -18,6 +18,10 @@
package org.apache.atlas.repository;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
public final class Constants {
/**
......@@ -62,6 +66,18 @@ public final class Constants {
public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp";
public static AttributeInfo getAttributeInfoForSystemAttributes(String field) {
switch (field) {
case STATE_PROPERTY_KEY:
case GUID_PROPERTY_KEY:
return TypesUtil.newAttributeInfo(field, DataTypes.STRING_TYPE);
case TIMESTAMP_PROPERTY_KEY:
case MODIFICATION_TIMESTAMP_PROPERTY_KEY:
return TypesUtil.newAttributeInfo(field, DataTypes.LONG_TYPE);
}
return null;
}
/**
* search backing index name.
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -105,6 +105,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException {
if (aInfo.name.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX)) {
return aInfo.name;
}
return GraphHelper.getQualifiedFieldName(dataType, aInfo.name);
}
......
repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
......@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger
import org.apache.atlas.AtlasException
import org.apache.atlas.query.Expressions.{PathExpression, SelectExpression}
import org.apache.atlas.repository.Constants
import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory}
import org.apache.atlas.typesystem.types._
......@@ -203,6 +204,11 @@ object TypeUtils {
return Some(FieldInfo(typ,fMap.get.fields.get(id)))
}
val systemField = Constants.getAttributeInfoForSystemAttributes(id)
if (systemField != null) {
return Some(FieldInfo(systemField.dataType(), systemField))
}
try {
val FIELD_QUALIFIER(clsNm, rest) = id
val idTyp = typSystem.getDataType(classOf[IDataType[_]], clsNm)
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -56,6 +56,8 @@ import java.util.Map;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
......@@ -94,32 +96,62 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
}
@Test
public void testSearchBySystemProperties() throws Exception {
//system property in select
String dslQuery = "from Department select __guid";
String jsonResults = discoveryService.searchByDSL(dslQuery);
assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
assertEquals(results.length(), 3);
JSONArray rows = results.getJSONArray("rows");
assertNotNull(rows);
assertEquals(rows.length(), 1);
assertNotNull(rows.getJSONObject(0).getString("__guid"));
//system property in where clause
String guid = rows.getJSONObject(0).getString("__guid");
dslQuery = "Department where __guid = '" + guid + "' and __state = 'ACTIVE'";
jsonResults = discoveryService.searchByDSL(dslQuery);
assertNotNull(jsonResults);
results = new JSONObject(jsonResults);
assertEquals(results.length(), 3);
rows = results.getJSONArray("rows");
assertNotNull(rows);
assertEquals(rows.length(), 1);
}
@Test
public void testSearchByDSLReturnsEntity() throws Exception {
String dslQuery = "from Department";
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
Assert.assertEquals(results.length(), 3);
assertEquals(results.length(), 3);
System.out.println("results = " + results);
Object query = results.get("query");
Assert.assertNotNull(query);
assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
Assert.assertNotNull(dataType);
assertNotNull(dataType);
String typeName = dataType.getString("typeName");
Assert.assertNotNull(typeName);
Assert.assertEquals(typeName, "Department");
assertNotNull(typeName);
assertEquals(typeName, "Department");
JSONArray rows = results.getJSONArray("rows");
Assert.assertNotNull(rows);
Assert.assertEquals(rows.length(), 1);
assertNotNull(rows);
assertEquals(rows.length(), 1);
//Assert that entity state is set in the result entities
String entityState = rows.getJSONObject(0).getJSONObject("$id$").getString("state");
Assert.assertEquals(entityState, Id.EntityState.ACTIVE.name());
assertEquals(entityState, Id.EntityState.ACTIVE.name());
}
@Test(expectedExceptions = Throwable.class)
......@@ -167,11 +199,11 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
Assert.assertTrue(resultList.size() > 0);
for (Map<String, Object> vertexProps : resultList) {
Object object = vertexProps.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(object);
assertNotNull(object);
Long timestampAsLong = Long.valueOf((String)object);
Assert.assertTrue(timestampAsLong > 1420070400000L);
object = vertexProps.get(Constants.TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(object);
assertNotNull(object);
}
}
......@@ -490,23 +522,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
public void testSearchByDSLQueriesWithOrderBy(String dslQuery, Integer expectedNumRows, String orderBy, boolean ascending) throws Exception {
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
Assert.assertEquals(results.length(), 3);
assertEquals(results.length(), 3);
Object query = results.get("query");
Assert.assertNotNull(query);
assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
Assert.assertNotNull(dataType);
assertNotNull(dataType);
String typeName = dataType.getString("typeName");
Assert.assertNotNull(typeName);
assertNotNull(typeName);
JSONArray rows = results.getJSONArray("rows");
Assert.assertNotNull(rows);
Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
assertNotNull(rows);
assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
List<String> returnedList = new ArrayList<String> ();
for (int i = 0; i < rows.length(); i++) {
JSONObject row = rows.getJSONObject(i);
......@@ -546,23 +578,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
public void testSearchByDSLQueries(String dslQuery, Integer expectedNumRows) throws Exception {
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
Assert.assertEquals(results.length(), 3);
assertEquals(results.length(), 3);
System.out.println("results = " + results);
Object query = results.get("query");
Assert.assertNotNull(query);
assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
Assert.assertNotNull(dataType);
assertNotNull(dataType);
String typeName = dataType.getString("typeName");
Assert.assertNotNull(typeName);
assertNotNull(typeName);
JSONArray rows = results.getJSONArray("rows");
Assert.assertNotNull(rows);
Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
assertNotNull(rows);
assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
}
......@@ -570,23 +602,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
public void testSearchByDSLQueriesWithLimit(String dslQuery, Integer expectedNumRows) throws Exception {
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
Assert.assertEquals(results.length(), 3);
assertEquals(results.length(), 3);
System.out.println("results = " + results);
Object query = results.get("query");
Assert.assertNotNull(query);
assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
Assert.assertNotNull(dataType);
assertNotNull(dataType);
String typeName = dataType.getString("typeName");
Assert.assertNotNull(typeName);
assertNotNull(typeName);
JSONArray rows = results.getJSONArray("rows");
Assert.assertNotNull(rows);
Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
assertNotNull(rows);
assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
}
......@@ -609,7 +641,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
String dslQuery = "from D where a = 1";
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
System.out.println("results = " + results);
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
......@@ -38,7 +38,7 @@ public class AttributeInfo {
public final String reverseAttributeName;
private IDataType dataType;
AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes) throws AtlasException {
public AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes) throws AtlasException {
this.name = def.name;
this.dataType =
(tempTypes != null && tempTypes.containsKey(def.dataTypeName)) ? tempTypes.get(def.dataTypeName) :
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
......@@ -21,8 +21,10 @@ package org.apache.atlas.typesystem.types.utils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
......@@ -32,6 +34,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import scala.collection.JavaConversions;
/**
......@@ -100,4 +103,15 @@ public class TypesUtil {
return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs),
JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes));
}
private static final TypeSystem ts = TypeSystem.getInstance();
public static AttributeInfo newAttributeInfo(String attribute, IDataType type) {
try {
return new AttributeInfo(ts, new AttributeDefinition(attribute, type.getName(), Multiplicity.REQUIRED,
false, null), null);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment