Skip to content

A
atlas
Commit 64de5233 by Shwetha GS
Options

ATLAS-106 Store createTimestamp and modified timestamp separately for an entity…

ATLAS-106 Store createTimestamp and modified timestamp separately for an entity (dkantor via shwethags)
parent d519ae8c
Showing with 106 additions and 19 deletions
release-log.txt
......@@ -7,6 +7,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-106 Store createTimestamp and modified timestamp separately for an entity (dkantor via shwethags)
ATLAS-433 Fix checkstyle issues for common and notification module (shwethags)
ATLAS-183 Add a Hook in Storm to post the topology metadata (svenkat,yhemanth via shwethags)
ATLAS-370 Implement deleteEntities at repository level (dkantor via shwethags)
......
repository/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -57,6 +57,8 @@ public final class Constants {
public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version";
public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp";
/**
* search backing index name.
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
......@@ -224,7 +224,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
// update the traits in entity once adding trait instance is successful
GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis()));
} catch (RepositoryException e) {
throw e;
} catch (Exception e) {
......@@ -285,6 +286,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
for (String traitName : traitNames) {
GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
}
GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis()));
}
@Override
......
repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
......@@ -21,7 +21,6 @@ import com.thinkaurelius.titan.core.SchemaViolationException;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
......@@ -166,8 +165,9 @@ public final class TypedInstanceToGraphMapper {
if (Operation.CREATE.equals(operation)) {
//TODO - Handle Trait updates
addTraits(typedInstance, instanceVertex, classType);
} else if (Operation.UPDATE_FULL.equals(operation) || Operation.UPDATE_PARTIAL.equals(operation)) {
GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis()));
}
return getId(typedInstance)._getId();
}
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -24,10 +24,12 @@ import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
......@@ -43,6 +45,10 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
......@@ -71,6 +77,13 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
repositoryService.createEntities(hrDept2);
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
Id janeGuid = jane.getId();
ClassType personType = typeSystem.getDataType(ClassType.class, "Person");
ITypedReferenceableInstance instance = personType.createInstance(janeGuid);
instance.set("orgLevel", "L1");
repositoryService.updateEntities(instance);
}
@AfterClass
......@@ -115,15 +128,45 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
public void testRawSearch1() throws Exception {
// Query for all Vertices in Graph
Object r = discoveryService.searchByGremlin("g.V.toList()");
Assert.assertTrue(r instanceof List);
List<Map<String, Object>> resultList = (List<Map<String, Object>>) r;
Assert.assertTrue(resultList.size() > 0);
System.out.println("search result = " + r);
// Query for all Vertices of a Type
r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Department'}.toList()");
r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.ENTITY_TYPE_PROPERTY_KEY + " == 'Department'}.toList()");
Assert.assertTrue(r instanceof List);
resultList = (List<Map<String, Object>>) r;
Assert.assertTrue(resultList.size() > 0);
System.out.println("search result = " + r);
// Property Query: list all Person names
r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()");
r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.ENTITY_TYPE_PROPERTY_KEY + " == 'Person'}.'Person.name'.toList()");
Assert.assertTrue(r instanceof List);
resultList = (List<Map<String, Object>>) r;
Assert.assertTrue(resultList.size() > 0);
System.out.println("search result = " + r);
List<Object> names = new ArrayList<Object>(resultList.size());
for (Map<String, Object> vertexProps : resultList) {
names.addAll(vertexProps.values());
}
for (String name : Arrays.asList("John", "Max")) {
Assert.assertTrue(names.contains(name));
}
// Query for all Vertices modified after 01/01/2015 00:00:00 GMT
r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY + " > 1420070400000}.toList()");
Assert.assertTrue(r instanceof List);
resultList = (List<Map<String, Object>>) r;
Assert.assertTrue(resultList.size() > 0);
for (Map<String, Object> vertexProps : resultList) {
Object object = vertexProps.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(object);
Long timestampAsLong = Long.valueOf((String)object);
Assert.assertTrue(timestampAsLong > 1420070400000L);
object = vertexProps.get(Constants.TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(object);
}
}
@DataProvider(name = "dslQueriesProvider")
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
......@@ -18,11 +18,9 @@
package org.apache.atlas.repository.graph;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
......@@ -40,9 +38,9 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Vertex;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.List;
/**
* Test for GraphBackedMetadataRepository.deleteEntities
......@@ -102,7 +100,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
Assert.assertTrue(refValue instanceof List);
List<Object> employees = (List<Object>)refValue;
Assert.assertEquals(employees.size(), 4);
List<String> employeeGuids = new ArrayList<String>(4);
List<String> employeeGuids = new ArrayList(4);
for (Object listValue : employees) {
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
......@@ -123,6 +122,7 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
for (String employeeGuid : employeeGuids) {
verifyEntityDoesNotExist(employeeGuid);
}
// Verify all Person.address struct vertices were removed.
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
Assert.assertEquals(vertexCount, 0);
......@@ -159,11 +159,11 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
List<String> guids = repositoryService.createEntities(hrDept2);
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 5);
List<String> entityList = repositoryService.getEntityList("Department");
Assert.assertNotNull(entityList);
Assert.assertEquals(entityList.size(), 1);
String hrDeptGuid = entityList.get(0);
return hrDeptGuid;
return entityList.get(0);
}
private int countVertices(String propertyName, Object value) {
......@@ -176,12 +176,10 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
}
private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException {
try {
repositoryService.getEntityDefinition(hrDeptGuid);
Assert.fail("EntityNotFoundException was expected but none thrown");
}
catch(EntityNotFoundException e) {
} catch(EntityNotFoundException e) {
// good
}
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -24,6 +24,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
......@@ -52,9 +53,11 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import scala.actors.threadpool.Arrays;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
......@@ -237,6 +240,9 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testGetTraitNames")
public void testAddTrait() throws Exception {
final String aGUID = getGUID();
Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNull(modificationTimestampPreUpdate);
List<String> traitNames = repositoryService.getTraitNames(aGUID);
System.out.println("traitNames = " + traitNames);
......@@ -254,6 +260,11 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertEquals(traitNames.size(), 2);
Assert.assertTrue(traitNames.contains(TestUtils.PII));
Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION));
// Verify modification timestamp was updated.
GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(modificationTimestampPostUpdate);
}
@Test(dependsOnMethods = "testAddTrait")
......@@ -301,6 +312,9 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testAddTrait")
public void testDeleteTrait() throws Exception {
final String aGUID = getGUID();
Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(modificationTimestampPreUpdate);
List<String> traitNames = repositoryService.getTraitNames(aGUID);
Assert.assertEquals(traitNames.size(), 3);
......@@ -315,6 +329,12 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertEquals(traitNames.size(), 2);
Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION));
Assert.assertFalse(traitNames.contains(TestUtils.PII));
// Verify modification timestamp was updated.
GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(modificationTimestampPostUpdate);
Assert.assertTrue(modificationTimestampPostUpdate > modificationTimestampPreUpdate);
}
@Test(expectedExceptions = RepositoryException.class)
......@@ -479,8 +499,16 @@ public class GraphBackedMetadataRepositoryTest {
public void testUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Person", "name", "John");
Id johnGuid = john.getId();
ITypedReferenceableInstance max = repositoryService.getEntityDefinition("Person", "name", "Max");
String maxGuid = max.getId()._getId();
Vertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long creationTimestamp = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(creationTimestamp);
Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNull(modificationTimestampPreUpdate);
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
Id janeGuid = jane.getId();
......@@ -497,6 +525,13 @@ public class GraphBackedMetadataRepositoryTest {
ITypedReferenceableInstance refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), johnGuid._getId());
// Verify modification timestamp was updated.
vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(modificationTimestampPostUpdate);
Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate);
// Update max's mentor reference to jane.
instance = personType.createInstance(max.getId());
instance.set("mentor", janeGuid);
......@@ -508,6 +543,12 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId());
// Verify modification timestamp was updated.
vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(modificationTimestampPost2ndUpdate);
Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate);
}
private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment