ATLAS-106 Store createTimestamp and modified timestamp separately for an entity…

release-log.txt

@@ -7,6 +7,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-106 Store createTimestamp and modified timestamp separately for an entity (dkantor via shwethags)
 ATLAS-433 Fix checkstyle issues for common and notification module (shwethags)
 ATLAS-183 Add a Hook in Storm to post the topology metadata (svenkat,yhemanth via shwethags)
 ATLAS-370 Implement deleteEntities at repository level (dkantor via shwethags)

repository/src/main/java/org/apache/atlas/repository/Constants.java

@@ -57,6 +57,8 @@ public final class Constants {
 
     public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version";
     public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
+    public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp";
+
 
     /**
      * search backing index name.

repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java

@@ -224,7 +224,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
 
             // update the traits in entity once adding trait instance is successful
             GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
-
+            GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis()));
+            
         } catch (RepositoryException e) {
             throw e;
         } catch (Exception e) {
@@ -285,6 +286,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
         for (String traitName : traitNames) {
             GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
         }
+        GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis()));
     }
 
     @Override

repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java

@@ -21,7 +21,6 @@ import com.thinkaurelius.titan.core.SchemaViolationException;
 import com.tinkerpop.blueprints.Direction;
 import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.Vertex;
-
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.RepositoryException;
@@ -166,8 +165,9 @@ public final class TypedInstanceToGraphMapper {
         if (Operation.CREATE.equals(operation)) {
             //TODO - Handle Trait updates
             addTraits(typedInstance, instanceVertex, classType);
+        } else if (Operation.UPDATE_FULL.equals(operation) || Operation.UPDATE_PARTIAL.equals(operation)) {
+            GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis()));
         }
-
         return getId(typedInstance)._getId();
     }
 

repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java

@@ -24,10 +24,12 @@ import org.apache.atlas.BaseHiveRepositoryTest;
 import org.apache.atlas.RepositoryMetadataModule;
 import org.apache.atlas.TestUtils;
 import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.MetadataRepository;
 import org.apache.atlas.repository.graph.GraphProvider;
 import org.apache.atlas.typesystem.ITypedReferenceableInstance;
 import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.persistence.Id;
 import org.apache.atlas.typesystem.types.ClassType;
 import org.apache.atlas.typesystem.types.DataTypes;
 import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
@@ -43,6 +45,10 @@ import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
 import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
 
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
@@ -71,6 +77,13 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
         ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
 
         repositoryService.createEntities(hrDept2);
+        
+        ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
+        Id janeGuid = jane.getId();
+        ClassType personType = typeSystem.getDataType(ClassType.class, "Person");
+        ITypedReferenceableInstance instance = personType.createInstance(janeGuid);
+        instance.set("orgLevel", "L1");
+        repositoryService.updateEntities(instance);
     }
 
     @AfterClass
@@ -115,15 +128,45 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
     public void testRawSearch1() throws Exception {
         // Query for all Vertices in Graph
         Object r = discoveryService.searchByGremlin("g.V.toList()");
+        Assert.assertTrue(r instanceof List);
+        List<Map<String, Object>> resultList = (List<Map<String, Object>>) r;
+        Assert.assertTrue(resultList.size() > 0);
         System.out.println("search result = " + r);
 
         // Query for all Vertices of a Type
-        r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Department'}.toList()");
+        r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.ENTITY_TYPE_PROPERTY_KEY + " == 'Department'}.toList()");
+        Assert.assertTrue(r instanceof List);
+        resultList = (List<Map<String, Object>>) r;
+        Assert.assertTrue(resultList.size() > 0);
         System.out.println("search result = " + r);
 
         // Property Query: list all Person names
-        r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()");
+        r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.ENTITY_TYPE_PROPERTY_KEY + " == 'Person'}.'Person.name'.toList()");
+        Assert.assertTrue(r instanceof List);
+        resultList = (List<Map<String, Object>>) r;
+        Assert.assertTrue(resultList.size() > 0);
         System.out.println("search result = " + r);
+        List<Object> names = new ArrayList<Object>(resultList.size());
+        for (Map<String, Object> vertexProps : resultList) {
+            names.addAll(vertexProps.values());
+        }
+        for (String name : Arrays.asList("John", "Max")) {
+            Assert.assertTrue(names.contains(name));
+        }
+        
+        // Query for all Vertices modified after 01/01/2015 00:00:00 GMT
+        r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY + " > 1420070400000}.toList()");
+        Assert.assertTrue(r instanceof List);
+        resultList = (List<Map<String, Object>>) r;
+        Assert.assertTrue(resultList.size() > 0);
+        for (Map<String, Object> vertexProps : resultList) {
+            Object object = vertexProps.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+            Assert.assertNotNull(object);
+            Long timestampAsLong = Long.valueOf((String)object);
+            Assert.assertTrue(timestampAsLong > 1420070400000L);
+            object = vertexProps.get(Constants.TIMESTAMP_PROPERTY_KEY);
+            Assert.assertNotNull(object);
+        }
     }
 
     @DataProvider(name = "dslQueriesProvider")

repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java

@@ -18,11 +18,9 @@
 
 package org.apache.atlas.repository.graph;
 
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.inject.Inject;
-
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+import com.tinkerpop.blueprints.Vertex;
 import org.apache.atlas.RepositoryMetadataModule;
 import org.apache.atlas.TestUtils;
 import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
@@ -40,9 +38,9 @@ import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.util.TitanCleanup;
-import com.tinkerpop.blueprints.Vertex;
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Test for GraphBackedMetadataRepository.deleteEntities
@@ -102,7 +100,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
         Assert.assertTrue(refValue instanceof List);
         List<Object> employees = (List<Object>)refValue;
         Assert.assertEquals(employees.size(), 4);
-        List<String> employeeGuids = new ArrayList<String>(4);
+
+        List<String> employeeGuids = new ArrayList(4);
         for (Object listValue : employees) {
             Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
             ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
@@ -123,6 +122,7 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
         for (String employeeGuid : employeeGuids) {
             verifyEntityDoesNotExist(employeeGuid);
         }
+
         // Verify all Person.address struct vertices were removed.
         vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
         Assert.assertEquals(vertexCount, 0);
@@ -159,11 +159,11 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
         List<String> guids = repositoryService.createEntities(hrDept2);
         Assert.assertNotNull(guids);
         Assert.assertEquals(guids.size(), 5);
+
         List<String> entityList = repositoryService.getEntityList("Department");
         Assert.assertNotNull(entityList);
         Assert.assertEquals(entityList.size(), 1);
-        String hrDeptGuid = entityList.get(0);
-        return hrDeptGuid;
+        return entityList.get(0);
     }
     
     private int countVertices(String propertyName, Object value) {
@@ -176,12 +176,10 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
     }
     
     private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException {
-
         try {
             repositoryService.getEntityDefinition(hrDeptGuid);
             Assert.fail("EntityNotFoundException was expected but none thrown");
-        }
-        catch(EntityNotFoundException e) {
+        } catch(EntityNotFoundException e) {
             // good
         }
     }

repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java

@@ -24,6 +24,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup;
 import com.tinkerpop.blueprints.Compare;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
+
 import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.RepositoryMetadataModule;
 import org.apache.atlas.TestUtils;
@@ -52,9 +53,11 @@ import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
+
 import scala.actors.threadpool.Arrays;
 
 import javax.inject.Inject;
+
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
@@ -237,6 +240,9 @@ public class GraphBackedMetadataRepositoryTest {
     @Test(dependsOnMethods = "testGetTraitNames")
     public void testAddTrait() throws Exception {
         final String aGUID = getGUID();
+        Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
+        Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+        Assert.assertNull(modificationTimestampPreUpdate);
 
         List<String> traitNames = repositoryService.getTraitNames(aGUID);
         System.out.println("traitNames = " + traitNames);
@@ -254,6 +260,11 @@ public class GraphBackedMetadataRepositoryTest {
         Assert.assertEquals(traitNames.size(), 2);
         Assert.assertTrue(traitNames.contains(TestUtils.PII));
         Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION));
+        
+        // Verify modification timestamp was updated.
+        GraphHelper.getInstance().getVertexForGUID(aGUID);
+        Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+        Assert.assertNotNull(modificationTimestampPostUpdate);
     }
 
     @Test(dependsOnMethods = "testAddTrait")
@@ -301,6 +312,9 @@ public class GraphBackedMetadataRepositoryTest {
     @Test(dependsOnMethods = "testAddTrait")
     public void testDeleteTrait() throws Exception {
         final String aGUID = getGUID();
+        Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
+        Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+        Assert.assertNotNull(modificationTimestampPreUpdate);
 
         List<String> traitNames = repositoryService.getTraitNames(aGUID);
         Assert.assertEquals(traitNames.size(), 3);
@@ -315,6 +329,12 @@ public class GraphBackedMetadataRepositoryTest {
         Assert.assertEquals(traitNames.size(), 2);
         Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION));
         Assert.assertFalse(traitNames.contains(TestUtils.PII));
+        
+        // Verify modification timestamp was updated.
+        GraphHelper.getInstance().getVertexForGUID(aGUID);
+        Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+        Assert.assertNotNull(modificationTimestampPostUpdate);
+        Assert.assertTrue(modificationTimestampPostUpdate > modificationTimestampPreUpdate);
     }
 
     @Test(expectedExceptions = RepositoryException.class)
@@ -479,8 +499,16 @@ public class GraphBackedMetadataRepositoryTest {
     public void testUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
         ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Person", "name", "John");
         Id johnGuid = john.getId();
+
         ITypedReferenceableInstance max = repositoryService.getEntityDefinition("Person", "name", "Max");
         String maxGuid = max.getId()._getId();
+        Vertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
+        Long creationTimestamp = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY);
+        Assert.assertNotNull(creationTimestamp);
+
+        Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+        Assert.assertNull(modificationTimestampPreUpdate);
+
         ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
         Id janeGuid = jane.getId();
         
@@ -497,6 +525,13 @@ public class GraphBackedMetadataRepositoryTest {
         ITypedReferenceableInstance refTarget = (ITypedReferenceableInstance) object;
         Assert.assertEquals(refTarget.getId()._getId(), johnGuid._getId());
 
+
+        // Verify modification timestamp was updated.
+        vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
+        Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+        Assert.assertNotNull(modificationTimestampPostUpdate);
+        Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate);
+
         // Update max's mentor reference to jane.
         instance = personType.createInstance(max.getId());
         instance.set("mentor", janeGuid);
@@ -508,6 +543,12 @@ public class GraphBackedMetadataRepositoryTest {
         Assert.assertTrue(object instanceof ITypedReferenceableInstance);
         refTarget = (ITypedReferenceableInstance) object;
         Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId());
+        
+        // Verify modification timestamp was updated.
+        vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
+        Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+        Assert.assertNotNull(modificationTimestampPost2ndUpdate);
+        Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate);
     }
 
     private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception {