ATLAS-370 Implement deleteEntities at repository level (dkantor via shwethags)

.gitignore

@@ -39,6 +39,9 @@ test-output
 #Python
 *.pyc
 
+# review board
+.reviewboardrc
+
 # other files
 .DS_Store
 *.swp

release-log.txt

@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-370 Implement deleteEntities at repository level (dkantor via shwethags)
 ATLAS-406 Resizing lineage window – should be an anchor on a corner – like ppt for graphic (sanjayp via shwethags)
 ATLAS-432 QuickStart lineage is broken (yhemanth via shwethags)
 ATLAS-421 typo in Architecture.twiki (dbist13 via shwethags)

repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java

@@ -104,13 +104,15 @@ public interface MetadataRepository {
     List<String> getEntityList(String entityType) throws RepositoryException;
 
     /**
-     * Deletes an entity definition (instance) corresponding to a given type.
+     * Deletes entities for the specified guids.
      *
-     * @param guid globally unique identifier for the entity
-     * @return true if deleted else false
+     * @param guids globally unique identifiers for the deletion candidate entities
+     * @return guids of deleted entities
      * @throws RepositoryException
      */
-    // boolean deleteEntity(String guid) throws RepositoryException;
+     List <String> deleteEntities(String... guids) throws RepositoryException;
+    
+    
     // Trait management functions
 
     /**

repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java

@@ -24,18 +24,22 @@ import com.tinkerpop.blueprints.Direction;
 import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
+
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.MetadataRepository;
 import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.repository.graph.TypedInstanceToGraphMapper.Operation;
 import org.apache.atlas.typesystem.ITypedReferenceableInstance;
 import org.apache.atlas.typesystem.ITypedStruct;
 import org.apache.atlas.typesystem.exception.EntityExistsException;
 import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.typesystem.persistence.Id;
 import org.apache.atlas.typesystem.types.AttributeInfo;
 import org.apache.atlas.typesystem.types.ClassType;
 import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.TraitType;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.atlas.typesystem.types.TypeUtils;
 import org.slf4j.Logger;
@@ -43,6 +47,7 @@ import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
@@ -258,7 +263,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                 titanGraph.removeEdge(traitEdge);
 
                 if (traitVertex != null) { // remove the trait instance from the repository
-                    titanGraph.removeVertex(traitVertex);
+                    TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
+                    instanceToGraphMapper.deleteTraitVertex(traitNameToBeDeleted, traitVertex);
 
                     // update the traits in entity once trait removal is successful
                     traitNames.remove(traitNameToBeDeleted);
@@ -270,6 +276,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
         }
     }
 
+    
     private void updateTraits(Vertex instanceVertex, List<String> traitNames) {
         // remove the key
         instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
@@ -304,4 +311,34 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             throw new RepositoryException(e);
         }
     }
+
+    @Override
+    @GraphTransaction
+    public List<String> deleteEntities(String... guids) throws RepositoryException {
+
+        if (guids == null || guids.length == 0) {
+            throw new IllegalArgumentException("guids must be non-null and non-empty");
+        }
+        
+        TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
+        for (String guid : guids) {
+            if (guid == null) {
+                LOG.warn("deleteEntities: Ignoring null guid");
+                continue;
+            }
+            try {
+                Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
+                String typeName = GraphHelper.getTypeName(instanceVertex);
+                instanceToGraphMapper.deleteEntity(typeName, instanceVertex);
+            } catch (EntityNotFoundException e) {
+                // Entity does not exist - treat as non-error, since the caller
+                // wanted to delete the entity and it's already gone.
+                LOG.info("Deletion request ignored for non-existent entity with guid " + guid);
+                continue;
+            } catch (AtlasException e) {
+                throw new RepositoryException(e);
+            }
+        }
+        return instanceToGraphMapper.getDeletedEntities();
+    }
 }

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -26,6 +26,7 @@ import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.Graph;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
+
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.typesystem.IReferenceableInstance;
@@ -39,6 +40,7 @@ import org.apache.atlas.typesystem.types.DataTypes;
 import org.apache.atlas.typesystem.types.HierarchicalType;
 import org.apache.atlas.typesystem.types.IDataType;
 import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils.Pair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -175,11 +177,44 @@ public final class GraphHelper {
         LOG.info("Removed edge {}", edge);
         if (cascade) {
            Vertex referredVertex = edge.getVertex(Direction.IN);
-           titanGraph.removeVertex(referredVertex);
-           LOG.info("Removed vertex {}", referredVertex);
+           removeVertex(referredVertex);
         }
         return edge;
     }
+    
+    /**
+     * Remove the specified edge from the graph.
+     * 
+     * @param edge
+     */
+    public void removeEdge(Edge edge) {
+        LOG.debug("Removing edge {}", edge);
+        titanGraph.removeEdge(edge);
+        LOG.info("Removed edge {}", edge);
+    }
+    
+    /**
+     * Return the edge and target vertex for the specified edge ID.
+     * 
+     * @param edgeId
+     * @return edge and target vertex
+     */
+    public Pair<Edge, Vertex> getEdgeAndTargetVertex(String edgeId) {
+        final Edge edge = titanGraph.getEdge(edgeId);
+        Vertex referredVertex = edge.getVertex(Direction.IN);
+        return Pair.of(edge, referredVertex);
+    }
+    
+    /**
+     * Remove the specified vertex from the graph.
+     * 
+     * @param vertex
+     */
+    public void removeVertex(Vertex vertex) {
+        LOG.debug("Removing vertex {}", vertex);
+        titanGraph.removeVertex(vertex);
+        LOG.info("Removed vertex {}", vertex);
+    }
 
     public Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
         return getVertexForProperty(Constants.GUID_PROPERTY_KEY, guid);
@@ -269,7 +304,6 @@ public final class GraphHelper {
         return result;
     }
 
-
     public static void dumpToLog(final Graph graph) {
         LOG.debug("*******************Graph Dump****************************");
         LOG.debug("Vertices of {}", graph);

repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.inject.Inject;
+
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+import com.tinkerpop.blueprints.Vertex;
+
+/**
+ * Test for GraphBackedMetadataRepository.deleteEntities
+ *
+ * Guice loads the dependencies and injects the necessary objects
+ *
+ */
+@Guice(modules = RepositoryMetadataModule.class)
+public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
+
+    @Inject
+    private GraphProvider<TitanGraph> graphProvider;
+
+    @Inject
+    private GraphBackedMetadataRepository repositoryService;
+
+    @Inject
+    private GraphBackedDiscoveryService discoveryService;
+
+    private TypeSystem typeSystem;
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        typeSystem = TypeSystem.getInstance();
+        typeSystem.reset();
+
+        new GraphBackedSearchIndexer(graphProvider);
+
+        TestUtils.defineDeptEmployeeTypes(typeSystem);
+        TestUtils.createHiveTypes(typeSystem);
+    }
+
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        TypeSystem.getInstance().reset();
+        try {
+            graphProvider.get().shutdown();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        try {
+            TitanCleanup.clear(graphProvider.get());
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+
+
+    @Test
+    public void testDeleteEntities() throws Exception {
+        String hrDeptGuid = createHrDeptGraph();
+
+        ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
+        Object refValue = hrDept.get("employees");
+        Assert.assertTrue(refValue instanceof List);
+        List<Object> employees = (List<Object>)refValue;
+        Assert.assertEquals(employees.size(), 4);
+        List<String> employeeGuids = new ArrayList<String>(4);
+        for (Object listValue : employees) {
+            Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
+            ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
+            employeeGuids.add(employee.getId()._getId());
+        }
+        
+        // There should be 4 vertices for Address structs (one for each Person.address attribute value).
+        int vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
+        Assert.assertEquals(vertexCount, 4);
+        vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
+        Assert.assertEquals(vertexCount, 1);
+        
+        List<String> deletedEntities = repositoryService.deleteEntities(hrDeptGuid);
+        Assert.assertTrue(deletedEntities.contains(hrDeptGuid));
+        
+        // Verify Department entity and its contained Person entities were deleted.
+        verifyEntityDoesNotExist(hrDeptGuid);
+        for (String employeeGuid : employeeGuids) {
+            verifyEntityDoesNotExist(employeeGuid);
+        }
+        // Verify all Person.address struct vertices were removed.
+        vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
+        Assert.assertEquals(vertexCount, 0);
+        
+        // Verify all SecurityClearance trait vertices were removed.
+        vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
+        Assert.assertEquals(vertexCount, 0);
+    }
+
+    @Test(dependsOnMethods = "testDeleteEntities")
+    public void testDeleteContainedEntity() throws Exception {
+        String hrDeptGuid = createHrDeptGraph();
+        ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
+        Object refValue = hrDept.get("employees");
+        Assert.assertTrue(refValue instanceof List);
+        List<Object> employees = (List<Object>)refValue;
+        Assert.assertEquals(employees.size(), 4);
+        Object listValue = employees.get(2);
+        Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
+        ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
+        String employeeGuid = employee.getId()._getId();
+        
+        List<String> deletedEntities = repositoryService.deleteEntities(employeeGuid);
+        Assert.assertTrue(deletedEntities.contains(employeeGuid));
+        verifyEntityDoesNotExist(employeeGuid);
+        
+    }
+
+    private String createHrDeptGraph() throws Exception {
+        Referenceable deptEg1 = TestUtils.createDeptEg1(typeSystem);
+        ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
+        ITypedReferenceableInstance hrDept2 = deptType.convert(deptEg1, Multiplicity.REQUIRED);
+
+        List<String> guids = repositoryService.createEntities(hrDept2);
+        Assert.assertNotNull(guids);
+        Assert.assertEquals(guids.size(), 5);
+        List<String> entityList = repositoryService.getEntityList("Department");
+        Assert.assertNotNull(entityList);
+        Assert.assertEquals(entityList.size(), 1);
+        String hrDeptGuid = entityList.get(0);
+        return hrDeptGuid;
+    }
+    
+    private int countVertices(String propertyName, Object value) {
+        Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value);
+        int vertexCount = 0;
+        for (Vertex vertex : vertices) {
+            vertexCount++;
+        }
+        return vertexCount;
+    }
+    
+    private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException {
+
+        try {
+            repositoryService.getEntityDefinition(hrDeptGuid);
+            Assert.fail("EntityNotFoundException was expected but none thrown");
+        }
+        catch(EntityNotFoundException e) {
+            // good
+        }
+    }
+
+}