ATLAS-2169: delete request fails when hard-delete is configured

intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java

@@ -195,7 +195,7 @@ public class EntityMutationResponse {
     public void addEntity(EntityOperation op, AtlasEntityHeader header) {
         // if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE
         if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) {
-            if (entityHeaderExists(getCreatedEntities(), header)) {
+            if (entityHeaderExists(getCreatedEntities(), header.getGuid())) {
                 return;
             }
         }
@@ -211,17 +211,42 @@ public class EntityMutationResponse {
             mutatedEntities.put(op, opEntities);
         }
 
-        if (!entityHeaderExists(opEntities, header)) {
+        if (!entityHeaderExists(opEntities, header.getGuid())) {
             opEntities.add(header);
         }
     }
 
-    private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, AtlasEntityHeader newEntityHeader) {
+    @JsonIgnore
+    public void addEntity(EntityOperation op, AtlasObjectId entity) {
+        if (mutatedEntities == null) {
+            mutatedEntities = new HashMap<>();
+        } else {
+            // if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE
+            if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) {
+                if (entityHeaderExists(getCreatedEntities(), entity.getGuid())) {
+                    return;
+                }
+            }
+        }
+
+        List<AtlasEntityHeader> opEntities = mutatedEntities.get(op);
+
+        if (opEntities == null) {
+            opEntities = new ArrayList<>();
+            mutatedEntities.put(op, opEntities);
+        }
+
+        if (!entityHeaderExists(opEntities, entity.getGuid())) {
+            opEntities.add(new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), entity.getUniqueAttributes()));
+        }
+    }
+
+    private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, String guid) {
         boolean ret = false;
 
-        if (CollectionUtils.isNotEmpty(entityHeaders) && newEntityHeader != null) {
+        if (CollectionUtils.isNotEmpty(entityHeaders) && guid != null) {
             for (AtlasEntityHeader entityHeader : entityHeaders) {
-                if (StringUtils.equals(entityHeader.getGuid(), newEntityHeader.getGuid())) {
+                if (StringUtils.equals(entityHeader.getGuid(), guid)) {
                     ret = true;
                     break;
                 }

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -68,9 +68,6 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
 
-import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
-import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
-import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
 
 /**
  * Utility class for graph operations.
@@ -888,24 +885,23 @@ public final class GraphHelper {
      * Guid and AtlasVertex combo
      */
     public static class VertexInfo {
-        private String guid;
-        private AtlasVertex vertex;
-        private String typeName;
+        private final AtlasObjectId entity;
+        private final AtlasVertex   vertex;
 
-        public VertexInfo(String guid, AtlasVertex vertex, String typeName) {
-            this.guid = guid;
+        public VertexInfo(AtlasObjectId entity, AtlasVertex vertex) {
+            this.entity = entity;
             this.vertex = vertex;
-            this.typeName = typeName;
         }
 
-        public String getGuid() {
-            return guid;
-        }
+        public AtlasObjectId getEntity() { return entity; }
         public AtlasVertex getVertex() {
             return vertex;
         }
+        public String getGuid() {
+            return entity.getGuid();
+        }
         public String getTypeName() {
-            return typeName;
+            return entity.getTypeName();
         }
 
         @Override
@@ -913,14 +909,13 @@ public final class GraphHelper {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             VertexInfo that = (VertexInfo) o;
-            return Objects.equals(guid, that.guid) &&
-                    Objects.equals(vertex, that.vertex) &&
-                    Objects.equals(typeName, that.typeName);
+            return Objects.equals(entity, that.entity) &&
+                    Objects.equals(vertex, that.vertex);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(guid, vertex, typeName);
+            return Objects.hash(entity, vertex);
         }
     }
 
@@ -1304,39 +1299,4 @@ public final class GraphHelper {
 
         return StringUtils.isNotEmpty(edge.getLabel()) ? edgeLabel.startsWith("r:") : false;
     }
-
-    public static AtlasObjectId getReferenceObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection,
-                                                     AtlasVertex parentVertex) {
-        AtlasObjectId ret = null;
-
-        if (relationshipDirection == OUT) {
-            ret = getAtlasObjectIdForInVertex(edge);
-
-        } else if (relationshipDirection == IN) {
-            ret = getAtlasObjectIdForOutVertex(edge);
-
-        } else if (relationshipDirection == BOTH){
-            // since relationship direction is BOTH, edge direction can be inward or outward
-            // compare with parent entity vertex and pick the right reference vertex
-            if (verticesEquals(parentVertex, edge.getOutVertex())) {
-                ret = getAtlasObjectIdForInVertex(edge);
-            } else {
-                ret = getAtlasObjectIdForOutVertex(edge);
-            }
-        }
-
-        return ret;
-    }
-
-    public static AtlasObjectId getAtlasObjectIdForOutVertex(AtlasEdge edge) {
-        return new AtlasObjectId(getGuid(edge.getOutVertex()), getTypeName(edge.getOutVertex()));
-    }
-
-    public static AtlasObjectId getAtlasObjectIdForInVertex(AtlasEdge edge) {
-        return new AtlasObjectId(getGuid(edge.getInVertex()), getTypeName(edge.getInVertex()));
-    }
-
-    private static boolean verticesEquals(AtlasVertex vertexA, AtlasVertex vertexB) {
-        return StringUtils.equals(getGuid(vertexB), getGuid(vertexA));
-    }
 }
\ No newline at end of file

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java

@@ -158,7 +158,7 @@ public class AtlasEntityChangeNotifier {
             return;
         }
 
-        List<Referenceable> typedRefInsts = toReferenceables(entityHeaders);
+        List<Referenceable> typedRefInsts = toReferenceables(entityHeaders, operation);
 
         for (EntityChangeListener listener : entityChangeListeners) {
             try {
@@ -180,11 +180,18 @@ public class AtlasEntityChangeNotifier {
         }
     }
 
-    private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders) throws AtlasBaseException {
+    private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders, EntityOperation operation) throws AtlasBaseException {
         List<Referenceable> ret = new ArrayList<>(entityHeaders.size());
 
-        for (AtlasEntityHeader entityHeader : entityHeaders) {
-            ret.add(toReferenceable(entityHeader.getGuid()));
+        // delete notifications don't need all attributes. Hence the special handling for delete operation
+        if (operation == EntityOperation.DELETE) {
+            for (AtlasEntityHeader entityHeader : entityHeaders) {
+                ret.add(new Referenceable(entityHeader.getGuid(), entityHeader.getTypeName(), entityHeader.getAttributes()));
+            }
+        } else {
+            for (AtlasEntityHeader entityHeader : entityHeaders) {
+                ret.add(toReferenceable(entityHeader.getGuid()));
+            }
         }
 
         return ret;

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java

@@ -647,14 +647,16 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
 
     private EntityMutationResponse deleteVertices(Collection<AtlasVertex> deletionCandidates) throws AtlasBaseException {
         EntityMutationResponse response = new EntityMutationResponse();
-        deleteHandler.deleteEntities(deletionCandidates);
-        RequestContextV1 req = RequestContextV1.get();
-        for (AtlasObjectId id : req.getDeletedEntityIds()) {
-            response.addEntity(DELETE, EntityGraphMapper.constructHeader(id));
+        RequestContextV1       req      = RequestContextV1.get();
+
+        deleteHandler.deleteEntities(deletionCandidates); // this will update req with list of deleted/updated entities
+
+        for (AtlasObjectId entity : req.getDeletedEntities()) {
+            response.addEntity(DELETE, entity);
         }
 
-        for (AtlasObjectId id : req.getUpdatedEntityIds()) {
-            response.addEntity(UPDATE, EntityGraphMapper.constructHeader(id));
+        for (AtlasObjectId entity : req.getUpdatedEntities()) {
+            response.addEntity(UPDATE, entity);
         }
 
         return response;

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java

@@ -85,6 +85,7 @@ public class EntityGraphMapper {
     private final AtlasGraph             graph;
     private final DeleteHandlerV1        deleteHandler;
     private final AtlasTypeRegistry      typeRegistry;
+    private final EntityGraphRetriever   entityRetriever;
     private final AtlasRelationshipStore relationshipStore;
 
     @Inject
@@ -92,6 +93,7 @@ public class EntityGraphMapper {
                              AtlasRelationshipStore relationshipStore) {
         this.deleteHandler     = deleteHandler;
         this.typeRegistry      = typeRegistry;
+        this.entityRetriever   = new EntityGraphRetriever(typeRegistry);
         this.graph             = atlasGraph;
         this.relationshipStore = relationshipStore;
     }
@@ -192,16 +194,16 @@ public class EntityGraphMapper {
 
         RequestContextV1 req = RequestContextV1.get();
 
-        for (AtlasObjectId id : req.getDeletedEntityIds()) {
-            resp.addEntity(DELETE, constructHeader(id));
+        for (AtlasObjectId entity : req.getDeletedEntities()) {
+            resp.addEntity(DELETE, entity);
         }
 
-        for (AtlasObjectId id : req.getUpdatedEntityIds()) {
+        for (AtlasObjectId entity : req.getUpdatedEntities()) {
             if (isPartialUpdate) {
-                resp.addEntity(PARTIAL_UPDATE, constructHeader(id));
+                resp.addEntity(PARTIAL_UPDATE, entity);
             }
             else {
-                resp.addEntity(UPDATE, constructHeader(id));
+                resp.addEntity(UPDATE, entity);
             }
         }
 
@@ -423,7 +425,6 @@ public class EntityGraphMapper {
                         AtlasVertex attrVertex = context.getDiscoveryContext().getResolvedEntityVertex(getGuid(ctx.getValue()));
 
                         recordEntityUpdate(attrVertex);
-                        updateModificationMetadata(attrVertex);
                     }
 
                     //delete old reference
@@ -494,9 +495,13 @@ public class EntityGraphMapper {
         }
 
         if (inverseUpdated) {
-            updateModificationMetadata(inverseVertex);
-            AtlasObjectId inverseEntityId = new AtlasObjectId(getIdFromVertex(inverseVertex), inverseType.getTypeName());
-            RequestContextV1.get().recordEntityUpdate(inverseEntityId);
+            RequestContextV1 requestContext = RequestContextV1.get();
+
+            if (!requestContext.isDeletedEntity(GraphHelper.getGuid(inverseVertex))) {
+                updateModificationMetadata(inverseVertex);
+
+                requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(inverseVertex));
+            }
         }
     }
 
@@ -1441,31 +1446,14 @@ public class EntityGraphMapper {
         return ret;
     }
 
-    private void recordEntityUpdate(AtlasVertex vertex) {
-        AtlasObjectId    objectId = new AtlasObjectId(GraphHelper.getGuid(vertex), GraphHelper.getTypeName(vertex));
-        RequestContextV1 req      = RequestContextV1.get();
-
-        if (!objectIdsContain(req.getUpdatedEntityIds(), objectId) && !objectIdsContain(req.getCreatedEntityIds(), objectId)) {
-            req.recordEntityUpdate(objectId);
-        }
-    }
-
-    private boolean objectIdsContain(Collection<AtlasObjectId> objectIds, AtlasObjectId objectId) {
-        boolean ret = false;
+    private void recordEntityUpdate(AtlasVertex vertex) throws AtlasBaseException {
+        RequestContextV1 req  = RequestContextV1.get();
 
-        if (CollectionUtils.isEmpty(objectIds)) {
-            ret = false;
+        if (!req.isUpdatedEntity(GraphHelper.getGuid(vertex))) {
+            updateModificationMetadata(vertex);
 
-        } else {
-            for (AtlasObjectId id : objectIds) {
-                if (StringUtils.equals(id.getGuid(), objectId.getGuid())) {
-                    ret = true;
-                    break;
-                }
-            }
+            req.recordEntityUpdate(entityRetriever.toAtlasObjectId(vertex));
         }
-
-        return ret;
     }
 
     private static void compactAttributes(AtlasEntity entity) {

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java

@@ -147,6 +147,70 @@ public final class EntityGraphRetriever {
         return atlasVertex != null ? mapVertexToAtlasEntityHeader(atlasVertex, attributes) : null;
     }
 
+    public AtlasEntityHeader toAtlasEntityHeader(AtlasEntity entity) {
+        AtlasEntityHeader ret        = null;
+        String            typeName   = entity.getTypeName();
+        AtlasEntityType   entityType = typeRegistry.getEntityTypeByName(typeName);
+
+        if (entityType != null) {
+            Map<String, Object> uniqueAttributes = new HashMap<>();
+
+            for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) {
+                Object attrValue = entity.getAttribute(attribute.getName());
+
+                if (attrValue != null) {
+                    uniqueAttributes.put(attribute.getName(), attrValue);
+                }
+            }
+
+            ret = new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), uniqueAttributes);
+        }
+
+        return ret;
+    }
+
+    public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseException {
+        AtlasObjectId   ret        = null;
+        String          typeName   = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class);
+        AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
+
+        if (entityType != null) {
+            Map<String, Object> uniqueAttributes = new HashMap<>();
+
+            for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) {
+                Object attrValue = getVertexAttribute(entityVertex, attribute);
+
+                if (attrValue != null) {
+                    uniqueAttributes.put(attribute.getName(), attrValue);
+                }
+            }
+
+            ret = new AtlasObjectId(entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class), typeName, uniqueAttributes);
+        }
+
+        return ret;
+    }
+
+    public AtlasVertex getReferencedEntityVertex(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex parentVertex) throws AtlasBaseException {
+        AtlasVertex entityVertex = null;
+
+        if (relationshipDirection == OUT) {
+            entityVertex = edge.getInVertex();
+        } else if (relationshipDirection == IN) {
+            entityVertex = edge.getOutVertex();
+        } else if (relationshipDirection == BOTH){
+            // since relationship direction is BOTH, edge direction can be inward or outward
+            // compare with parent entity vertex and pick the right reference vertex
+            if (StringUtils.equals(GraphHelper.getGuid(parentVertex), GraphHelper.getGuid(edge.getOutVertex()))) {
+                entityVertex = edge.getInVertex();
+            } else {
+                entityVertex = edge.getOutVertex();
+            }
+        }
+
+        return entityVertex;
+    }
+
     public AtlasVertex getEntityVertex(String guid) throws AtlasBaseException {
         AtlasVertex ret = AtlasGraphUtilsV1.findByGuid(guid);
 

server-api/src/main/java/org/apache/atlas/RequestContextV1.java

@@ -31,18 +31,15 @@ public class RequestContextV1 {
 
     private static final ThreadLocal<RequestContextV1> CURRENT_CONTEXT = new ThreadLocal<>();
 
-    private Set<AtlasObjectId> createdEntityIds = new LinkedHashSet<>();
-    private Set<AtlasObjectId> updatedEntityIds = new LinkedHashSet<>();
-    private Set<AtlasObjectId> deletedEntityIds = new LinkedHashSet<>();
-    private Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>();
+    private final Map<String, AtlasObjectId>          updatedEntities = new HashMap<>();
+    private final Map<String, AtlasObjectId>          deletedEntities = new HashMap<>();
+    private final Map<String, AtlasEntityWithExtInfo> entityCacheV2   = new HashMap<>();
+    private final Metrics                             metrics         = new Metrics();
+    private final long                                requestTime     = System.currentTimeMillis();
 
     private String user;
-    private final long requestTime;
-
-    private Metrics metrics = new Metrics();
 
     private RequestContextV1() {
-        requestTime = System.currentTimeMillis();
     }
 
     //To handle gets from background threads where createContext() is not called
@@ -62,9 +59,9 @@ public class RequestContextV1 {
         RequestContextV1 instance = CURRENT_CONTEXT.get();
 
         if (instance != null) {
-            if (instance.entityCacheV2 != null) {
-                instance.entityCacheV2.clear();
-            }
+            instance.updatedEntities.clear();
+            instance.deletedEntities.clear();
+            instance.entityCacheV2.clear();
         }
 
         CURRENT_CONTEXT.remove();
@@ -78,24 +75,16 @@ public class RequestContextV1 {
         this.user = user;
     }
 
-    public void recordEntityCreate(Collection<AtlasObjectId> createdEntityIds) {
-        this.createdEntityIds.addAll(createdEntityIds);
-    }
-
-    public void recordEntityCreate(AtlasObjectId createdEntityId) {
-        this.createdEntityIds.add(createdEntityId);
-    }
-
-    public void recordEntityUpdate(Collection<AtlasObjectId> updatedEntityIds) {
-        this.updatedEntityIds.addAll(updatedEntityIds);
-    }
-
-    public void recordEntityUpdate(AtlasObjectId entityId) {
-        this.updatedEntityIds.add(entityId);
+    public void recordEntityUpdate(AtlasObjectId entity) {
+        if (entity != null && entity.getGuid() != null) {
+            updatedEntities.put(entity.getGuid(), entity);
+        }
     }
 
-    public void recordEntityDelete(AtlasObjectId entityId) {
-        deletedEntityIds.add(entityId);
+    public void recordEntityDelete(AtlasObjectId entity) {
+        if (entity != null && entity.getGuid() != null) {
+            deletedEntities.put(entity.getGuid(), entity);
+        }
     }
 
     /**
@@ -108,16 +97,12 @@ public class RequestContextV1 {
         }
     }
 
-    public Collection<AtlasObjectId> getCreatedEntityIds() {
-        return createdEntityIds;
-    }
-
-    public Collection<AtlasObjectId> getUpdatedEntityIds() {
-        return updatedEntityIds;
+    public Collection<AtlasObjectId> getUpdatedEntities() {
+        return updatedEntities.values();
     }
 
-    public Collection<AtlasObjectId> getDeletedEntityIds() {
-        return deletedEntityIds;
+    public Collection<AtlasObjectId> getDeletedEntities() {
+        return deletedEntities.values();
     }
 
     /**
@@ -135,8 +120,12 @@ public class RequestContextV1 {
         return requestTime;
     }
 
-    public boolean isDeletedEntity(AtlasObjectId entityId) {
-        return deletedEntityIds.contains(entityId);
+    public boolean isUpdatedEntity(String guid) {
+        return updatedEntities.containsKey(guid);
+    }
+
+    public boolean isDeletedEntity(String guid) {
+        return deletedEntities.containsKey(guid);
     }
 
     public static Metrics getMetrics() {