Skip to content

A
atlas
Commit 154dda0e by Madhan Neethiraj
Options

ATLAS-2169: delete request fails when hard-delete is configured

parent ad6b07a9
Showing with 300 additions and 224 deletions
intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
...@@ -195,7 +195,7 @@ public class EntityMutationResponse { ...@@ -195,7 +195,7 @@ public class EntityMutationResponse {
public void addEntity(EntityOperation op, AtlasEntityHeader header) { public void addEntity(EntityOperation op, AtlasEntityHeader header) {
// if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE // if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE
if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) { if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) {
if (entityHeaderExists(getCreatedEntities(), header)) { if (entityHeaderExists(getCreatedEntities(), header.getGuid())) {
return; return;
} }
} }
...@@ -211,17 +211,42 @@ public class EntityMutationResponse { ...@@ -211,17 +211,42 @@ public class EntityMutationResponse {
mutatedEntities.put(op, opEntities); mutatedEntities.put(op, opEntities);
} }
if (!entityHeaderExists(opEntities, header)) { if (!entityHeaderExists(opEntities, header.getGuid())) {
opEntities.add(header); opEntities.add(header);
} }
} }
private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, AtlasEntityHeader newEntityHeader) { @JsonIgnore
public void addEntity(EntityOperation op, AtlasObjectId entity) {
if (mutatedEntities == null) {
mutatedEntities = new HashMap<>();
} else {
// if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE
if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) {
if (entityHeaderExists(getCreatedEntities(), entity.getGuid())) {
return;
}
}
}
List<AtlasEntityHeader> opEntities = mutatedEntities.get(op);
if (opEntities == null) {
opEntities = new ArrayList<>();
mutatedEntities.put(op, opEntities);
}
if (!entityHeaderExists(opEntities, entity.getGuid())) {
opEntities.add(new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), entity.getUniqueAttributes()));
}
}
private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, String guid) {
boolean ret = false; boolean ret = false;
if (CollectionUtils.isNotEmpty(entityHeaders) && newEntityHeader != null) { if (CollectionUtils.isNotEmpty(entityHeaders) && guid != null) {
for (AtlasEntityHeader entityHeader : entityHeaders) { for (AtlasEntityHeader entityHeader : entityHeaders) {
if (StringUtils.equals(entityHeader.getGuid(), newEntityHeader.getGuid())) { if (StringUtils.equals(entityHeader.getGuid(), guid)) {
ret = true; ret = true;
break; break;
} }
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
...@@ -68,9 +68,6 @@ import java.util.Objects; ...@@ -68,9 +68,6 @@ import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
/** /**
* Utility class for graph operations. * Utility class for graph operations.
...@@ -888,24 +885,23 @@ public final class GraphHelper { ...@@ -888,24 +885,23 @@ public final class GraphHelper {
* Guid and AtlasVertex combo * Guid and AtlasVertex combo
*/ */
public static class VertexInfo { public static class VertexInfo {
private String guid; private final AtlasObjectId entity;
private AtlasVertex vertex; private final AtlasVertex vertex;
private String typeName;
public VertexInfo(String guid, AtlasVertex vertex, String typeName) { public VertexInfo(AtlasObjectId entity, AtlasVertex vertex) {
this.guid = guid; this.entity = entity;
this.vertex = vertex; this.vertex = vertex;
this.typeName = typeName;
} }
public String getGuid() { public AtlasObjectId getEntity() { return entity; }
return guid;
}
public AtlasVertex getVertex() { public AtlasVertex getVertex() {
return vertex; return vertex;
} }
public String getGuid() {
return entity.getGuid();
}
public String getTypeName() { public String getTypeName() {
return typeName; return entity.getTypeName();
} }
@Override @Override
...@@ -913,14 +909,13 @@ public final class GraphHelper { ...@@ -913,14 +909,13 @@ public final class GraphHelper {
if (this == o) return true; if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false; if (o == null || getClass() != o.getClass()) return false;
VertexInfo that = (VertexInfo) o; VertexInfo that = (VertexInfo) o;
return Objects.equals(guid, that.guid) && return Objects.equals(entity, that.entity) &&
Objects.equals(vertex, that.vertex) && Objects.equals(vertex, that.vertex);
Objects.equals(typeName, that.typeName);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(guid, vertex, typeName); return Objects.hash(entity, vertex);
} }
} }
...@@ -1304,39 +1299,4 @@ public final class GraphHelper { ...@@ -1304,39 +1299,4 @@ public final class GraphHelper {
return StringUtils.isNotEmpty(edge.getLabel()) ? edgeLabel.startsWith("r:") : false; return StringUtils.isNotEmpty(edge.getLabel()) ? edgeLabel.startsWith("r:") : false;
} }
public static AtlasObjectId getReferenceObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection,
AtlasVertex parentVertex) {
AtlasObjectId ret = null;
if (relationshipDirection == OUT) {
ret = getAtlasObjectIdForInVertex(edge);
} else if (relationshipDirection == IN) {
ret = getAtlasObjectIdForOutVertex(edge);
} else if (relationshipDirection == BOTH){
// since relationship direction is BOTH, edge direction can be inward or outward
// compare with parent entity vertex and pick the right reference vertex
if (verticesEquals(parentVertex, edge.getOutVertex())) {
ret = getAtlasObjectIdForInVertex(edge);
} else {
ret = getAtlasObjectIdForOutVertex(edge);
}
}
return ret;
}
public static AtlasObjectId getAtlasObjectIdForOutVertex(AtlasEdge edge) {
return new AtlasObjectId(getGuid(edge.getOutVertex()), getTypeName(edge.getOutVertex()));
}
public static AtlasObjectId getAtlasObjectIdForInVertex(AtlasEdge edge) {
return new AtlasObjectId(getGuid(edge.getInVertex()), getTypeName(edge.getInVertex()));
}
private static boolean verticesEquals(AtlasVertex vertexA, AtlasVertex vertexB) {
return StringUtils.equals(getGuid(vertexB), getGuid(vertexA));
}
} }
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java
...@@ -158,7 +158,7 @@ public class AtlasEntityChangeNotifier { ...@@ -158,7 +158,7 @@ public class AtlasEntityChangeNotifier {
return; return;
} }
List<Referenceable> typedRefInsts = toReferenceables(entityHeaders); List<Referenceable> typedRefInsts = toReferenceables(entityHeaders, operation);
for (EntityChangeListener listener : entityChangeListeners) { for (EntityChangeListener listener : entityChangeListeners) {
try { try {
...@@ -180,12 +180,19 @@ public class AtlasEntityChangeNotifier { ...@@ -180,12 +180,19 @@ public class AtlasEntityChangeNotifier {
} }
} }
private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders) throws AtlasBaseException { private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders, EntityOperation operation) throws AtlasBaseException {
List<Referenceable> ret = new ArrayList<>(entityHeaders.size()); List<Referenceable> ret = new ArrayList<>(entityHeaders.size());
// delete notifications don't need all attributes. Hence the special handling for delete operation
if (operation == EntityOperation.DELETE) {
for (AtlasEntityHeader entityHeader : entityHeaders) {
ret.add(new Referenceable(entityHeader.getGuid(), entityHeader.getTypeName(), entityHeader.getAttributes()));
}
} else {
for (AtlasEntityHeader entityHeader : entityHeaders) { for (AtlasEntityHeader entityHeader : entityHeaders) {
ret.add(toReferenceable(entityHeader.getGuid())); ret.add(toReferenceable(entityHeader.getGuid()));
} }
}
return ret; return ret;
} }
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
...@@ -647,14 +647,16 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { ...@@ -647,14 +647,16 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
private EntityMutationResponse deleteVertices(Collection<AtlasVertex> deletionCandidates) throws AtlasBaseException { private EntityMutationResponse deleteVertices(Collection<AtlasVertex> deletionCandidates) throws AtlasBaseException {
EntityMutationResponse response = new EntityMutationResponse(); EntityMutationResponse response = new EntityMutationResponse();
deleteHandler.deleteEntities(deletionCandidates);
RequestContextV1 req = RequestContextV1.get(); RequestContextV1 req = RequestContextV1.get();
for (AtlasObjectId id : req.getDeletedEntityIds()) {
response.addEntity(DELETE, EntityGraphMapper.constructHeader(id)); deleteHandler.deleteEntities(deletionCandidates); // this will update req with list of deleted/updated entities
for (AtlasObjectId entity : req.getDeletedEntities()) {
response.addEntity(DELETE, entity);
} }
for (AtlasObjectId id : req.getUpdatedEntityIds()) { for (AtlasObjectId entity : req.getUpdatedEntities()) {
response.addEntity(UPDATE, EntityGraphMapper.constructHeader(id)); response.addEntity(UPDATE, entity);
} }
return response; return response;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
...@@ -43,36 +43,28 @@ import org.apache.atlas.type.AtlasTypeRegistry; ...@@ -43,36 +43,28 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.*;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.graph.GraphHelper.getReferenceObjectId;
import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge; import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
import static org.apache.atlas.repository.graph.GraphHelper.string; import static org.apache.atlas.repository.graph.GraphHelper.string;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromEdge; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromEdge;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getState; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getState;
public abstract class DeleteHandlerV1 { public abstract class DeleteHandlerV1 {
public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class); public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class);
private AtlasTypeRegistry typeRegistry; private final AtlasTypeRegistry typeRegistry;
private boolean shouldUpdateInverseReferences; private final EntityGraphRetriever entityRetriever;
private boolean softDelete; private final boolean shouldUpdateInverseReferences;
private final boolean softDelete;
protected static final GraphHelper graphHelper = GraphHelper.getInstance(); protected static final GraphHelper graphHelper = GraphHelper.getInstance();
public DeleteHandlerV1(AtlasTypeRegistry typeRegistry, boolean shouldUpdateInverseReference, boolean softDelete) { public DeleteHandlerV1(AtlasTypeRegistry typeRegistry, boolean shouldUpdateInverseReference, boolean softDelete) {
this.typeRegistry = typeRegistry; this.typeRegistry = typeRegistry;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.shouldUpdateInverseReferences = shouldUpdateInverseReference; this.shouldUpdateInverseReferences = shouldUpdateInverseReference;
this.softDelete = softDelete; this.softDelete = softDelete;
} }
...@@ -87,33 +79,24 @@ public abstract class DeleteHandlerV1 { ...@@ -87,33 +79,24 @@ public abstract class DeleteHandlerV1 {
*/ */
public void deleteEntities(Collection<AtlasVertex> instanceVertices) throws AtlasBaseException { public void deleteEntities(Collection<AtlasVertex> instanceVertices) throws AtlasBaseException {
RequestContextV1 requestContext = RequestContextV1.get(); RequestContextV1 requestContext = RequestContextV1.get();
Set<AtlasVertex> deletionCandidateVertices = new HashSet<>(); Set<AtlasVertex> deletionCandidateVertices = new HashSet<>();
for (AtlasVertex instanceVertex : instanceVertices) { for (AtlasVertex instanceVertex : instanceVertices) {
String guid = AtlasGraphUtilsV1.getIdFromVertex(instanceVertex); String guid = AtlasGraphUtilsV1.getIdFromVertex(instanceVertex);
AtlasEntity.Status state = getState(instanceVertex); AtlasEntity.Status state = getState(instanceVertex);
if (state == DELETED) { if (state == DELETED || requestContext.isDeletedEntity(guid)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping deletion of {} as it is already deleted", guid); LOG.debug("Skipping deletion of {} as it is already deleted", guid);
continue;
} }
String typeName = AtlasGraphUtilsV1.getTypeName(instanceVertex);
AtlasObjectId objId = new AtlasObjectId(guid, typeName);
if (requestContext.getDeletedEntityIds().contains(objId)) {
LOG.debug("Skipping deletion of {} as it is already deleted", guid);
continue; continue;
} }
// Get GUIDs and vertices for all deletion candidates. // Record all deletion candidate entities in RequestContext
Set<GraphHelper.VertexInfo> compositeVertices = getOwnedVertices(instanceVertex);
// Record all deletion candidate GUIDs in RequestContext
// and gather deletion candidate vertices. // and gather deletion candidate vertices.
for (GraphHelper.VertexInfo vertexInfo : compositeVertices) { for (GraphHelper.VertexInfo vertexInfo : getOwnedVertices(instanceVertex)) {
requestContext.recordEntityDelete(new AtlasObjectId(vertexInfo.getGuid(), vertexInfo.getTypeName())); requestContext.recordEntityDelete(vertexInfo.getEntity());
deletionCandidateVertices.add(vertexInfo.getVertex()); deletionCandidateVertices.add(vertexInfo.getVertex());
} }
} }
...@@ -134,7 +117,10 @@ public abstract class DeleteHandlerV1 { ...@@ -134,7 +117,10 @@ public abstract class DeleteHandlerV1 {
public void deleteRelationships(Collection<AtlasEdge> edges) throws AtlasBaseException { public void deleteRelationships(Collection<AtlasEdge> edges) throws AtlasBaseException {
for (AtlasEdge edge : edges) { for (AtlasEdge edge : edges) {
if (getState(edge) == DELETED) { if (getState(edge) == DELETED) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping deletion of {} as it is already deleted", getIdFromEdge(edge)); LOG.debug("Skipping deletion of {} as it is already deleted", getIdFromEdge(edge));
}
continue; continue;
} }
...@@ -150,83 +136,104 @@ public abstract class DeleteHandlerV1 { ...@@ -150,83 +136,104 @@ public abstract class DeleteHandlerV1 {
* @return set of VertexInfo for all composite entities * @return set of VertexInfo for all composite entities
* @throws AtlasException * @throws AtlasException
*/ */
public Set<GraphHelper.VertexInfo> getOwnedVertices(AtlasVertex entityVertex) throws AtlasBaseException { public Collection<GraphHelper.VertexInfo> getOwnedVertices(AtlasVertex entityVertex) throws AtlasBaseException {
Set<GraphHelper.VertexInfo> result = new LinkedHashSet<>(); Map<String, GraphHelper.VertexInfo> vertexInfoMap = new HashMap<>();
Stack<AtlasVertex> vertices = new Stack<>(); Stack<AtlasVertex> vertices = new Stack<>();
vertices.push(entityVertex); vertices.push(entityVertex);
while (vertices.size() > 0) { while (vertices.size() > 0) {
AtlasVertex vertex = vertices.pop(); AtlasVertex vertex = vertices.pop();
AtlasEntity.Status state = getState(vertex); AtlasEntity.Status state = getState(vertex);
if (state == DELETED) { if (state == DELETED) {
//If the reference vertex is marked for deletion, skip it //If the reference vertex is marked for deletion, skip it
continue; continue;
} }
String typeName = GraphHelper.getTypeName(vertex);
String guid = GraphHelper.getGuid(vertex); String guid = GraphHelper.getGuid(vertex);
result.add(new GraphHelper.VertexInfo(guid, vertex, typeName)); if (vertexInfoMap.containsKey(guid)) {
continue;
}
AtlasObjectId entity = entityRetriever.toAtlasObjectId(vertex);
String typeName = entity.getTypeName();
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType == null) { if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName); throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName);
} }
vertexInfoMap.put(guid, new GraphHelper.VertexInfo(entity, vertex));
for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) { for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) {
if (! attributeInfo.isOwnedRef()) { if (! attributeInfo.isOwnedRef()) {
continue; continue;
} }
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(entityType, attributeInfo.getName()); String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(entityType, attributeInfo.getName());
AtlasType attrType = attributeInfo.getAttributeType(); AtlasType attrType = attributeInfo.getAttributeType();
switch (attrType.getTypeCategory()) { switch (attrType.getTypeCategory()) {
case OBJECT_ID_TYPE: case OBJECT_ID_TYPE: {
AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel); AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex(); vertices.push(edge.getInVertex());
vertices.push(compositeVertex); }
} }
break; break;
case ARRAY:
case ARRAY: {
AtlasArrayType arrType = (AtlasArrayType) attrType; AtlasArrayType arrType = (AtlasArrayType) attrType;
if (arrType.getElementType().getTypeCategory() != TypeCategory.OBJECT_ID_TYPE) { if (arrType.getElementType().getTypeCategory() != TypeCategory.OBJECT_ID_TYPE) {
continue; continue;
} }
Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(vertex, edgeLabel); Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(vertex, edgeLabel);
if (edges != null) { if (edges != null) {
while (edges.hasNext()) { while (edges.hasNext()) {
edge = edges.next(); AtlasEdge edge = edges.next();
if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex(); vertices.push(edge.getInVertex());
vertices.push(compositeVertex); }
} }
} }
} }
break; break;
case MAP:
case MAP: {
AtlasMapType mapType = (AtlasMapType) attrType; AtlasMapType mapType = (AtlasMapType) attrType;
TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory(); TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory();
if (valueTypeCategory != TypeCategory.OBJECT_ID_TYPE) { if (valueTypeCategory != TypeCategory.OBJECT_ID_TYPE) {
continue; continue;
} }
String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(entityType, attributeInfo.getName()); String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(entityType, attributeInfo.getName());
List<String> keys = vertex.getProperty(propertyName, List.class); List<String> keys = vertex.getProperty(propertyName, List.class);
if (keys != null) { if (keys != null) {
for (String key : keys) { for (String key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key); String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key);
edge = graphHelper.getEdgeForLabel(vertex, mapEdgeLabel); AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, mapEdgeLabel);
if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) { if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex(); vertices.push(edge.getInVertex());
vertices.push(compositeVertex); }
} }
} }
} }
break; break;
default:
} }
} }
} }
return result;
return vertexInfoMap.values();
} }
/** /**
...@@ -240,19 +247,19 @@ public abstract class DeleteHandlerV1 { ...@@ -240,19 +247,19 @@ public abstract class DeleteHandlerV1 {
*/ */
public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned,
boolean forceDeleteStructTrait, AtlasVertex vertex) throws AtlasBaseException { boolean forceDeleteStructTrait, AtlasVertex vertex) throws AtlasBaseException {
// default edge direction is outward // default edge direction is outward
return deleteEdgeReference(edge, typeCategory, isOwned, forceDeleteStructTrait, AtlasRelationshipEdgeDirection.OUT, vertex); return deleteEdgeReference(edge, typeCategory, isOwned, forceDeleteStructTrait, AtlasRelationshipEdgeDirection.OUT, vertex);
} }
public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, boolean forceDeleteStructTrait, public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, boolean forceDeleteStructTrait,
AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex entityVertex) throws AtlasBaseException { AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex entityVertex) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting {}", string(edge)); LOG.debug("Deleting {}", string(edge));
boolean forceDelete = }
(typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait;
boolean forceDelete = (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait;
if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) {
|| (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) {
//If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities. //If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities.
//If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled //If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled
//through this delete, hence delete the edge and the reference vertex. //through this delete, hence delete the edge and the reference vertex.
...@@ -270,8 +277,18 @@ public abstract class DeleteHandlerV1 { ...@@ -270,8 +277,18 @@ public abstract class DeleteHandlerV1 {
if (isRelationshipEdge(edge)) { if (isRelationshipEdge(edge)) {
deleteEdge(edge, false); deleteEdge(edge, false);
AtlasObjectId deletedReferenceObjectId = getReferenceObjectId(edge, relationshipDirection, entityVertex); AtlasVertex referencedVertex = entityRetriever.getReferencedEntityVertex(edge, relationshipDirection, entityVertex);
RequestContextV1.get().recordEntityUpdate(deletedReferenceObjectId);
if (referencedVertex != null) {
RequestContextV1 requestContext = RequestContextV1.get();
if (!requestContext.isUpdatedEntity(GraphHelper.getGuid(referencedVertex))) {
GraphHelper.setProperty(referencedVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, requestContext.getRequestTime());
GraphHelper.setProperty(referencedVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser());
requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(referencedVertex));
}
}
} else { } else {
//legacy case - not a relationship edge //legacy case - not a relationship edge
//If deleting just the edge, reverse attribute should be updated for any references //If deleting just the edge, reverse attribute should be updated for any references
...@@ -279,6 +296,7 @@ public abstract class DeleteHandlerV1 { ...@@ -279,6 +296,7 @@ public abstract class DeleteHandlerV1 {
deleteEdge(edge, true, false); deleteEdge(edge, true, false);
} }
} }
return !softDelete || forceDelete; return !softDelete || forceDelete;
} }
...@@ -286,13 +304,12 @@ public abstract class DeleteHandlerV1 { ...@@ -286,13 +304,12 @@ public abstract class DeleteHandlerV1 {
//update inverse attribute //update inverse attribute
if (updateInverseAttribute) { if (updateInverseAttribute) {
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edge.getLabel()); AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edge.getLabel());
AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName()); AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName());
if (parentType instanceof AtlasEntityType) { if (parentType instanceof AtlasEntityType) {
AtlasEntityType parentEntityType = (AtlasEntityType) parentType; AtlasEntityType parentEntityType = (AtlasEntityType) parentType;
AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName()); AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName());
if (attribute.getInverseRefAttribute() != null) { if (attribute.getInverseRefAttribute() != null) {
deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), attribute.getInverseRefAttribute()); deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), attribute.getInverseRefAttribute());
} }
...@@ -326,10 +343,11 @@ public abstract class DeleteHandlerV1 { ...@@ -326,10 +343,11 @@ public abstract class DeleteHandlerV1 {
* @throws AtlasException * @throws AtlasException
*/ */
protected void deleteTypeVertex(AtlasVertex instanceVertex, boolean force) throws AtlasBaseException { protected void deleteTypeVertex(AtlasVertex instanceVertex, boolean force) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting {}", string(instanceVertex)); LOG.debug("Deleting {}", string(instanceVertex));
String typeName = GraphHelper.getTypeName(instanceVertex); }
String typeName = GraphHelper.getTypeName(instanceVertex);
AtlasType parentType = typeRegistry.getType(typeName); AtlasType parentType = typeRegistry.getType(typeName);
if (parentType instanceof AtlasStructType) { if (parentType instanceof AtlasStructType) {
...@@ -337,11 +355,12 @@ public abstract class DeleteHandlerV1 { ...@@ -337,11 +355,12 @@ public abstract class DeleteHandlerV1 {
boolean isEntityType = (parentType instanceof AtlasEntityType); boolean isEntityType = (parentType instanceof AtlasEntityType);
for (AtlasStructType.AtlasAttribute attributeInfo : structType.getAllAttributes().values()) { for (AtlasStructType.AtlasAttribute attributeInfo : structType.getAllAttributes().values()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting attribute {} for {}", attributeInfo.getName(), string(instanceVertex)); LOG.debug("Deleting attribute {} for {}", attributeInfo.getName(), string(instanceVertex));
boolean isOwned = isEntityType && attributeInfo.isOwnedRef(); }
boolean isOwned = isEntityType && attributeInfo.isOwnedRef();
AtlasType attrType = attributeInfo.getAttributeType(); AtlasType attrType = attributeInfo.getAttributeType();
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(structType, attributeInfo.getName()); String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(structType, attributeInfo.getName());
switch (attrType.getTypeCategory()) { switch (attrType.getTypeCategory()) {
...@@ -359,11 +378,14 @@ public abstract class DeleteHandlerV1 { ...@@ -359,11 +378,14 @@ public abstract class DeleteHandlerV1 {
//For array attribute, if the element is struct/class, delete all the references //For array attribute, if the element is struct/class, delete all the references
AtlasArrayType arrType = (AtlasArrayType) attrType; AtlasArrayType arrType = (AtlasArrayType) attrType;
AtlasType elemType = arrType.getElementType(); AtlasType elemType = arrType.getElementType();
if (AtlasGraphUtilsV1.isReference(elemType.getTypeCategory())) { if (AtlasGraphUtilsV1.isReference(elemType.getTypeCategory())) {
Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel); Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel);
if (edges != null) { if (edges != null) {
while (edges.hasNext()) { while (edges.hasNext()) {
AtlasEdge edge = edges.next(); AtlasEdge edge = edges.next();
deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false, instanceVertex); deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false, instanceVertex);
} }
} }
...@@ -379,13 +401,16 @@ public abstract class DeleteHandlerV1 { ...@@ -379,13 +401,16 @@ public abstract class DeleteHandlerV1 {
if (AtlasGraphUtilsV1.isReference(valueTypeCategory)) { if (AtlasGraphUtilsV1.isReference(valueTypeCategory)) {
List<Object> keys = EntityGraphMapper.getArrayElementsProperty(keyType, instanceVertex, propertyName); List<Object> keys = EntityGraphMapper.getArrayElementsProperty(keyType, instanceVertex, propertyName);
if (keys != null) { if (keys != null) {
for (Object key : keys) { for (Object key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, (String) key); String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, (String) key);
deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, isOwned); deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, isOwned);
} }
} }
} }
break;
} }
} }
} }
...@@ -393,9 +418,9 @@ public abstract class DeleteHandlerV1 { ...@@ -393,9 +418,9 @@ public abstract class DeleteHandlerV1 {
deleteVertex(instanceVertex, force); deleteVertex(instanceVertex, force);
} }
public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, TypeCategory typeCategory, public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, TypeCategory typeCategory, boolean isOwned) throws AtlasBaseException {
boolean isOwned) throws AtlasBaseException {
AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
if (edge != null) { if (edge != null) {
deleteEdgeReference(edge, typeCategory, isOwned, false, outVertex); deleteEdgeReference(edge, typeCategory, isOwned, false, outVertex);
} }
...@@ -407,19 +432,22 @@ public abstract class DeleteHandlerV1 { ...@@ -407,19 +432,22 @@ public abstract class DeleteHandlerV1 {
* @throws AtlasException * @throws AtlasException
*/ */
private void deleteAllTraits(AtlasVertex instanceVertex) throws AtlasBaseException { private void deleteAllTraits(AtlasVertex instanceVertex) throws AtlasBaseException {
String typeName = GraphHelper.getTypeName(instanceVertex);
List<String> traitNames = GraphHelper.getTraitNames(instanceVertex); List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting traits {} for {}", traitNames, string(instanceVertex)); LOG.debug("Deleting traits {} for {}", traitNames, string(instanceVertex));
String typeName = GraphHelper.getTypeName(instanceVertex); }
for (String traitNameToBeDeleted : traitNames) { for (String traitNameToBeDeleted : traitNames) {
String relationshipLabel = GraphHelper.getTraitLabel(typeName, traitNameToBeDeleted); String relationshipLabel = GraphHelper.getTraitLabel(typeName, traitNameToBeDeleted);
deleteEdgeReference(instanceVertex, relationshipLabel, TypeCategory.CLASSIFICATION, false); deleteEdgeReference(instanceVertex, relationshipLabel, TypeCategory.CLASSIFICATION, false);
} }
} }
protected AtlasAttribute getAttributeForEdge(String edgeLabel) throws AtlasBaseException { protected AtlasAttribute getAttributeForEdge(String edgeLabel) throws AtlasBaseException {
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edgeLabel); AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edgeLabel);
AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName()); AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName());
AtlasStructType parentStructType = (AtlasStructType) parentType; AtlasStructType parentStructType = (AtlasStructType) parentType;
...@@ -438,15 +466,15 @@ public abstract class DeleteHandlerV1 { ...@@ -438,15 +466,15 @@ public abstract class DeleteHandlerV1 {
* @throws AtlasException * @throws AtlasException
*/ */
protected void deleteEdgeBetweenVertices(AtlasVertex outVertex, AtlasVertex inVertex, AtlasAttribute attribute) throws AtlasBaseException { protected void deleteEdgeBetweenVertices(AtlasVertex outVertex, AtlasVertex inVertex, AtlasAttribute attribute) throws AtlasBaseException {
LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex), if (LOG.isDebugEnabled()) {
attribute.getName()); LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex), attribute.getName());
String typeName = GraphHelper.getTypeName(outVertex); }
String outId = GraphHelper.getGuid(outVertex);
AtlasObjectId objId = new AtlasObjectId(outId, typeName); final String typeName = GraphHelper.getTypeName(outVertex);
AtlasEntity.Status state = getState(outVertex); final String outId = GraphHelper.getGuid(outVertex);
final AtlasEntity.Status state = getState(outVertex);
if (state == DELETED || (outId != null && RequestContextV1.get().isDeletedEntity(objId))) { if (state == DELETED || (outId != null && RequestContextV1.get().isDeletedEntity(outId))) {
//If the reference vertex is marked for deletion, skip updating the reference //If the reference vertex is marked for deletion, skip updating the reference
return; return;
} }
...@@ -455,95 +483,101 @@ public abstract class DeleteHandlerV1 { ...@@ -455,95 +483,101 @@ public abstract class DeleteHandlerV1 {
String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(parentType, attribute.getName()); String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(parentType, attribute.getName());
String edgeLabel = EDGE_LABEL_PREFIX + propertyName; String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
AtlasEdge edge = null; AtlasEdge edge = null;
AtlasAttributeDef attrDef = attribute.getAttributeDef(); AtlasAttributeDef attrDef = attribute.getAttributeDef();
AtlasType attrType = attribute.getAttributeType(); AtlasType attrType = attribute.getAttributeType();
switch (attrType.getTypeCategory()) { switch (attrType.getTypeCategory()) {
case OBJECT_ID_TYPE: case OBJECT_ID_TYPE: {
//If its class attribute, its the only edge between two vertices //If its class attribute, its the only edge between two vertices
if (attrDef.getIsOptional()) { if (attrDef.getIsOptional()) {
edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
if (shouldUpdateInverseReferences) { if (shouldUpdateInverseReferences) {
GraphHelper.setProperty(outVertex, propertyName, null); GraphHelper.setProperty(outVertex, propertyName, null);
} }
} else { } else {
// Cannot unset a required attribute. // Cannot unset a required attribute.
throw new AtlasBaseException("Cannot unset required attribute " + propertyName + throw new AtlasBaseException("Cannot unset required attribute " + propertyName + " on " + GraphHelper.vertexString(outVertex) + " edge = " + edgeLabel);
" on " + GraphHelper.vertexString(outVertex) + " edge = " + edgeLabel); }
} }
break; break;
case ARRAY: case ARRAY: {
//If its array attribute, find the right edge between the two vertices and update array property //If its array attribute, find the right edge between the two vertices and update array property
List<String> elements = GraphHelper.getListProperty(outVertex, propertyName); List<String> elements = GraphHelper.getListProperty(outVertex, propertyName);
if (elements != null) { if (elements != null) {
elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty() elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty()
for (String elementEdgeId : elements) { for (String elementEdgeId : elements) {
AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId); AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId);
if (elementEdge == null) { if (elementEdge == null) {
continue; continue;
} }
AtlasVertex elementVertex = elementEdge.getInVertex(); AtlasVertex elementVertex = elementEdge.getInVertex();
if (elementVertex.equals(inVertex)) { if (elementVertex.equals(inVertex)) {
edge = elementEdge; edge = elementEdge;
//TODO element.size includes deleted items as well. should exclude //TODO element.size includes deleted items as well. should exclude
if (!attrDef.getIsOptional() if (!attrDef.getIsOptional() && elements.size() <= attrDef.getValuesMinCount()) {
&& elements.size() <= attrDef.getValuesMinCount()) {
// Deleting this edge would violate the attribute's lower bound. // Deleting this edge would violate the attribute's lower bound.
throw new AtlasBaseException( throw new AtlasBaseException("Cannot remove array element from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge));
"Cannot remove array element from required attribute " +
propertyName + " on "
+ GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge));
} }
if (shouldUpdateInverseReferences) { if (shouldUpdateInverseReferences) {
//if composite attribute, remove the reference as well. else, just remove the edge //if composite attribute, remove the reference as well. else, just remove the edge
//for example, when table is deleted, process still references the table //for example, when table is deleted, process still references the table
//but when column is deleted, table will not reference the deleted column //but when column is deleted, table will not reference the deleted column
LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge), if (LOG.isDebugEnabled()) {
attribute.getName()); LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge), attribute.getName());
}
// Remove all occurrences of the edge ID from the list. // Remove all occurrences of the edge ID from the list.
// This prevents dangling edge IDs (i.e. edge IDs for deleted edges) // This prevents dangling edge IDs (i.e. edge IDs for deleted edges)
// from the remaining in the list if there are duplicates. // from the remaining in the list if there are duplicates.
elements.removeAll(Collections.singletonList(elementEdge.getId().toString())); elements.removeAll(Collections.singletonList(elementEdge.getId().toString()));
GraphHelper.setProperty(outVertex, propertyName, elements); GraphHelper.setProperty(outVertex, propertyName, elements);
break; break;
}
} }
} }
} }
} }
break; break;
case MAP: case MAP: {
//If its map attribute, find the right edge between two vertices and update map property //If its map attribute, find the right edge between two vertices and update map property
List<String> keys = GraphHelper.getListProperty(outVertex, propertyName); List<String> keys = GraphHelper.getListProperty(outVertex, propertyName);
if (keys != null) { if (keys != null) {
keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty() keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty()
for (String key : keys) { for (String key : keys) {
String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key); String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
String mapEdgeId = GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class); String mapEdgeId = GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class);
AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId); AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId);
if(mapEdge != null) {
if (mapEdge != null) {
AtlasVertex mapVertex = mapEdge.getInVertex(); AtlasVertex mapVertex = mapEdge.getInVertex();
if (mapVertex.getId().toString().equals(inVertex.getId().toString())) { if (mapVertex.getId().toString().equals(inVertex.getId().toString())) {
//TODO keys.size includes deleted items as well. should exclude //TODO keys.size includes deleted items as well. should exclude
if (attrDef.getIsOptional() || keys.size() > attrDef.getValuesMinCount()) { if (attrDef.getIsOptional() || keys.size() > attrDef.getValuesMinCount()) {
edge = mapEdge; edge = mapEdge;
} else { } else {
// Deleting this entry would violate the attribute's lower bound. // Deleting this entry would violate the attribute's lower bound.
throw new AtlasBaseException( throw new AtlasBaseException("Cannot remove map entry " + keyPropertyName + " from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge));
"Cannot remove map entry " + keyPropertyName + " from required attribute " +
propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge));
} }
if (shouldUpdateInverseReferences) { if (shouldUpdateInverseReferences) {
//remove this key //remove this key
LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key, if (LOG.isDebugEnabled()) {
attribute.getName()); LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key, attribute.getName());
}
keys.remove(key); keys.remove(key);
GraphHelper.setProperty(outVertex, propertyName, keys); GraphHelper.setProperty(outVertex, propertyName, keys);
GraphHelper.setProperty(outVertex, keyPropertyName, null); GraphHelper.setProperty(outVertex, keyPropertyName, null);
...@@ -553,6 +587,7 @@ public abstract class DeleteHandlerV1 { ...@@ -553,6 +587,7 @@ public abstract class DeleteHandlerV1 {
} }
} }
} }
}
break; break;
case STRUCT: case STRUCT:
...@@ -560,26 +595,32 @@ public abstract class DeleteHandlerV1 { ...@@ -560,26 +595,32 @@ public abstract class DeleteHandlerV1 {
break; break;
default: default:
throw new IllegalStateException("There can't be an edge from " + GraphHelper.getVertexDetails(outVertex) + " to " throw new IllegalStateException("There can't be an edge from " + GraphHelper.getVertexDetails(outVertex) + " to " + GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attribute.getName() + " which is not class/array/map attribute. found " + attrType.getTypeCategory().name());
+ GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attribute.getName() + " which is not class/array/map attribute. found " + attrType.getTypeCategory().name());
} }
if (edge != null) { if (edge != null) {
deleteEdge(edge, false); deleteEdge(edge, false);
RequestContextV1 requestContext = RequestContextV1.get(); RequestContextV1 requestContext = RequestContextV1.get();
GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
requestContext.getRequestTime()); if (! requestContext.isUpdatedEntity(outId)) {
GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, requestContext.getRequestTime());
GraphHelper.setProperty(outVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser()); GraphHelper.setProperty(outVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser());
requestContext.recordEntityUpdate(new AtlasObjectId(outId, typeName));
requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(outVertex));
}
} }
} }
protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws AtlasBaseException { protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws AtlasBaseException {
//Update external references(incoming edges) to this vertex //Update external references(incoming edges) to this vertex
if (LOG.isDebugEnabled()) {
LOG.debug("Setting the external references to {} to null(removing edges)", string(instanceVertex)); LOG.debug("Setting the external references to {} to null(removing edges)", string(instanceVertex));
}
for (AtlasEdge edge : (Iterable<AtlasEdge>) instanceVertex.getEdges(AtlasEdgeDirection.IN)) { for (AtlasEdge edge : (Iterable<AtlasEdge>) instanceVertex.getEdges(AtlasEdgeDirection.IN)) {
AtlasEntity.Status edgeState = getState(edge); AtlasEntity.Status edgeState = getState(edge);
if (edgeState == AtlasEntity.Status.ACTIVE) { if (edgeState == AtlasEntity.Status.ACTIVE) {
//Delete only the active edge references //Delete only the active edge references
AtlasAttribute attribute = getAttributeForEdge(edge.getLabel()); AtlasAttribute attribute = getAttributeForEdge(edge.getLabel());
...@@ -587,6 +628,7 @@ public abstract class DeleteHandlerV1 { ...@@ -587,6 +628,7 @@ public abstract class DeleteHandlerV1 {
deleteEdgeBetweenVertices(edge.getOutVertex(), edge.getInVertex(), attribute); deleteEdgeBetweenVertices(edge.getOutVertex(), edge.getInVertex(), attribute);
} }
} }
_deleteVertex(instanceVertex, force); _deleteVertex(instanceVertex, force);
} }
} }
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
...@@ -85,6 +85,7 @@ public class EntityGraphMapper { ...@@ -85,6 +85,7 @@ public class EntityGraphMapper {
private final AtlasGraph graph; private final AtlasGraph graph;
private final DeleteHandlerV1 deleteHandler; private final DeleteHandlerV1 deleteHandler;
private final AtlasTypeRegistry typeRegistry; private final AtlasTypeRegistry typeRegistry;
private final EntityGraphRetriever entityRetriever;
private final AtlasRelationshipStore relationshipStore; private final AtlasRelationshipStore relationshipStore;
@Inject @Inject
...@@ -92,6 +93,7 @@ public class EntityGraphMapper { ...@@ -92,6 +93,7 @@ public class EntityGraphMapper {
AtlasRelationshipStore relationshipStore) { AtlasRelationshipStore relationshipStore) {
this.deleteHandler = deleteHandler; this.deleteHandler = deleteHandler;
this.typeRegistry = typeRegistry; this.typeRegistry = typeRegistry;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.graph = atlasGraph; this.graph = atlasGraph;
this.relationshipStore = relationshipStore; this.relationshipStore = relationshipStore;
} }
...@@ -192,16 +194,16 @@ public class EntityGraphMapper { ...@@ -192,16 +194,16 @@ public class EntityGraphMapper {
RequestContextV1 req = RequestContextV1.get(); RequestContextV1 req = RequestContextV1.get();
for (AtlasObjectId id : req.getDeletedEntityIds()) { for (AtlasObjectId entity : req.getDeletedEntities()) {
resp.addEntity(DELETE, constructHeader(id)); resp.addEntity(DELETE, entity);
} }
for (AtlasObjectId id : req.getUpdatedEntityIds()) { for (AtlasObjectId entity : req.getUpdatedEntities()) {
if (isPartialUpdate) { if (isPartialUpdate) {
resp.addEntity(PARTIAL_UPDATE, constructHeader(id)); resp.addEntity(PARTIAL_UPDATE, entity);
} }
else { else {
resp.addEntity(UPDATE, constructHeader(id)); resp.addEntity(UPDATE, entity);
} }
} }
...@@ -423,7 +425,6 @@ public class EntityGraphMapper { ...@@ -423,7 +425,6 @@ public class EntityGraphMapper {
AtlasVertex attrVertex = context.getDiscoveryContext().getResolvedEntityVertex(getGuid(ctx.getValue())); AtlasVertex attrVertex = context.getDiscoveryContext().getResolvedEntityVertex(getGuid(ctx.getValue()));
recordEntityUpdate(attrVertex); recordEntityUpdate(attrVertex);
updateModificationMetadata(attrVertex);
} }
//delete old reference //delete old reference
...@@ -494,9 +495,13 @@ public class EntityGraphMapper { ...@@ -494,9 +495,13 @@ public class EntityGraphMapper {
} }
if (inverseUpdated) { if (inverseUpdated) {
RequestContextV1 requestContext = RequestContextV1.get();
if (!requestContext.isDeletedEntity(GraphHelper.getGuid(inverseVertex))) {
updateModificationMetadata(inverseVertex); updateModificationMetadata(inverseVertex);
AtlasObjectId inverseEntityId = new AtlasObjectId(getIdFromVertex(inverseVertex), inverseType.getTypeName());
RequestContextV1.get().recordEntityUpdate(inverseEntityId); requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(inverseVertex));
}
} }
} }
...@@ -1441,32 +1446,15 @@ public class EntityGraphMapper { ...@@ -1441,32 +1446,15 @@ public class EntityGraphMapper {
return ret; return ret;
} }
private void recordEntityUpdate(AtlasVertex vertex) { private void recordEntityUpdate(AtlasVertex vertex) throws AtlasBaseException {
AtlasObjectId objectId = new AtlasObjectId(GraphHelper.getGuid(vertex), GraphHelper.getTypeName(vertex));
RequestContextV1 req = RequestContextV1.get(); RequestContextV1 req = RequestContextV1.get();
if (!objectIdsContain(req.getUpdatedEntityIds(), objectId) && !objectIdsContain(req.getCreatedEntityIds(), objectId)) { if (!req.isUpdatedEntity(GraphHelper.getGuid(vertex))) {
req.recordEntityUpdate(objectId); updateModificationMetadata(vertex);
}
}
private boolean objectIdsContain(Collection<AtlasObjectId> objectIds, AtlasObjectId objectId) {
boolean ret = false;
if (CollectionUtils.isEmpty(objectIds)) {
ret = false;
} else { req.recordEntityUpdate(entityRetriever.toAtlasObjectId(vertex));
for (AtlasObjectId id : objectIds) {
if (StringUtils.equals(id.getGuid(), objectId.getGuid())) {
ret = true;
break;
} }
} }
}
return ret;
}
private static void compactAttributes(AtlasEntity entity) { private static void compactAttributes(AtlasEntity entity) {
if (entity != null) { if (entity != null) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
...@@ -147,6 +147,70 @@ public final class EntityGraphRetriever { ...@@ -147,6 +147,70 @@ public final class EntityGraphRetriever {
return atlasVertex != null ? mapVertexToAtlasEntityHeader(atlasVertex, attributes) : null; return atlasVertex != null ? mapVertexToAtlasEntityHeader(atlasVertex, attributes) : null;
} }
public AtlasEntityHeader toAtlasEntityHeader(AtlasEntity entity) {
AtlasEntityHeader ret = null;
String typeName = entity.getTypeName();
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType != null) {
Map<String, Object> uniqueAttributes = new HashMap<>();
for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) {
Object attrValue = entity.getAttribute(attribute.getName());
if (attrValue != null) {
uniqueAttributes.put(attribute.getName(), attrValue);
}
}
ret = new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), uniqueAttributes);
}
return ret;
}
public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseException {
AtlasObjectId ret = null;
String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType != null) {
Map<String, Object> uniqueAttributes = new HashMap<>();
for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) {
Object attrValue = getVertexAttribute(entityVertex, attribute);
if (attrValue != null) {
uniqueAttributes.put(attribute.getName(), attrValue);
}
}
ret = new AtlasObjectId(entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class), typeName, uniqueAttributes);
}
return ret;
}
public AtlasVertex getReferencedEntityVertex(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex parentVertex) throws AtlasBaseException {
AtlasVertex entityVertex = null;
if (relationshipDirection == OUT) {
entityVertex = edge.getInVertex();
} else if (relationshipDirection == IN) {
entityVertex = edge.getOutVertex();
} else if (relationshipDirection == BOTH){
// since relationship direction is BOTH, edge direction can be inward or outward
// compare with parent entity vertex and pick the right reference vertex
if (StringUtils.equals(GraphHelper.getGuid(parentVertex), GraphHelper.getGuid(edge.getOutVertex()))) {
entityVertex = edge.getInVertex();
} else {
entityVertex = edge.getOutVertex();
}
}
return entityVertex;
}
public AtlasVertex getEntityVertex(String guid) throws AtlasBaseException { public AtlasVertex getEntityVertex(String guid) throws AtlasBaseException {
AtlasVertex ret = AtlasGraphUtilsV1.findByGuid(guid); AtlasVertex ret = AtlasGraphUtilsV1.findByGuid(guid);
......
server-api/src/main/java/org/apache/atlas/RequestContextV1.java
...@@ -31,18 +31,15 @@ public class RequestContextV1 { ...@@ -31,18 +31,15 @@ public class RequestContextV1 {
private static final ThreadLocal<RequestContextV1> CURRENT_CONTEXT = new ThreadLocal<>(); private static final ThreadLocal<RequestContextV1> CURRENT_CONTEXT = new ThreadLocal<>();
private Set<AtlasObjectId> createdEntityIds = new LinkedHashSet<>(); private final Map<String, AtlasObjectId> updatedEntities = new HashMap<>();
private Set<AtlasObjectId> updatedEntityIds = new LinkedHashSet<>(); private final Map<String, AtlasObjectId> deletedEntities = new HashMap<>();
private Set<AtlasObjectId> deletedEntityIds = new LinkedHashSet<>(); private final Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>();
private Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>(); private final Metrics metrics = new Metrics();
private final long requestTime = System.currentTimeMillis();
private String user; private String user;
private final long requestTime;
private Metrics metrics = new Metrics();
private RequestContextV1() { private RequestContextV1() {
requestTime = System.currentTimeMillis();
} }
//To handle gets from background threads where createContext() is not called //To handle gets from background threads where createContext() is not called
...@@ -62,10 +59,10 @@ public class RequestContextV1 { ...@@ -62,10 +59,10 @@ public class RequestContextV1 {
RequestContextV1 instance = CURRENT_CONTEXT.get(); RequestContextV1 instance = CURRENT_CONTEXT.get();
if (instance != null) { if (instance != null) {
if (instance.entityCacheV2 != null) { instance.updatedEntities.clear();
instance.deletedEntities.clear();
instance.entityCacheV2.clear(); instance.entityCacheV2.clear();
} }
}
CURRENT_CONTEXT.remove(); CURRENT_CONTEXT.remove();
} }
...@@ -78,24 +75,16 @@ public class RequestContextV1 { ...@@ -78,24 +75,16 @@ public class RequestContextV1 {
this.user = user; this.user = user;
} }
public void recordEntityCreate(Collection<AtlasObjectId> createdEntityIds) { public void recordEntityUpdate(AtlasObjectId entity) {
this.createdEntityIds.addAll(createdEntityIds); if (entity != null && entity.getGuid() != null) {
updatedEntities.put(entity.getGuid(), entity);
} }
public void recordEntityCreate(AtlasObjectId createdEntityId) {
this.createdEntityIds.add(createdEntityId);
}
public void recordEntityUpdate(Collection<AtlasObjectId> updatedEntityIds) {
this.updatedEntityIds.addAll(updatedEntityIds);
} }
public void recordEntityUpdate(AtlasObjectId entityId) { public void recordEntityDelete(AtlasObjectId entity) {
this.updatedEntityIds.add(entityId); if (entity != null && entity.getGuid() != null) {
deletedEntities.put(entity.getGuid(), entity);
} }
public void recordEntityDelete(AtlasObjectId entityId) {
deletedEntityIds.add(entityId);
} }
/** /**
...@@ -108,16 +97,12 @@ public class RequestContextV1 { ...@@ -108,16 +97,12 @@ public class RequestContextV1 {
} }
} }
public Collection<AtlasObjectId> getCreatedEntityIds() { public Collection<AtlasObjectId> getUpdatedEntities() {
return createdEntityIds; return updatedEntities.values();
}
public Collection<AtlasObjectId> getUpdatedEntityIds() {
return updatedEntityIds;
} }
public Collection<AtlasObjectId> getDeletedEntityIds() { public Collection<AtlasObjectId> getDeletedEntities() {
return deletedEntityIds; return deletedEntities.values();
} }
/** /**
...@@ -135,8 +120,12 @@ public class RequestContextV1 { ...@@ -135,8 +120,12 @@ public class RequestContextV1 {
return requestTime; return requestTime;
} }
public boolean isDeletedEntity(AtlasObjectId entityId) { public boolean isUpdatedEntity(String guid) {
return deletedEntityIds.contains(entityId); return updatedEntities.containsKey(guid);
}
public boolean isDeletedEntity(String guid) {
return deletedEntities.containsKey(guid);
} }
public static Metrics getMetrics() { public static Metrics getMetrics() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment