Skip to content

A
atlas
Commit 154dda0e by Madhan Neethiraj
Options

ATLAS-2169: delete request fails when hard-delete is configured

parent ad6b07a9
Showing with 300 additions and 224 deletions
intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
......@@ -195,7 +195,7 @@ public class EntityMutationResponse {
public void addEntity(EntityOperation op, AtlasEntityHeader header) {
// if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE
if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) {
if (entityHeaderExists(getCreatedEntities(), header)) {
if (entityHeaderExists(getCreatedEntities(), header.getGuid())) {
return;
}
}
......@@ -211,17 +211,42 @@ public class EntityMutationResponse {
mutatedEntities.put(op, opEntities);
}
if (!entityHeaderExists(opEntities, header)) {
if (!entityHeaderExists(opEntities, header.getGuid())) {
opEntities.add(header);
}
}
private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, AtlasEntityHeader newEntityHeader) {
@JsonIgnore
public void addEntity(EntityOperation op, AtlasObjectId entity) {
if (mutatedEntities == null) {
mutatedEntities = new HashMap<>();
} else {
// if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE
if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) {
if (entityHeaderExists(getCreatedEntities(), entity.getGuid())) {
return;
}
}
}
List<AtlasEntityHeader> opEntities = mutatedEntities.get(op);
if (opEntities == null) {
opEntities = new ArrayList<>();
mutatedEntities.put(op, opEntities);
}
if (!entityHeaderExists(opEntities, entity.getGuid())) {
opEntities.add(new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), entity.getUniqueAttributes()));
}
}
private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, String guid) {
boolean ret = false;
if (CollectionUtils.isNotEmpty(entityHeaders) && newEntityHeader != null) {
if (CollectionUtils.isNotEmpty(entityHeaders) && guid != null) {
for (AtlasEntityHeader entityHeader : entityHeaders) {
if (StringUtils.equals(entityHeader.getGuid(), newEntityHeader.getGuid())) {
if (StringUtils.equals(entityHeader.getGuid(), guid)) {
ret = true;
break;
}
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -68,9 +68,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
/**
* Utility class for graph operations.
......@@ -888,24 +885,23 @@ public final class GraphHelper {
* Guid and AtlasVertex combo
*/
public static class VertexInfo {
private String guid;
private AtlasVertex vertex;
private String typeName;
private final AtlasObjectId entity;
private final AtlasVertex vertex;
public VertexInfo(String guid, AtlasVertex vertex, String typeName) {
this.guid = guid;
public VertexInfo(AtlasObjectId entity, AtlasVertex vertex) {
this.entity = entity;
this.vertex = vertex;
this.typeName = typeName;
}
public String getGuid() {
return guid;
}
public AtlasObjectId getEntity() { return entity; }
public AtlasVertex getVertex() {
return vertex;
}
public String getGuid() {
return entity.getGuid();
}
public String getTypeName() {
return typeName;
return entity.getTypeName();
}
@Override
......@@ -913,14 +909,13 @@ public final class GraphHelper {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
VertexInfo that = (VertexInfo) o;
return Objects.equals(guid, that.guid) &&
Objects.equals(vertex, that.vertex) &&
Objects.equals(typeName, that.typeName);
return Objects.equals(entity, that.entity) &&
Objects.equals(vertex, that.vertex);
}
@Override
public int hashCode() {
return Objects.hash(guid, vertex, typeName);
return Objects.hash(entity, vertex);
}
}
......@@ -1304,39 +1299,4 @@ public final class GraphHelper {
return StringUtils.isNotEmpty(edge.getLabel()) ? edgeLabel.startsWith("r:") : false;
}
public static AtlasObjectId getReferenceObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection,
AtlasVertex parentVertex) {
AtlasObjectId ret = null;
if (relationshipDirection == OUT) {
ret = getAtlasObjectIdForInVertex(edge);
} else if (relationshipDirection == IN) {
ret = getAtlasObjectIdForOutVertex(edge);
} else if (relationshipDirection == BOTH){
// since relationship direction is BOTH, edge direction can be inward or outward
// compare with parent entity vertex and pick the right reference vertex
if (verticesEquals(parentVertex, edge.getOutVertex())) {
ret = getAtlasObjectIdForInVertex(edge);
} else {
ret = getAtlasObjectIdForOutVertex(edge);
}
}
return ret;
}
public static AtlasObjectId getAtlasObjectIdForOutVertex(AtlasEdge edge) {
return new AtlasObjectId(getGuid(edge.getOutVertex()), getTypeName(edge.getOutVertex()));
}
public static AtlasObjectId getAtlasObjectIdForInVertex(AtlasEdge edge) {
return new AtlasObjectId(getGuid(edge.getInVertex()), getTypeName(edge.getInVertex()));
}
private static boolean verticesEquals(AtlasVertex vertexA, AtlasVertex vertexB) {
return StringUtils.equals(getGuid(vertexB), getGuid(vertexA));
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java
......@@ -158,7 +158,7 @@ public class AtlasEntityChangeNotifier {
return;
}
List<Referenceable> typedRefInsts = toReferenceables(entityHeaders);
List<Referenceable> typedRefInsts = toReferenceables(entityHeaders, operation);
for (EntityChangeListener listener : entityChangeListeners) {
try {
......@@ -180,12 +180,19 @@ public class AtlasEntityChangeNotifier {
}
}
private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders) throws AtlasBaseException {
private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders, EntityOperation operation) throws AtlasBaseException {
List<Referenceable> ret = new ArrayList<>(entityHeaders.size());
// delete notifications don't need all attributes. Hence the special handling for delete operation
if (operation == EntityOperation.DELETE) {
for (AtlasEntityHeader entityHeader : entityHeaders) {
ret.add(new Referenceable(entityHeader.getGuid(), entityHeader.getTypeName(), entityHeader.getAttributes()));
}
} else {
for (AtlasEntityHeader entityHeader : entityHeaders) {
ret.add(toReferenceable(entityHeader.getGuid()));
}
}
return ret;
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
......@@ -647,14 +647,16 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
private EntityMutationResponse deleteVertices(Collection<AtlasVertex> deletionCandidates) throws AtlasBaseException {
EntityMutationResponse response = new EntityMutationResponse();
deleteHandler.deleteEntities(deletionCandidates);
RequestContextV1 req = RequestContextV1.get();
for (AtlasObjectId id : req.getDeletedEntityIds()) {
response.addEntity(DELETE, EntityGraphMapper.constructHeader(id));
deleteHandler.deleteEntities(deletionCandidates); // this will update req with list of deleted/updated entities
for (AtlasObjectId entity : req.getDeletedEntities()) {
response.addEntity(DELETE, entity);
}
for (AtlasObjectId id : req.getUpdatedEntityIds()) {
response.addEntity(UPDATE, EntityGraphMapper.constructHeader(id));
for (AtlasObjectId entity : req.getUpdatedEntities()) {
response.addEntity(UPDATE, entity);
}
return response;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -43,36 +43,28 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.*;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.graph.GraphHelper.getReferenceObjectId;
import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromEdge;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getState;
public abstract class DeleteHandlerV1 {
public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class);
private AtlasTypeRegistry typeRegistry;
private boolean shouldUpdateInverseReferences;
private boolean softDelete;
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphRetriever entityRetriever;
private final boolean shouldUpdateInverseReferences;
private final boolean softDelete;
protected static final GraphHelper graphHelper = GraphHelper.getInstance();
public DeleteHandlerV1(AtlasTypeRegistry typeRegistry, boolean shouldUpdateInverseReference, boolean softDelete) {
this.typeRegistry = typeRegistry;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.shouldUpdateInverseReferences = shouldUpdateInverseReference;
this.softDelete = softDelete;
}
......@@ -87,33 +79,24 @@ public abstract class DeleteHandlerV1 {
*/
public void deleteEntities(Collection<AtlasVertex> instanceVertices) throws AtlasBaseException {
RequestContextV1 requestContext = RequestContextV1.get();
Set<AtlasVertex> deletionCandidateVertices = new HashSet<>();
for (AtlasVertex instanceVertex : instanceVertices) {
String guid = AtlasGraphUtilsV1.getIdFromVertex(instanceVertex);
AtlasEntity.Status state = getState(instanceVertex);
if (state == DELETED) {
if (state == DELETED || requestContext.isDeletedEntity(guid)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping deletion of {} as it is already deleted", guid);
continue;
}
String typeName = AtlasGraphUtilsV1.getTypeName(instanceVertex);
AtlasObjectId objId = new AtlasObjectId(guid, typeName);
if (requestContext.getDeletedEntityIds().contains(objId)) {
LOG.debug("Skipping deletion of {} as it is already deleted", guid);
continue;
}
// Get GUIDs and vertices for all deletion candidates.
Set<GraphHelper.VertexInfo> compositeVertices = getOwnedVertices(instanceVertex);
// Record all deletion candidate GUIDs in RequestContext
// Record all deletion candidate entities in RequestContext
// and gather deletion candidate vertices.
for (GraphHelper.VertexInfo vertexInfo : compositeVertices) {
requestContext.recordEntityDelete(new AtlasObjectId(vertexInfo.getGuid(), vertexInfo.getTypeName()));
for (GraphHelper.VertexInfo vertexInfo : getOwnedVertices(instanceVertex)) {
requestContext.recordEntityDelete(vertexInfo.getEntity());
deletionCandidateVertices.add(vertexInfo.getVertex());
}
}
......@@ -134,7 +117,10 @@ public abstract class DeleteHandlerV1 {
public void deleteRelationships(Collection<AtlasEdge> edges) throws AtlasBaseException {
for (AtlasEdge edge : edges) {
if (getState(edge) == DELETED) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping deletion of {} as it is already deleted", getIdFromEdge(edge));
}
continue;
}
......@@ -150,83 +136,104 @@ public abstract class DeleteHandlerV1 {
* @return set of VertexInfo for all composite entities
* @throws AtlasException
*/
public Set<GraphHelper.VertexInfo> getOwnedVertices(AtlasVertex entityVertex) throws AtlasBaseException {
Set<GraphHelper.VertexInfo> result = new LinkedHashSet<>();
public Collection<GraphHelper.VertexInfo> getOwnedVertices(AtlasVertex entityVertex) throws AtlasBaseException {
Map<String, GraphHelper.VertexInfo> vertexInfoMap = new HashMap<>();
Stack<AtlasVertex> vertices = new Stack<>();
vertices.push(entityVertex);
while (vertices.size() > 0) {
AtlasVertex vertex = vertices.pop();
AtlasEntity.Status state = getState(vertex);
if (state == DELETED) {
//If the reference vertex is marked for deletion, skip it
continue;
}
String typeName = GraphHelper.getTypeName(vertex);
String guid = GraphHelper.getGuid(vertex);
result.add(new GraphHelper.VertexInfo(guid, vertex, typeName));
if (vertexInfoMap.containsKey(guid)) {
continue;
}
AtlasObjectId entity = entityRetriever.toAtlasObjectId(vertex);
String typeName = entity.getTypeName();
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName);
}
vertexInfoMap.put(guid, new GraphHelper.VertexInfo(entity, vertex));
for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) {
if (! attributeInfo.isOwnedRef()) {
continue;
}
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(entityType, attributeInfo.getName());
AtlasType attrType = attributeInfo.getAttributeType();
switch (attrType.getTypeCategory()) {
case OBJECT_ID_TYPE:
case OBJECT_ID_TYPE: {
AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, edgeLabel);
if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex();
vertices.push(compositeVertex);
vertices.push(edge.getInVertex());
}
}
break;
case ARRAY:
case ARRAY: {
AtlasArrayType arrType = (AtlasArrayType) attrType;
if (arrType.getElementType().getTypeCategory() != TypeCategory.OBJECT_ID_TYPE) {
continue;
}
Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(vertex, edgeLabel);
if (edges != null) {
while (edges.hasNext()) {
edge = edges.next();
AtlasEdge edge = edges.next();
if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex();
vertices.push(compositeVertex);
vertices.push(edge.getInVertex());
}
}
}
}
break;
case MAP:
case MAP: {
AtlasMapType mapType = (AtlasMapType) attrType;
TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory();
if (valueTypeCategory != TypeCategory.OBJECT_ID_TYPE) {
continue;
}
String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(entityType, attributeInfo.getName());
List<String> keys = vertex.getProperty(propertyName, List.class);
if (keys != null) {
for (String key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key);
edge = graphHelper.getEdgeForLabel(vertex, mapEdgeLabel);
AtlasEdge edge = graphHelper.getEdgeForLabel(vertex, mapEdgeLabel);
if (edge != null && getState(edge) == AtlasEntity.Status.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex();
vertices.push(compositeVertex);
vertices.push(edge.getInVertex());
}
}
}
}
break;
default:
}
}
}
return result;
return vertexInfoMap.values();
}
/**
......@@ -240,19 +247,19 @@ public abstract class DeleteHandlerV1 {
*/
public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned,
boolean forceDeleteStructTrait, AtlasVertex vertex) throws AtlasBaseException {
// default edge direction is outward
return deleteEdgeReference(edge, typeCategory, isOwned, forceDeleteStructTrait, AtlasRelationshipEdgeDirection.OUT, vertex);
}
public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, boolean forceDeleteStructTrait,
AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex entityVertex) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting {}", string(edge));
boolean forceDelete =
(typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait;
}
boolean forceDelete = (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait;
if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION
|| (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) {
if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) {
//If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities.
//If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled
//through this delete, hence delete the edge and the reference vertex.
......@@ -270,8 +277,18 @@ public abstract class DeleteHandlerV1 {
if (isRelationshipEdge(edge)) {
deleteEdge(edge, false);
AtlasObjectId deletedReferenceObjectId = getReferenceObjectId(edge, relationshipDirection, entityVertex);
RequestContextV1.get().recordEntityUpdate(deletedReferenceObjectId);
AtlasVertex referencedVertex = entityRetriever.getReferencedEntityVertex(edge, relationshipDirection, entityVertex);
if (referencedVertex != null) {
RequestContextV1 requestContext = RequestContextV1.get();
if (!requestContext.isUpdatedEntity(GraphHelper.getGuid(referencedVertex))) {
GraphHelper.setProperty(referencedVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, requestContext.getRequestTime());
GraphHelper.setProperty(referencedVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser());
requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(referencedVertex));
}
}
} else {
//legacy case - not a relationship edge
//If deleting just the edge, reverse attribute should be updated for any references
......@@ -279,6 +296,7 @@ public abstract class DeleteHandlerV1 {
deleteEdge(edge, true, false);
}
}
return !softDelete || forceDelete;
}
......@@ -286,13 +304,12 @@ public abstract class DeleteHandlerV1 {
//update inverse attribute
if (updateInverseAttribute) {
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edge.getLabel());
AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName());
if (parentType instanceof AtlasEntityType) {
AtlasEntityType parentEntityType = (AtlasEntityType) parentType;
AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName());
if (attribute.getInverseRefAttribute() != null) {
deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), attribute.getInverseRefAttribute());
}
......@@ -326,10 +343,11 @@ public abstract class DeleteHandlerV1 {
* @throws AtlasException
*/
protected void deleteTypeVertex(AtlasVertex instanceVertex, boolean force) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting {}", string(instanceVertex));
String typeName = GraphHelper.getTypeName(instanceVertex);
}
String typeName = GraphHelper.getTypeName(instanceVertex);
AtlasType parentType = typeRegistry.getType(typeName);
if (parentType instanceof AtlasStructType) {
......@@ -337,11 +355,12 @@ public abstract class DeleteHandlerV1 {
boolean isEntityType = (parentType instanceof AtlasEntityType);
for (AtlasStructType.AtlasAttribute attributeInfo : structType.getAllAttributes().values()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting attribute {} for {}", attributeInfo.getName(), string(instanceVertex));
boolean isOwned = isEntityType && attributeInfo.isOwnedRef();
}
boolean isOwned = isEntityType && attributeInfo.isOwnedRef();
AtlasType attrType = attributeInfo.getAttributeType();
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(structType, attributeInfo.getName());
switch (attrType.getTypeCategory()) {
......@@ -359,11 +378,14 @@ public abstract class DeleteHandlerV1 {
//For array attribute, if the element is struct/class, delete all the references
AtlasArrayType arrType = (AtlasArrayType) attrType;
AtlasType elemType = arrType.getElementType();
if (AtlasGraphUtilsV1.isReference(elemType.getTypeCategory())) {
Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel);
if (edges != null) {
while (edges.hasNext()) {
AtlasEdge edge = edges.next();
deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false, instanceVertex);
}
}
......@@ -379,13 +401,16 @@ public abstract class DeleteHandlerV1 {
if (AtlasGraphUtilsV1.isReference(valueTypeCategory)) {
List<Object> keys = EntityGraphMapper.getArrayElementsProperty(keyType, instanceVertex, propertyName);
if (keys != null) {
for (Object key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, (String) key);
deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, isOwned);
}
}
}
break;
}
}
}
......@@ -393,9 +418,9 @@ public abstract class DeleteHandlerV1 {
deleteVertex(instanceVertex, force);
}
public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, TypeCategory typeCategory,
boolean isOwned) throws AtlasBaseException {
public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, TypeCategory typeCategory, boolean isOwned) throws AtlasBaseException {
AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
if (edge != null) {
deleteEdgeReference(edge, typeCategory, isOwned, false, outVertex);
}
......@@ -407,19 +432,22 @@ public abstract class DeleteHandlerV1 {
* @throws AtlasException
*/
private void deleteAllTraits(AtlasVertex instanceVertex) throws AtlasBaseException {
String typeName = GraphHelper.getTypeName(instanceVertex);
List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting traits {} for {}", traitNames, string(instanceVertex));
String typeName = GraphHelper.getTypeName(instanceVertex);
}
for (String traitNameToBeDeleted : traitNames) {
String relationshipLabel = GraphHelper.getTraitLabel(typeName, traitNameToBeDeleted);
deleteEdgeReference(instanceVertex, relationshipLabel, TypeCategory.CLASSIFICATION, false);
}
}
protected AtlasAttribute getAttributeForEdge(String edgeLabel) throws AtlasBaseException {
AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edgeLabel);
AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName());
AtlasStructType parentStructType = (AtlasStructType) parentType;
......@@ -438,15 +466,15 @@ public abstract class DeleteHandlerV1 {
* @throws AtlasException
*/
protected void deleteEdgeBetweenVertices(AtlasVertex outVertex, AtlasVertex inVertex, AtlasAttribute attribute) throws AtlasBaseException {
LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex),
attribute.getName());
String typeName = GraphHelper.getTypeName(outVertex);
String outId = GraphHelper.getGuid(outVertex);
if (LOG.isDebugEnabled()) {
LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex), attribute.getName());
}
AtlasObjectId objId = new AtlasObjectId(outId, typeName);
AtlasEntity.Status state = getState(outVertex);
final String typeName = GraphHelper.getTypeName(outVertex);
final String outId = GraphHelper.getGuid(outVertex);
final AtlasEntity.Status state = getState(outVertex);
if (state == DELETED || (outId != null && RequestContextV1.get().isDeletedEntity(objId))) {
if (state == DELETED || (outId != null && RequestContextV1.get().isDeletedEntity(outId))) {
//If the reference vertex is marked for deletion, skip updating the reference
return;
}
......@@ -455,95 +483,101 @@ public abstract class DeleteHandlerV1 {
String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(parentType, attribute.getName());
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
AtlasEdge edge = null;
AtlasAttributeDef attrDef = attribute.getAttributeDef();
AtlasType attrType = attribute.getAttributeType();
switch (attrType.getTypeCategory()) {
case OBJECT_ID_TYPE:
case OBJECT_ID_TYPE: {
//If its class attribute, its the only edge between two vertices
if (attrDef.getIsOptional()) {
edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
if (shouldUpdateInverseReferences) {
GraphHelper.setProperty(outVertex, propertyName, null);
}
} else {
// Cannot unset a required attribute.
throw new AtlasBaseException("Cannot unset required attribute " + propertyName +
" on " + GraphHelper.vertexString(outVertex) + " edge = " + edgeLabel);
throw new AtlasBaseException("Cannot unset required attribute " + propertyName + " on " + GraphHelper.vertexString(outVertex) + " edge = " + edgeLabel);
}
}
break;
case ARRAY:
case ARRAY: {
//If its array attribute, find the right edge between the two vertices and update array property
List<String> elements = GraphHelper.getListProperty(outVertex, propertyName);
if (elements != null) {
elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty()
for (String elementEdgeId : elements) {
AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId);
if (elementEdge == null) {
continue;
}
AtlasVertex elementVertex = elementEdge.getInVertex();
if (elementVertex.equals(inVertex)) {
edge = elementEdge;
//TODO element.size includes deleted items as well. should exclude
if (!attrDef.getIsOptional()
&& elements.size() <= attrDef.getValuesMinCount()) {
if (!attrDef.getIsOptional() && elements.size() <= attrDef.getValuesMinCount()) {
// Deleting this edge would violate the attribute's lower bound.
throw new AtlasBaseException(
"Cannot remove array element from required attribute " +
propertyName + " on "
+ GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge));
throw new AtlasBaseException("Cannot remove array element from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge));
}
if (shouldUpdateInverseReferences) {
//if composite attribute, remove the reference as well. else, just remove the edge
//for example, when table is deleted, process still references the table
//but when column is deleted, table will not reference the deleted column
LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge),
attribute.getName());
if (LOG.isDebugEnabled()) {
LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge), attribute.getName());
}
// Remove all occurrences of the edge ID from the list.
// This prevents dangling edge IDs (i.e. edge IDs for deleted edges)
// from the remaining in the list if there are duplicates.
elements.removeAll(Collections.singletonList(elementEdge.getId().toString()));
GraphHelper.setProperty(outVertex, propertyName, elements);
break;
}
}
}
}
}
break;
case MAP:
case MAP: {
//If its map attribute, find the right edge between two vertices and update map property
List<String> keys = GraphHelper.getListProperty(outVertex, propertyName);
if (keys != null) {
keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty()
for (String key : keys) {
String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
String mapEdgeId = GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class);
AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId);
if(mapEdge != null) {
if (mapEdge != null) {
AtlasVertex mapVertex = mapEdge.getInVertex();
if (mapVertex.getId().toString().equals(inVertex.getId().toString())) {
//TODO keys.size includes deleted items as well. should exclude
if (attrDef.getIsOptional() || keys.size() > attrDef.getValuesMinCount()) {
edge = mapEdge;
} else {
// Deleting this entry would violate the attribute's lower bound.
throw new AtlasBaseException(
"Cannot remove map entry " + keyPropertyName + " from required attribute " +
propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge));
throw new AtlasBaseException("Cannot remove map entry " + keyPropertyName + " from required attribute " + propertyName + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge));
}
if (shouldUpdateInverseReferences) {
//remove this key
LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key,
attribute.getName());
if (LOG.isDebugEnabled()) {
LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key, attribute.getName());
}
keys.remove(key);
GraphHelper.setProperty(outVertex, propertyName, keys);
GraphHelper.setProperty(outVertex, keyPropertyName, null);
......@@ -553,6 +587,7 @@ public abstract class DeleteHandlerV1 {
}
}
}
}
break;
case STRUCT:
......@@ -560,26 +595,32 @@ public abstract class DeleteHandlerV1 {
break;
default:
throw new IllegalStateException("There can't be an edge from " + GraphHelper.getVertexDetails(outVertex) + " to "
+ GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attribute.getName() + " which is not class/array/map attribute. found " + attrType.getTypeCategory().name());
throw new IllegalStateException("There can't be an edge from " + GraphHelper.getVertexDetails(outVertex) + " to " + GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attribute.getName() + " which is not class/array/map attribute. found " + attrType.getTypeCategory().name());
}
if (edge != null) {
deleteEdge(edge, false);
RequestContextV1 requestContext = RequestContextV1.get();
GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
requestContext.getRequestTime());
if (! requestContext.isUpdatedEntity(outId)) {
GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, requestContext.getRequestTime());
GraphHelper.setProperty(outVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser());
requestContext.recordEntityUpdate(new AtlasObjectId(outId, typeName));
requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(outVertex));
}
}
}
protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws AtlasBaseException {
//Update external references(incoming edges) to this vertex
if (LOG.isDebugEnabled()) {
LOG.debug("Setting the external references to {} to null(removing edges)", string(instanceVertex));
}
for (AtlasEdge edge : (Iterable<AtlasEdge>) instanceVertex.getEdges(AtlasEdgeDirection.IN)) {
AtlasEntity.Status edgeState = getState(edge);
if (edgeState == AtlasEntity.Status.ACTIVE) {
//Delete only the active edge references
AtlasAttribute attribute = getAttributeForEdge(edge.getLabel());
......@@ -587,6 +628,7 @@ public abstract class DeleteHandlerV1 {
deleteEdgeBetweenVertices(edge.getOutVertex(), edge.getInVertex(), attribute);
}
}
_deleteVertex(instanceVertex, force);
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
......@@ -85,6 +85,7 @@ public class EntityGraphMapper {
private final AtlasGraph graph;
private final DeleteHandlerV1 deleteHandler;
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphRetriever entityRetriever;
private final AtlasRelationshipStore relationshipStore;
@Inject
......@@ -92,6 +93,7 @@ public class EntityGraphMapper {
AtlasRelationshipStore relationshipStore) {
this.deleteHandler = deleteHandler;
this.typeRegistry = typeRegistry;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.graph = atlasGraph;
this.relationshipStore = relationshipStore;
}
......@@ -192,16 +194,16 @@ public class EntityGraphMapper {
RequestContextV1 req = RequestContextV1.get();
for (AtlasObjectId id : req.getDeletedEntityIds()) {
resp.addEntity(DELETE, constructHeader(id));
for (AtlasObjectId entity : req.getDeletedEntities()) {
resp.addEntity(DELETE, entity);
}
for (AtlasObjectId id : req.getUpdatedEntityIds()) {
for (AtlasObjectId entity : req.getUpdatedEntities()) {
if (isPartialUpdate) {
resp.addEntity(PARTIAL_UPDATE, constructHeader(id));
resp.addEntity(PARTIAL_UPDATE, entity);
}
else {
resp.addEntity(UPDATE, constructHeader(id));
resp.addEntity(UPDATE, entity);
}
}
......@@ -423,7 +425,6 @@ public class EntityGraphMapper {
AtlasVertex attrVertex = context.getDiscoveryContext().getResolvedEntityVertex(getGuid(ctx.getValue()));
recordEntityUpdate(attrVertex);
updateModificationMetadata(attrVertex);
}
//delete old reference
......@@ -494,9 +495,13 @@ public class EntityGraphMapper {
}
if (inverseUpdated) {
RequestContextV1 requestContext = RequestContextV1.get();
if (!requestContext.isDeletedEntity(GraphHelper.getGuid(inverseVertex))) {
updateModificationMetadata(inverseVertex);
AtlasObjectId inverseEntityId = new AtlasObjectId(getIdFromVertex(inverseVertex), inverseType.getTypeName());
RequestContextV1.get().recordEntityUpdate(inverseEntityId);
requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(inverseVertex));
}
}
}
......@@ -1441,32 +1446,15 @@ public class EntityGraphMapper {
return ret;
}
private void recordEntityUpdate(AtlasVertex vertex) {
AtlasObjectId objectId = new AtlasObjectId(GraphHelper.getGuid(vertex), GraphHelper.getTypeName(vertex));
private void recordEntityUpdate(AtlasVertex vertex) throws AtlasBaseException {
RequestContextV1 req = RequestContextV1.get();
if (!objectIdsContain(req.getUpdatedEntityIds(), objectId) && !objectIdsContain(req.getCreatedEntityIds(), objectId)) {
req.recordEntityUpdate(objectId);
}
}
private boolean objectIdsContain(Collection<AtlasObjectId> objectIds, AtlasObjectId objectId) {
boolean ret = false;
if (CollectionUtils.isEmpty(objectIds)) {
ret = false;
if (!req.isUpdatedEntity(GraphHelper.getGuid(vertex))) {
updateModificationMetadata(vertex);
} else {
for (AtlasObjectId id : objectIds) {
if (StringUtils.equals(id.getGuid(), objectId.getGuid())) {
ret = true;
break;
req.recordEntityUpdate(entityRetriever.toAtlasObjectId(vertex));
}
}
}
return ret;
}
private static void compactAttributes(AtlasEntity entity) {
if (entity != null) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
......@@ -147,6 +147,70 @@ public final class EntityGraphRetriever {
return atlasVertex != null ? mapVertexToAtlasEntityHeader(atlasVertex, attributes) : null;
}
public AtlasEntityHeader toAtlasEntityHeader(AtlasEntity entity) {
AtlasEntityHeader ret = null;
String typeName = entity.getTypeName();
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType != null) {
Map<String, Object> uniqueAttributes = new HashMap<>();
for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) {
Object attrValue = entity.getAttribute(attribute.getName());
if (attrValue != null) {
uniqueAttributes.put(attribute.getName(), attrValue);
}
}
ret = new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), uniqueAttributes);
}
return ret;
}
public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseException {
AtlasObjectId ret = null;
String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType != null) {
Map<String, Object> uniqueAttributes = new HashMap<>();
for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) {
Object attrValue = getVertexAttribute(entityVertex, attribute);
if (attrValue != null) {
uniqueAttributes.put(attribute.getName(), attrValue);
}
}
ret = new AtlasObjectId(entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class), typeName, uniqueAttributes);
}
return ret;
}
public AtlasVertex getReferencedEntityVertex(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex parentVertex) throws AtlasBaseException {
AtlasVertex entityVertex = null;
if (relationshipDirection == OUT) {
entityVertex = edge.getInVertex();
} else if (relationshipDirection == IN) {
entityVertex = edge.getOutVertex();
} else if (relationshipDirection == BOTH){
// since relationship direction is BOTH, edge direction can be inward or outward
// compare with parent entity vertex and pick the right reference vertex
if (StringUtils.equals(GraphHelper.getGuid(parentVertex), GraphHelper.getGuid(edge.getOutVertex()))) {
entityVertex = edge.getInVertex();
} else {
entityVertex = edge.getOutVertex();
}
}
return entityVertex;
}
public AtlasVertex getEntityVertex(String guid) throws AtlasBaseException {
AtlasVertex ret = AtlasGraphUtilsV1.findByGuid(guid);
......
server-api/src/main/java/org/apache/atlas/RequestContextV1.java
......@@ -31,18 +31,15 @@ public class RequestContextV1 {
private static final ThreadLocal<RequestContextV1> CURRENT_CONTEXT = new ThreadLocal<>();
private Set<AtlasObjectId> createdEntityIds = new LinkedHashSet<>();
private Set<AtlasObjectId> updatedEntityIds = new LinkedHashSet<>();
private Set<AtlasObjectId> deletedEntityIds = new LinkedHashSet<>();
private Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>();
private final Map<String, AtlasObjectId> updatedEntities = new HashMap<>();
private final Map<String, AtlasObjectId> deletedEntities = new HashMap<>();
private final Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>();
private final Metrics metrics = new Metrics();
private final long requestTime = System.currentTimeMillis();
private String user;
private final long requestTime;
private Metrics metrics = new Metrics();
private RequestContextV1() {
requestTime = System.currentTimeMillis();
}
//To handle gets from background threads where createContext() is not called
......@@ -62,10 +59,10 @@ public class RequestContextV1 {
RequestContextV1 instance = CURRENT_CONTEXT.get();
if (instance != null) {
if (instance.entityCacheV2 != null) {
instance.updatedEntities.clear();
instance.deletedEntities.clear();
instance.entityCacheV2.clear();
}
}
CURRENT_CONTEXT.remove();
}
......@@ -78,24 +75,16 @@ public class RequestContextV1 {
this.user = user;
}
public void recordEntityCreate(Collection<AtlasObjectId> createdEntityIds) {
this.createdEntityIds.addAll(createdEntityIds);
public void recordEntityUpdate(AtlasObjectId entity) {
if (entity != null && entity.getGuid() != null) {
updatedEntities.put(entity.getGuid(), entity);
}
public void recordEntityCreate(AtlasObjectId createdEntityId) {
this.createdEntityIds.add(createdEntityId);
}
public void recordEntityUpdate(Collection<AtlasObjectId> updatedEntityIds) {
this.updatedEntityIds.addAll(updatedEntityIds);
}
public void recordEntityUpdate(AtlasObjectId entityId) {
this.updatedEntityIds.add(entityId);
public void recordEntityDelete(AtlasObjectId entity) {
if (entity != null && entity.getGuid() != null) {
deletedEntities.put(entity.getGuid(), entity);
}
public void recordEntityDelete(AtlasObjectId entityId) {
deletedEntityIds.add(entityId);
}
/**
......@@ -108,16 +97,12 @@ public class RequestContextV1 {
}
}
public Collection<AtlasObjectId> getCreatedEntityIds() {
return createdEntityIds;
}
public Collection<AtlasObjectId> getUpdatedEntityIds() {
return updatedEntityIds;
public Collection<AtlasObjectId> getUpdatedEntities() {
return updatedEntities.values();
}
public Collection<AtlasObjectId> getDeletedEntityIds() {
return deletedEntityIds;
public Collection<AtlasObjectId> getDeletedEntities() {
return deletedEntities.values();
}
/**
......@@ -135,8 +120,12 @@ public class RequestContextV1 {
return requestTime;
}
public boolean isDeletedEntity(AtlasObjectId entityId) {
return deletedEntityIds.contains(entityId);
public boolean isUpdatedEntity(String guid) {
return updatedEntities.containsKey(guid);
}
public boolean isDeletedEntity(String guid) {
return deletedEntities.containsKey(guid);
}
public static Metrics getMetrics() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment