Skip to content

A
atlas
Commit 154dda0e by Madhan Neethiraj
Options

ATLAS-2169: delete request fails when hard-delete is configured

parent ad6b07a9
Showing with 170 additions and 136 deletions
intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
......@@ -195,7 +195,7 @@ public class EntityMutationResponse {
public void addEntity(EntityOperation op, AtlasEntityHeader header) {
// if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE
if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) {
if (entityHeaderExists(getCreatedEntities(), header)) {
if (entityHeaderExists(getCreatedEntities(), header.getGuid())) {
return;
}
}
......@@ -211,17 +211,42 @@ public class EntityMutationResponse {
mutatedEntities.put(op, opEntities);
}
if (!entityHeaderExists(opEntities, header)) {
if (!entityHeaderExists(opEntities, header.getGuid())) {
opEntities.add(header);
}
}
private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, AtlasEntityHeader newEntityHeader) {
@JsonIgnore
public void addEntity(EntityOperation op, AtlasObjectId entity) {
if (mutatedEntities == null) {
mutatedEntities = new HashMap<>();
} else {
// if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE
if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) {
if (entityHeaderExists(getCreatedEntities(), entity.getGuid())) {
return;
}
}
}
List<AtlasEntityHeader> opEntities = mutatedEntities.get(op);
if (opEntities == null) {
opEntities = new ArrayList<>();
mutatedEntities.put(op, opEntities);
}
if (!entityHeaderExists(opEntities, entity.getGuid())) {
opEntities.add(new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), entity.getUniqueAttributes()));
}
}
private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, String guid) {
boolean ret = false;
if (CollectionUtils.isNotEmpty(entityHeaders) && newEntityHeader != null) {
if (CollectionUtils.isNotEmpty(entityHeaders) && guid != null) {
for (AtlasEntityHeader entityHeader : entityHeaders) {
if (StringUtils.equals(entityHeader.getGuid(), newEntityHeader.getGuid())) {
if (StringUtils.equals(entityHeader.getGuid(), guid)) {
ret = true;
break;
}
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -68,9 +68,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
/**
* Utility class for graph operations.
......@@ -888,24 +885,23 @@ public final class GraphHelper {
* Guid and AtlasVertex combo
*/
public static class VertexInfo {
private String guid;
private AtlasVertex vertex;
private String typeName;
private final AtlasObjectId entity;
private final AtlasVertex vertex;
public VertexInfo(String guid, AtlasVertex vertex, String typeName) {
this.guid = guid;
public VertexInfo(AtlasObjectId entity, AtlasVertex vertex) {
this.entity = entity;
this.vertex = vertex;
this.typeName = typeName;
}
public String getGuid() {
return guid;
}
public AtlasObjectId getEntity() { return entity; }
public AtlasVertex getVertex() {
return vertex;
}
public String getGuid() {
return entity.getGuid();
}
public String getTypeName() {
return typeName;
return entity.getTypeName();
}
@Override
......@@ -913,14 +909,13 @@ public final class GraphHelper {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
VertexInfo that = (VertexInfo) o;
return Objects.equals(guid, that.guid) &&
Objects.equals(vertex, that.vertex) &&
Objects.equals(typeName, that.typeName);
return Objects.equals(entity, that.entity) &&
Objects.equals(vertex, that.vertex);
}
@Override
public int hashCode() {
return Objects.hash(guid, vertex, typeName);
return Objects.hash(entity, vertex);
}
}
......@@ -1304,39 +1299,4 @@ public final class GraphHelper {
return StringUtils.isNotEmpty(edge.getLabel()) ? edgeLabel.startsWith("r:") : false;
}
public static AtlasObjectId getReferenceObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection,
AtlasVertex parentVertex) {
AtlasObjectId ret = null;
if (relationshipDirection == OUT) {
ret = getAtlasObjectIdForInVertex(edge);
} else if (relationshipDirection == IN) {
ret = getAtlasObjectIdForOutVertex(edge);
} else if (relationshipDirection == BOTH){
// since relationship direction is BOTH, edge direction can be inward or outward
// compare with parent entity vertex and pick the right reference vertex
if (verticesEquals(parentVertex, edge.getOutVertex())) {
ret = getAtlasObjectIdForInVertex(edge);
} else {
ret = getAtlasObjectIdForOutVertex(edge);
}
}
return ret;
}
public static AtlasObjectId getAtlasObjectIdForOutVertex(AtlasEdge edge) {
return new AtlasObjectId(getGuid(edge.getOutVertex()), getTypeName(edge.getOutVertex()));
}
public static AtlasObjectId getAtlasObjectIdForInVertex(AtlasEdge edge) {
return new AtlasObjectId(getGuid(edge.getInVertex()), getTypeName(edge.getInVertex()));
}
private static boolean verticesEquals(AtlasVertex vertexA, AtlasVertex vertexB) {
return StringUtils.equals(getGuid(vertexB), getGuid(vertexA));
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java
......@@ -158,7 +158,7 @@ public class AtlasEntityChangeNotifier {
return;
}
List<Referenceable> typedRefInsts = toReferenceables(entityHeaders);
List<Referenceable> typedRefInsts = toReferenceables(entityHeaders, operation);
for (EntityChangeListener listener : entityChangeListeners) {
try {
......@@ -180,11 +180,18 @@ public class AtlasEntityChangeNotifier {
}
}
private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders) throws AtlasBaseException {
private List<Referenceable> toReferenceables(List<AtlasEntityHeader> entityHeaders, EntityOperation operation) throws AtlasBaseException {
List<Referenceable> ret = new ArrayList<>(entityHeaders.size());
for (AtlasEntityHeader entityHeader : entityHeaders) {
ret.add(toReferenceable(entityHeader.getGuid()));
// delete notifications don't need all attributes. Hence the special handling for delete operation
if (operation == EntityOperation.DELETE) {
for (AtlasEntityHeader entityHeader : entityHeaders) {
ret.add(new Referenceable(entityHeader.getGuid(), entityHeader.getTypeName(), entityHeader.getAttributes()));
}
} else {
for (AtlasEntityHeader entityHeader : entityHeaders) {
ret.add(toReferenceable(entityHeader.getGuid()));
}
}
return ret;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
......@@ -647,14 +647,16 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
private EntityMutationResponse deleteVertices(Collection<AtlasVertex> deletionCandidates) throws AtlasBaseException {
EntityMutationResponse response = new EntityMutationResponse();
deleteHandler.deleteEntities(deletionCandidates);
RequestContextV1 req = RequestContextV1.get();
for (AtlasObjectId id : req.getDeletedEntityIds()) {
response.addEntity(DELETE, EntityGraphMapper.constructHeader(id));
RequestContextV1 req = RequestContextV1.get();
deleteHandler.deleteEntities(deletionCandidates); // this will update req with list of deleted/updated entities
for (AtlasObjectId entity : req.getDeletedEntities()) {
response.addEntity(DELETE, entity);
}
for (AtlasObjectId id : req.getUpdatedEntityIds()) {
response.addEntity(UPDATE, EntityGraphMapper.constructHeader(id));
for (AtlasObjectId entity : req.getUpdatedEntities()) {
response.addEntity(UPDATE, entity);
}
return response;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
......@@ -85,6 +85,7 @@ public class EntityGraphMapper {
private final AtlasGraph graph;
private final DeleteHandlerV1 deleteHandler;
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphRetriever entityRetriever;
private final AtlasRelationshipStore relationshipStore;
@Inject
......@@ -92,6 +93,7 @@ public class EntityGraphMapper {
AtlasRelationshipStore relationshipStore) {
this.deleteHandler = deleteHandler;
this.typeRegistry = typeRegistry;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.graph = atlasGraph;
this.relationshipStore = relationshipStore;
}
......@@ -192,16 +194,16 @@ public class EntityGraphMapper {
RequestContextV1 req = RequestContextV1.get();
for (AtlasObjectId id : req.getDeletedEntityIds()) {
resp.addEntity(DELETE, constructHeader(id));
for (AtlasObjectId entity : req.getDeletedEntities()) {
resp.addEntity(DELETE, entity);
}
for (AtlasObjectId id : req.getUpdatedEntityIds()) {
for (AtlasObjectId entity : req.getUpdatedEntities()) {
if (isPartialUpdate) {
resp.addEntity(PARTIAL_UPDATE, constructHeader(id));
resp.addEntity(PARTIAL_UPDATE, entity);
}
else {
resp.addEntity(UPDATE, constructHeader(id));
resp.addEntity(UPDATE, entity);
}
}
......@@ -423,7 +425,6 @@ public class EntityGraphMapper {
AtlasVertex attrVertex = context.getDiscoveryContext().getResolvedEntityVertex(getGuid(ctx.getValue()));
recordEntityUpdate(attrVertex);
updateModificationMetadata(attrVertex);
}
//delete old reference
......@@ -494,9 +495,13 @@ public class EntityGraphMapper {
}
if (inverseUpdated) {
updateModificationMetadata(inverseVertex);
AtlasObjectId inverseEntityId = new AtlasObjectId(getIdFromVertex(inverseVertex), inverseType.getTypeName());
RequestContextV1.get().recordEntityUpdate(inverseEntityId);
RequestContextV1 requestContext = RequestContextV1.get();
if (!requestContext.isDeletedEntity(GraphHelper.getGuid(inverseVertex))) {
updateModificationMetadata(inverseVertex);
requestContext.recordEntityUpdate(entityRetriever.toAtlasObjectId(inverseVertex));
}
}
}
......@@ -1441,31 +1446,14 @@ public class EntityGraphMapper {
return ret;
}
private void recordEntityUpdate(AtlasVertex vertex) {
AtlasObjectId objectId = new AtlasObjectId(GraphHelper.getGuid(vertex), GraphHelper.getTypeName(vertex));
RequestContextV1 req = RequestContextV1.get();
if (!objectIdsContain(req.getUpdatedEntityIds(), objectId) && !objectIdsContain(req.getCreatedEntityIds(), objectId)) {
req.recordEntityUpdate(objectId);
}
}
private boolean objectIdsContain(Collection<AtlasObjectId> objectIds, AtlasObjectId objectId) {
boolean ret = false;
private void recordEntityUpdate(AtlasVertex vertex) throws AtlasBaseException {
RequestContextV1 req = RequestContextV1.get();
if (CollectionUtils.isEmpty(objectIds)) {
ret = false;
if (!req.isUpdatedEntity(GraphHelper.getGuid(vertex))) {
updateModificationMetadata(vertex);
} else {
for (AtlasObjectId id : objectIds) {
if (StringUtils.equals(id.getGuid(), objectId.getGuid())) {
ret = true;
break;
}
}
req.recordEntityUpdate(entityRetriever.toAtlasObjectId(vertex));
}
return ret;
}
private static void compactAttributes(AtlasEntity entity) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
......@@ -147,6 +147,70 @@ public final class EntityGraphRetriever {
return atlasVertex != null ? mapVertexToAtlasEntityHeader(atlasVertex, attributes) : null;
}
public AtlasEntityHeader toAtlasEntityHeader(AtlasEntity entity) {
AtlasEntityHeader ret = null;
String typeName = entity.getTypeName();
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType != null) {
Map<String, Object> uniqueAttributes = new HashMap<>();
for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) {
Object attrValue = entity.getAttribute(attribute.getName());
if (attrValue != null) {
uniqueAttributes.put(attribute.getName(), attrValue);
}
}
ret = new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), uniqueAttributes);
}
return ret;
}
public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseException {
AtlasObjectId ret = null;
String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType != null) {
Map<String, Object> uniqueAttributes = new HashMap<>();
for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) {
Object attrValue = getVertexAttribute(entityVertex, attribute);
if (attrValue != null) {
uniqueAttributes.put(attribute.getName(), attrValue);
}
}
ret = new AtlasObjectId(entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class), typeName, uniqueAttributes);
}
return ret;
}
public AtlasVertex getReferencedEntityVertex(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex parentVertex) throws AtlasBaseException {
AtlasVertex entityVertex = null;
if (relationshipDirection == OUT) {
entityVertex = edge.getInVertex();
} else if (relationshipDirection == IN) {
entityVertex = edge.getOutVertex();
} else if (relationshipDirection == BOTH){
// since relationship direction is BOTH, edge direction can be inward or outward
// compare with parent entity vertex and pick the right reference vertex
if (StringUtils.equals(GraphHelper.getGuid(parentVertex), GraphHelper.getGuid(edge.getOutVertex()))) {
entityVertex = edge.getInVertex();
} else {
entityVertex = edge.getOutVertex();
}
}
return entityVertex;
}
public AtlasVertex getEntityVertex(String guid) throws AtlasBaseException {
AtlasVertex ret = AtlasGraphUtilsV1.findByGuid(guid);
......
server-api/src/main/java/org/apache/atlas/RequestContextV1.java
......@@ -31,18 +31,15 @@ public class RequestContextV1 {
private static final ThreadLocal<RequestContextV1> CURRENT_CONTEXT = new ThreadLocal<>();
private Set<AtlasObjectId> createdEntityIds = new LinkedHashSet<>();
private Set<AtlasObjectId> updatedEntityIds = new LinkedHashSet<>();
private Set<AtlasObjectId> deletedEntityIds = new LinkedHashSet<>();
private Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>();
private final Map<String, AtlasObjectId> updatedEntities = new HashMap<>();
private final Map<String, AtlasObjectId> deletedEntities = new HashMap<>();
private final Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>();
private final Metrics metrics = new Metrics();
private final long requestTime = System.currentTimeMillis();
private String user;
private final long requestTime;
private Metrics metrics = new Metrics();
private RequestContextV1() {
requestTime = System.currentTimeMillis();
}
//To handle gets from background threads where createContext() is not called
......@@ -62,9 +59,9 @@ public class RequestContextV1 {
RequestContextV1 instance = CURRENT_CONTEXT.get();
if (instance != null) {
if (instance.entityCacheV2 != null) {
instance.entityCacheV2.clear();
}
instance.updatedEntities.clear();
instance.deletedEntities.clear();
instance.entityCacheV2.clear();
}
CURRENT_CONTEXT.remove();
......@@ -78,24 +75,16 @@ public class RequestContextV1 {
this.user = user;
}
public void recordEntityCreate(Collection<AtlasObjectId> createdEntityIds) {
this.createdEntityIds.addAll(createdEntityIds);
}
public void recordEntityCreate(AtlasObjectId createdEntityId) {
this.createdEntityIds.add(createdEntityId);
}
public void recordEntityUpdate(Collection<AtlasObjectId> updatedEntityIds) {
this.updatedEntityIds.addAll(updatedEntityIds);
}
public void recordEntityUpdate(AtlasObjectId entityId) {
this.updatedEntityIds.add(entityId);
public void recordEntityUpdate(AtlasObjectId entity) {
if (entity != null && entity.getGuid() != null) {
updatedEntities.put(entity.getGuid(), entity);
}
}
public void recordEntityDelete(AtlasObjectId entityId) {
deletedEntityIds.add(entityId);
public void recordEntityDelete(AtlasObjectId entity) {
if (entity != null && entity.getGuid() != null) {
deletedEntities.put(entity.getGuid(), entity);
}
}
/**
......@@ -108,16 +97,12 @@ public class RequestContextV1 {
}
}
public Collection<AtlasObjectId> getCreatedEntityIds() {
return createdEntityIds;
}
public Collection<AtlasObjectId> getUpdatedEntityIds() {
return updatedEntityIds;
public Collection<AtlasObjectId> getUpdatedEntities() {
return updatedEntities.values();
}
public Collection<AtlasObjectId> getDeletedEntityIds() {
return deletedEntityIds;
public Collection<AtlasObjectId> getDeletedEntities() {
return deletedEntities.values();
}
/**
......@@ -135,8 +120,12 @@ public class RequestContextV1 {
return requestTime;
}
public boolean isDeletedEntity(AtlasObjectId entityId) {
return deletedEntityIds.contains(entityId);
public boolean isUpdatedEntity(String guid) {
return updatedEntities.containsKey(guid);
}
public boolean isDeletedEntity(String guid) {
return deletedEntities.containsKey(guid);
}
public static Metrics getMetrics() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment