diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 28054ef..6a92282 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -50,6 +50,8 @@ public class AtlasEntityType extends AtlasStructType { private final AtlasEntityDef entityDef; private final String typeQryStr; + private static final String INTERNAL_TYPENAME = "__internal"; + private List<AtlasEntityType> superTypes = Collections.emptyList(); private Set<String> allSuperTypes = Collections.emptySet(); private Set<String> subTypes = Collections.emptySet(); @@ -59,6 +61,7 @@ public class AtlasEntityType extends AtlasStructType { private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap(); private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap(); private String typeAndAllSubTypesQryStr = ""; + private boolean isInternalType = false; public AtlasEntityType(AtlasEntityDef entityDef) { @@ -145,6 +148,10 @@ public class AtlasEntityType extends AtlasStructType { } for (String superTypeName : allSuperTypes) { + if (INTERNAL_TYPENAME.equals(superTypeName)) { + isInternalType = true; + } + AtlasEntityType superType = typeRegistry.getEntityTypeByName(superTypeName); Map<String, AtlasAttribute> superTypeRelationshipAttributes = superType.getRelationshipAttributes(); @@ -206,6 +213,10 @@ public class AtlasEntityType extends AtlasStructType { return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName); } + public boolean isInternalType() { + return isInternalType; + } + public Map<String, AtlasAttribute> getRelationshipAttributes() { return relationshipAttributes; } public AtlasAttribute getRelationshipAttribute(String attributeName) { return relationshipAttributes.get(attributeName); } diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 48db657..86a6b81 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -647,8 +647,8 @@ public final class GraphHelper { public static <T> T getSingleValuedProperty(AtlasElement element, String propertyName, Class<T> clazz) { String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); - if (LOG.isDebugEnabled()) { - LOG.debug("Reading property {} from {}", actualPropertyName, string(element)); + if (LOG.isTraceEnabled()) { + LOG.trace("Reading property {} from {}", actualPropertyName, string(element)); } return element.getProperty(actualPropertyName, clazz); @@ -658,8 +658,8 @@ public final class GraphHelper { public static Object getProperty(AtlasVertex<?,?> vertex, String propertyName) { String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); - if (LOG.isDebugEnabled()) { - LOG.debug("Reading property {} from {}", actualPropertyName, string(vertex)); + if (LOG.isTraceEnabled()) { + LOG.trace("Reading property {} from {}", actualPropertyName, string(vertex)); } if(AtlasGraphProvider.getGraphInstance().isMultiProperty(actualPropertyName)) { @@ -673,8 +673,8 @@ public final class GraphHelper { public static Object getProperty(AtlasEdge<?,?> edge, String propertyName) { String actualPropertyName = GraphHelper.encodePropertyKey(propertyName); - if (LOG.isDebugEnabled()) { - LOG.debug("Reading property {} from {}", actualPropertyName, string(edge)); + if (LOG.isTraceEnabled()) { + LOG.trace("Reading property {} from {}", actualPropertyName, string(edge)); } return edge.getProperty(actualPropertyName, Object.class); @@ -736,18 +736,14 @@ public final class GraphHelper { * @param edge */ public void removeEdge(AtlasEdge edge) { - String edgeString = null; - if (LOG.isDebugEnabled()) { - edgeString = string(edge); - - LOG.debug("Removing {}", edgeString); + LOG.debug("==> removeEdge({})", string(edge)); } graph.removeEdge(edge); if (LOG.isDebugEnabled()) { - LOG.info("Removed {}", edgeString); + LOG.info("<== removeEdge()"); } } @@ -757,18 +753,14 @@ public final class GraphHelper { * @param vertex */ public void removeVertex(AtlasVertex vertex) { - String vertexString = null; - if (LOG.isDebugEnabled()) { - vertexString = string(vertex); - - LOG.debug("Removing {}", vertexString); + LOG.debug("==> GraphHelper.removeVertex({})", string(vertex)); } graph.removeVertex(vertex); if (LOG.isDebugEnabled()) { - LOG.info("Removed {}", vertexString); + LOG.debug("<== GraphHelper.removeVertex()"); } } @@ -1580,7 +1572,7 @@ public final class GraphHelper { } // newly added - public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) { + public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) { String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName); if(AtlasGraphUtilsV1.isReference(elementType)) { return (List)instanceVertex.getListProperty(encodedPropertyName, AtlasEdge.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AbstractGlossaryDTO.java b/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AbstractGlossaryDTO.java index 722a463..72add0e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AbstractGlossaryDTO.java +++ b/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AbstractGlossaryDTO.java @@ -24,7 +24,6 @@ import org.apache.atlas.model.glossary.relations.AtlasRelatedCategoryHeader; import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader; import org.apache.atlas.model.instance.AtlasRelatedObjectId; -import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.repository.ogm.AbstractDataTransferObject; import org.apache.atlas.type.AtlasTypeRegistry; @@ -151,9 +150,7 @@ public abstract class AbstractGlossaryDTO<T extends AtlasBaseModelObject> extend ret = new HashSet<>(); for (Object t : (Collection) relatedObjectIds) { if (t instanceof AtlasRelatedObjectId) { - if (((AtlasRelatedObjectId) t).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) { - ret.add(constructRelatedTermId((AtlasRelatedObjectId) t)); - } + ret.add(constructRelatedTermId((AtlasRelatedObjectId) t)); } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryCategoryDTO.java b/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryCategoryDTO.java index b26ee76..4fe048d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryCategoryDTO.java +++ b/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryCategoryDTO.java @@ -60,17 +60,13 @@ public class AtlasGlossaryCategoryDTO extends AbstractGlossaryDTO<AtlasGlossaryC Object anchor = entity.getRelationshipAttribute("anchor"); if (anchor instanceof AtlasRelatedObjectId) { LOG.debug("Processing anchor"); - if (((AtlasRelatedObjectId) anchor).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) { - ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor)); - } + ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor)); } Object parentCategory = entity.getRelationshipAttribute("parentCategory"); if (parentCategory instanceof AtlasRelatedObjectId) { LOG.debug("Processing parentCategory"); - if (((AtlasRelatedObjectId) parentCategory).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) { - ret.setParentCategory(constructRelatedCategoryId((AtlasRelatedObjectId) parentCategory)); - } + ret.setParentCategory(constructRelatedCategoryId((AtlasRelatedObjectId) parentCategory)); } Object childrenCategories = entity.getRelationshipAttribute("childrenCategories"); @@ -90,9 +86,7 @@ public class AtlasGlossaryCategoryDTO extends AbstractGlossaryDTO<AtlasGlossaryC LOG.debug("Processing terms"); for (Object term : (Collection) terms) { if (term instanceof AtlasRelatedObjectId) { - if (((AtlasRelatedObjectId) term).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) { - ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) term)); - } + ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) term)); } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryDTO.java b/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryDTO.java index e378660..75f3f02 100644 --- a/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryDTO.java +++ b/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryDTO.java @@ -21,7 +21,6 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.glossary.AtlasGlossary; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasRelatedObjectId; -import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.type.AtlasTypeRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,9 +68,7 @@ public class AtlasGlossaryDTO extends AbstractGlossaryDTO<AtlasGlossary> { if (categoriesAttr instanceof Collection) { for (Object o : (Collection) categoriesAttr) { if (o instanceof AtlasRelatedObjectId) { - if (((AtlasRelatedObjectId) o).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) { - ret.addCategory(constructRelatedCategoryId((AtlasRelatedObjectId) o)); - } + ret.addCategory(constructRelatedCategoryId((AtlasRelatedObjectId) o)); } } } @@ -83,9 +80,7 @@ public class AtlasGlossaryDTO extends AbstractGlossaryDTO<AtlasGlossary> { if (termsAttr instanceof Collection) { for (Object o : (Collection) termsAttr) { if (o instanceof AtlasRelatedObjectId) { - if (((AtlasRelatedObjectId) o).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) { - ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) o)); - } + ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) o)); } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryTermDTO.java b/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryTermDTO.java index 92bb242..2d3bf19 100644 --- a/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryTermDTO.java +++ b/repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryTermDTO.java @@ -21,7 +21,6 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.glossary.AtlasGlossaryTerm; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasRelatedObjectId; -import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; @@ -65,9 +64,7 @@ public class AtlasGlossaryTermDTO extends AbstractGlossaryDTO<AtlasGlossaryTerm> Object anchor = entity.getRelationshipAttribute("anchor"); if (anchor instanceof AtlasRelatedObjectId) { LOG.debug("Processing anchor"); - if (((AtlasRelatedObjectId) anchor).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) { - ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor)); - } + ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor)); } Object categories = entity.getRelationshipAttribute("categories"); @@ -75,9 +72,7 @@ public class AtlasGlossaryTermDTO extends AbstractGlossaryDTO<AtlasGlossaryTerm> LOG.debug("Processing categories"); for (Object category : (Collection) categories) { if (category instanceof AtlasRelatedObjectId) { - if (((AtlasRelatedObjectId) category).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) { - ret.addCategory(constructTermCategorizationId((AtlasRelatedObjectId) category)); - } + ret.addCategory(constructTermCategorizationId((AtlasRelatedObjectId) category)); } } } @@ -90,9 +85,7 @@ public class AtlasGlossaryTermDTO extends AbstractGlossaryDTO<AtlasGlossaryTerm> for (Object assignedEntity : (Collection) assignedEntities) { if (assignedEntity instanceof AtlasRelatedObjectId) { AtlasRelatedObjectId id = (AtlasRelatedObjectId) assignedEntity; - if (id.getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) { - ret.addAssignedEntity(id); - } + ret.addAssignedEntity(id); } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java index e3f6f88..9aebde2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java @@ -30,6 +30,8 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations.EntityOperation; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.v1.model.instance.Referenceable; import org.apache.atlas.v1.model.instance.Struct; import org.apache.atlas.repository.Constants; @@ -48,6 +50,7 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Set; import static org.apache.atlas.util.AtlasRepositoryConfiguration.isV2EntityNotificationEnabled; @@ -61,15 +64,20 @@ public class AtlasEntityChangeNotifier { private final Set<EntityChangeListenerV2> entityChangeListenersV2; private final AtlasInstanceConverter instanceConverter; private final FullTextMapperV2 fullTextMapperV2; + private final AtlasTypeRegistry atlasTypeRegistry; @Inject - public AtlasEntityChangeNotifier(Set<EntityChangeListener> entityChangeListeners, Set<EntityChangeListenerV2> entityChangeListenersV2, - AtlasInstanceConverter instanceConverter, final FullTextMapperV2 fullTextMapperV2) { + public AtlasEntityChangeNotifier(Set<EntityChangeListener> entityChangeListeners, + Set<EntityChangeListenerV2> entityChangeListenersV2, + AtlasInstanceConverter instanceConverter, + FullTextMapperV2 fullTextMapperV2, + AtlasTypeRegistry atlasTypeRegistry) { this.entityChangeListeners = entityChangeListeners; this.entityChangeListenersV2 = entityChangeListenersV2; this.instanceConverter = instanceConverter; this.fullTextMapperV2 = fullTextMapperV2; + this.atlasTypeRegistry = atlasTypeRegistry; } public void onEntitiesMutated(EntityMutationResponse entityMutationResponse, boolean isImport) throws AtlasBaseException { @@ -282,6 +290,17 @@ public class AtlasEntityChangeNotifier { if (CollectionUtils.isNotEmpty(entityHeaders)) { for (AtlasEntityHeader entityHeader : entityHeaders) { String entityGuid = entityHeader.getGuid(); + String typeName = entityHeader.getTypeName(); + + // Skip all internal types as the HARD DELETE will cause lookup errors + AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName(typeName); + if (Objects.nonNull(entityType) && entityType.isInternalType()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping internal type = {}", typeName); + } + continue; + } + AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid); if (entityWithExtInfo != null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index c00015e..c0f2fc3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -72,14 +72,12 @@ import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getSt public abstract class DeleteHandlerV1 { public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class); - + protected static final GraphHelper graphHelper = GraphHelper.getInstance(); private final AtlasTypeRegistry typeRegistry; private final EntityGraphRetriever entityRetriever; private final boolean shouldUpdateInverseReferences; private final boolean softDelete; - protected static final GraphHelper graphHelper = GraphHelper.getInstance(); - public DeleteHandlerV1(AtlasTypeRegistry typeRegistry, boolean shouldUpdateInverseReference, boolean softDelete) { this.typeRegistry = typeRegistry; this.entityRetriever = new EntityGraphRetriever(typeRegistry); @@ -122,7 +120,7 @@ public abstract class DeleteHandlerV1 { // Delete traits and vertices. for (AtlasVertex deletionCandidateVertex : deletionCandidateVertices) { deleteAllClassifications(deletionCandidateVertex); - deleteTypeVertex(deletionCandidateVertex, false); + deleteTypeVertex(deletionCandidateVertex, isInternalType(deletionCandidateVertex)); } } @@ -134,7 +132,8 @@ public abstract class DeleteHandlerV1 { */ public void deleteRelationships(Collection<AtlasEdge> edges) throws AtlasBaseException { for (AtlasEdge edge : edges) { - if (getState(edge) == DELETED) { + boolean isInternal = isInternalType(edge.getInVertex()) || isInternalType(edge.getOutVertex()); + if (!isInternal && getState(edge) == DELETED) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping deletion of {} as it is already deleted", getIdFromEdge(edge)); } @@ -142,7 +141,7 @@ public abstract class DeleteHandlerV1 { continue; } - deleteEdge(edge, false); + deleteEdge(edge, isInternal); } } @@ -272,12 +271,21 @@ public abstract class DeleteHandlerV1 { public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, boolean forceDeleteStructTrait, AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex entityVertex) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("Deleting {}", string(edge)); + LOG.debug("Deleting {}, force = {}", string(edge), forceDeleteStructTrait); } - boolean forceDelete = (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait; + boolean isInternalType = isInternalType(entityVertex); + boolean forceDelete = (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) + && (forceDeleteStructTrait || isInternalType); + + if (LOG.isDebugEnabled()) { + LOG.debug("isInternal = {}, forceDelete = {}", isInternalType, forceDelete); + } if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Processing delete for typeCategory={}, isOwned={}", typeCategory, isOwned); + } //If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities. //If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled //through this delete, hence delete the edge and the reference vertex. @@ -293,7 +301,7 @@ public abstract class DeleteHandlerV1 { // for relationship edges, inverse vertex's relationship attribute doesn't need to be updated. // only delete the reference relationship edge if (isRelationshipEdge(edge)) { - deleteEdge(edge, false); + deleteEdge(edge, isInternalType); AtlasVertex referencedVertex = entityRetriever.getReferencedEntityVertex(edge, relationshipDirection, entityVertex); @@ -311,53 +319,13 @@ public abstract class DeleteHandlerV1 { //legacy case - not a relationship edge //If deleting just the edge, reverse attribute should be updated for any references //For example, for the department type system, if the person's manager edge is deleted, subordinates of manager should be updated - deleteEdge(edge, true, false); + deleteEdge(edge, true, isInternalType); } } return !softDelete || forceDelete; } - protected void deleteEdge(AtlasEdge edge, boolean updateInverseAttribute, boolean force) throws AtlasBaseException { - //update inverse attribute - if (updateInverseAttribute) { - AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edge.getLabel()); - AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName()); - - if (parentType instanceof AtlasEntityType) { - AtlasEntityType parentEntityType = (AtlasEntityType) parentType; - AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName()); - - if (attribute.getInverseRefAttribute() != null) { - deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), attribute.getInverseRefAttribute()); - } - } - } - - deleteEdge(edge, force); - } - - - protected void deleteTypeVertex(AtlasVertex instanceVertex, TypeCategory typeCategory, boolean force) throws AtlasBaseException { - switch (typeCategory) { - case STRUCT: - deleteTypeVertex(instanceVertex, force); - break; - - case CLASSIFICATION: - deleteClassificationVertex(instanceVertex, force); - break; - - case ENTITY: - case OBJECT_ID_TYPE: - deleteEntities(Collections.singletonList(instanceVertex)); - break; - - default: - throw new IllegalStateException("Type category " + typeCategory + " not handled"); - } - } - public List<AtlasVertex> addTagPropagation(AtlasVertex classificationVertex, List<AtlasVertex> propagatedEntityVertices) { List<AtlasVertex> ret = null; @@ -448,18 +416,51 @@ public abstract class DeleteHandlerV1 { updateModificationMetadata(entityVertex); } - private void removeFromPropagatedTraitNames(AtlasVertex entityVertex, String classificationName) { - if (entityVertex != null && StringUtils.isNotEmpty(classificationName)) { - List<String> propagatedTraitNames = getTraitNames(entityVertex, true); + public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, TypeCategory typeCategory, boolean isOwned) throws AtlasBaseException { + AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); - propagatedTraitNames.remove(classificationName); + if (edge != null) { + deleteEdgeReference(edge, typeCategory, isOwned, false, outVertex); + } + } - entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY); + protected void deleteEdge(AtlasEdge edge, boolean updateInverseAttribute, boolean force) throws AtlasBaseException { + //update inverse attribute + if (updateInverseAttribute) { + AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edge.getLabel()); + AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName()); - for (String propagatedTraitName : propagatedTraitNames) { - addToPropagatedTraitNames(entityVertex, propagatedTraitName); + if (parentType instanceof AtlasEntityType) { + AtlasEntityType parentEntityType = (AtlasEntityType) parentType; + AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName()); + + if (attribute.getInverseRefAttribute() != null) { + deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), attribute.getInverseRefAttribute()); + } } } + + deleteEdge(edge, force); + } + + protected void deleteTypeVertex(AtlasVertex instanceVertex, TypeCategory typeCategory, boolean force) throws AtlasBaseException { + switch (typeCategory) { + case STRUCT: + deleteTypeVertex(instanceVertex, force); + break; + + case CLASSIFICATION: + deleteClassificationVertex(instanceVertex, force); + break; + + case ENTITY: + case OBJECT_ID_TYPE: + deleteEntities(Collections.singletonList(instanceVertex)); + break; + + default: + throw new IllegalStateException("Type category " + typeCategory + " not handled"); + } } /** @@ -469,7 +470,7 @@ public abstract class DeleteHandlerV1 { */ protected void deleteTypeVertex(AtlasVertex instanceVertex, boolean force) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("Deleting {}", string(instanceVertex)); + LOG.debug("Deleting {}, force={}", string(instanceVertex), force); } String typeName = GraphHelper.getTypeName(instanceVertex); @@ -543,31 +544,6 @@ public abstract class DeleteHandlerV1 { deleteVertex(instanceVertex, force); } - public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, TypeCategory typeCategory, boolean isOwned) throws AtlasBaseException { - AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); - - if (edge != null) { - deleteEdgeReference(edge, typeCategory, isOwned, false, outVertex); - } - } - - /** - * Delete all associated classifications from the specified entity vertex. - * @param instanceVertex - * @throws AtlasException - */ - private void deleteAllClassifications(AtlasVertex instanceVertex) throws AtlasBaseException { - List<AtlasEdge> classificationEdges = getClassificationEdges(instanceVertex); - - for (AtlasEdge edge : classificationEdges) { - deleteEdgeReference(edge, TypeCategory.CLASSIFICATION, false, false, instanceVertex); - } - - //remove traitNames and propagatedTraitNames property from instanceVertex - instanceVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY); - instanceVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY); - } - protected AtlasAttribute getAttributeForEdge(String edgeLabel) throws AtlasBaseException { AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edgeLabel); AtlasType parentType = typeRegistry.getType(atlasEdgeLabel.getTypeName()); @@ -721,7 +697,7 @@ public abstract class DeleteHandlerV1 { } if (edge != null) { - deleteEdge(edge, false); + deleteEdge(edge, isInternalType(inVertex) || isInternalType(outVertex)); RequestContextV1 requestContext = RequestContextV1.get(); @@ -761,4 +737,40 @@ public abstract class DeleteHandlerV1 { _deleteVertex(classificationVertex, force); } + + private boolean isInternalType(final AtlasVertex instanceVertex) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(GraphHelper.getTypeName(instanceVertex)); + return Objects.nonNull(entityType) && entityType.isInternalType(); + } + + private void removeFromPropagatedTraitNames(AtlasVertex entityVertex, String classificationName) { + if (entityVertex != null && StringUtils.isNotEmpty(classificationName)) { + List<String> propagatedTraitNames = getTraitNames(entityVertex, true); + + propagatedTraitNames.remove(classificationName); + + entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY); + + for (String propagatedTraitName : propagatedTraitNames) { + addToPropagatedTraitNames(entityVertex, propagatedTraitName); + } + } + } + + /** + * Delete all associated classifications from the specified entity vertex. + * @param instanceVertex + * @throws AtlasException + */ + private void deleteAllClassifications(AtlasVertex instanceVertex) throws AtlasBaseException { + List<AtlasEdge> classificationEdges = getClassificationEdges(instanceVertex); + + for (AtlasEdge edge : classificationEdges) { + deleteEdgeReference(edge, TypeCategory.CLASSIFICATION, false, false, instanceVertex); + } + + //remove traitNames and propagatedTraitNames property from instanceVertex + instanceVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY); + instanceVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index 23ffdcd..7d5b7fb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -683,6 +683,13 @@ public final class EntityGraphRetriever { String edgeLabel = EDGE_LABEL_PREFIX + propertyName; for (Object element : arrayElements) { + // When internal types are deleted, sometimes the collection type attribute will contain a null value + // Graph layer does erroneous mapping of the null element, hence avoiding the processing of the null element + if (element == null) { + LOG.debug("Skipping null arrayElement"); + continue; + } + Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel, entityExtInfo, isOwnedAttribute, edgeDirection); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java index 29518c4..edf1eed 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java @@ -20,6 +20,7 @@ package org.apache.atlas.repository.store.graph.v1; import org.apache.atlas.annotation.ConditionalOnAtlasProperty; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasTypeRegistry; @@ -38,11 +39,19 @@ public class HardDeleteHandlerV1 extends DeleteHandlerV1 { @Override protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> SoftDeleteHandlerV1._deleteVertex({}, {})", GraphHelper.string(instanceVertex), force); + } + graphHelper.removeVertex(instanceVertex); } @Override protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> HardDeleteHandlerV1.deleteEdge({}, {})", GraphHelper.string(edge), force); + } + graphHelper.removeEdge(edge); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java index c2f7c71..83c6e07 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java @@ -45,6 +45,10 @@ public class SoftDeleteHandlerV1 extends DeleteHandlerV1 { @Override protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> SoftDeleteHandlerV1._deleteVertex({}, {})", GraphHelper.string(instanceVertex), force); + } + if (force) { graphHelper.removeVertex(instanceVertex); } else { @@ -60,6 +64,10 @@ public class SoftDeleteHandlerV1 extends DeleteHandlerV1 { @Override protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> SoftDeleteHandlerV1.deleteEdge({}, {})",GraphHelper.string(edge), force); + } + if (force) { graphHelper.removeEdge(edge); } else { diff --git a/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java b/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java index 6deef80..b8b65d7 100644 --- a/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java @@ -332,7 +332,7 @@ public class GlossaryServiceTest { try { glossaryService.getGlossary(bankGlossary.getGuid()); } catch (AtlasBaseException e) { - assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INSTANCE_GUID_DELETED); + assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INSTANCE_GUID_NOT_FOUND); } } catch (AtlasBaseException e) { fail("Glossary delete should've succeeded", e);