ATLAS-2582: Hard delete of internal types

intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java

@@ -50,6 +50,8 @@ public class AtlasEntityType extends AtlasStructType {
     private final AtlasEntityDef entityDef;
     private final String         typeQryStr;
 
+    private static final String INTERNAL_TYPENAME = "__internal";
+
     private List<AtlasEntityType>                    superTypes                 = Collections.emptyList();
     private Set<String>                              allSuperTypes              = Collections.emptySet();
     private Set<String>                              subTypes                   = Collections.emptySet();
@@ -59,6 +61,7 @@ public class AtlasEntityType extends AtlasStructType {
     private Map<String, AtlasAttribute>              relationshipAttributes     = Collections.emptyMap();
     private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap();
     private String                                   typeAndAllSubTypesQryStr   = "";
+    private boolean                                  isInternalType             = false;
 
 
     public AtlasEntityType(AtlasEntityDef entityDef) {
@@ -145,6 +148,10 @@ public class AtlasEntityType extends AtlasStructType {
         }
 
         for (String superTypeName : allSuperTypes) {
+            if (INTERNAL_TYPENAME.equals(superTypeName)) {
+                isInternalType = true;
+            }
+
             AtlasEntityType superType = typeRegistry.getEntityTypeByName(superTypeName);
 
             Map<String, AtlasAttribute> superTypeRelationshipAttributes = superType.getRelationshipAttributes();
@@ -206,6 +213,10 @@ public class AtlasEntityType extends AtlasStructType {
         return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName);
     }
 
+    public boolean isInternalType() {
+        return isInternalType;
+    }
+
     public Map<String, AtlasAttribute> getRelationshipAttributes() { return relationshipAttributes; }
 
     public AtlasAttribute getRelationshipAttribute(String attributeName) { return relationshipAttributes.get(attributeName); }

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -647,8 +647,8 @@ public final class GraphHelper {
     public static <T> T getSingleValuedProperty(AtlasElement element, String propertyName, Class<T> clazz) {
         String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
 
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("Reading property {} from {}", actualPropertyName, string(element));
+        if (LOG.isTraceEnabled()) {
+            LOG.trace("Reading property {} from {}", actualPropertyName, string(element));
         }
 
         return element.getProperty(actualPropertyName, clazz);
@@ -658,8 +658,8 @@ public final class GraphHelper {
     public static Object getProperty(AtlasVertex<?,?> vertex, String propertyName) {
         String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
 
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("Reading property {} from {}", actualPropertyName, string(vertex));
+        if (LOG.isTraceEnabled()) {
+            LOG.trace("Reading property {} from {}", actualPropertyName, string(vertex));
         }
 
         if(AtlasGraphProvider.getGraphInstance().isMultiProperty(actualPropertyName)) {
@@ -673,8 +673,8 @@ public final class GraphHelper {
     public static Object getProperty(AtlasEdge<?,?> edge, String propertyName) {
         String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
 
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("Reading property {} from {}", actualPropertyName, string(edge));
+        if (LOG.isTraceEnabled()) {
+            LOG.trace("Reading property {} from {}", actualPropertyName, string(edge));
         }
 
         return edge.getProperty(actualPropertyName, Object.class);
@@ -736,18 +736,14 @@ public final class GraphHelper {
      * @param edge
      */
     public void removeEdge(AtlasEdge edge) {
-        String edgeString = null;
-
         if (LOG.isDebugEnabled()) {
-            edgeString = string(edge);
-
-            LOG.debug("Removing {}", edgeString);
+            LOG.debug("==> removeEdge({})", string(edge));
         }
 
         graph.removeEdge(edge);
 
         if (LOG.isDebugEnabled()) {
-            LOG.info("Removed {}", edgeString);
+            LOG.info("<== removeEdge()");
         }
     }
 
@@ -757,18 +753,14 @@ public final class GraphHelper {
      * @param vertex
      */
     public void removeVertex(AtlasVertex vertex) {
-        String vertexString = null;
-
         if (LOG.isDebugEnabled()) {
-            vertexString = string(vertex);
-
-            LOG.debug("Removing {}", vertexString);
+            LOG.debug("==> GraphHelper.removeVertex({})", string(vertex));
         }
 
         graph.removeVertex(vertex);
 
         if (LOG.isDebugEnabled()) {
-            LOG.info("Removed {}", vertexString);
+            LOG.debug("<== GraphHelper.removeVertex()");
         }
     }
 
@@ -1580,7 +1572,7 @@ public final class GraphHelper {
     }
 
     // newly added
-    public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
+    public static List<Object>  getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
         String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
         if(AtlasGraphUtilsV1.isReference(elementType)) {
             return (List)instanceVertex.getListProperty(encodedPropertyName, AtlasEdge.class);

repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AbstractGlossaryDTO.java

@@ -24,7 +24,6 @@ import org.apache.atlas.model.glossary.relations.AtlasRelatedCategoryHeader;
 import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader;
 import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader;
 import org.apache.atlas.model.instance.AtlasRelatedObjectId;
-import org.apache.atlas.model.instance.AtlasRelationship;
 import org.apache.atlas.model.instance.AtlasStruct;
 import org.apache.atlas.repository.ogm.AbstractDataTransferObject;
 import org.apache.atlas.type.AtlasTypeRegistry;
@@ -151,9 +150,7 @@ public abstract class AbstractGlossaryDTO<T extends AtlasBaseModelObject> extend
             ret = new HashSet<>();
             for (Object t : (Collection) relatedObjectIds) {
                 if (t instanceof AtlasRelatedObjectId) {
-                    if (((AtlasRelatedObjectId) t).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
-                        ret.add(constructRelatedTermId((AtlasRelatedObjectId) t));
-                    }
+                    ret.add(constructRelatedTermId((AtlasRelatedObjectId) t));
                 }
             }
         }

repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryCategoryDTO.java

@@ -60,17 +60,13 @@ public class AtlasGlossaryCategoryDTO extends AbstractGlossaryDTO<AtlasGlossaryC
         Object anchor = entity.getRelationshipAttribute("anchor");
         if (anchor instanceof AtlasRelatedObjectId) {
             LOG.debug("Processing anchor");
-            if (((AtlasRelatedObjectId) anchor).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
-                ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor));
-            }
+            ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor));
         }
 
         Object parentCategory = entity.getRelationshipAttribute("parentCategory");
         if (parentCategory instanceof AtlasRelatedObjectId) {
             LOG.debug("Processing parentCategory");
-            if (((AtlasRelatedObjectId) parentCategory).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
-                ret.setParentCategory(constructRelatedCategoryId((AtlasRelatedObjectId) parentCategory));
-            }
+            ret.setParentCategory(constructRelatedCategoryId((AtlasRelatedObjectId) parentCategory));
         }
 
         Object childrenCategories = entity.getRelationshipAttribute("childrenCategories");
@@ -90,9 +86,7 @@ public class AtlasGlossaryCategoryDTO extends AbstractGlossaryDTO<AtlasGlossaryC
             LOG.debug("Processing terms");
             for (Object term : (Collection) terms) {
                 if (term instanceof AtlasRelatedObjectId) {
-                    if (((AtlasRelatedObjectId) term).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
-                        ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) term));
-                    }
+                    ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) term));
                 }
             }
         }

repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryDTO.java

@@ -21,7 +21,6 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.glossary.AtlasGlossary;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasRelatedObjectId;
-import org.apache.atlas.model.instance.AtlasRelationship;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -69,9 +68,7 @@ public class AtlasGlossaryDTO extends AbstractGlossaryDTO<AtlasGlossary> {
             if (categoriesAttr instanceof Collection) {
                 for (Object o : (Collection) categoriesAttr) {
                     if (o instanceof AtlasRelatedObjectId) {
-                        if (((AtlasRelatedObjectId) o).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
-                            ret.addCategory(constructRelatedCategoryId((AtlasRelatedObjectId) o));
-                        }
+                        ret.addCategory(constructRelatedCategoryId((AtlasRelatedObjectId) o));
                     }
                 }
             }
@@ -83,9 +80,7 @@ public class AtlasGlossaryDTO extends AbstractGlossaryDTO<AtlasGlossary> {
             if (termsAttr instanceof Collection) {
                 for (Object o : (Collection) termsAttr) {
                     if (o instanceof AtlasRelatedObjectId) {
-                        if (((AtlasRelatedObjectId) o).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
-                            ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) o));
-                        }
+                        ret.addTerm(constructRelatedTermId((AtlasRelatedObjectId) o));
                     }
                 }
             }

repository/src/main/java/org/apache/atlas/repository/ogm/glossary/AtlasGlossaryTermDTO.java

@@ -21,7 +21,6 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.glossary.AtlasGlossaryTerm;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasRelatedObjectId;
-import org.apache.atlas.model.instance.AtlasRelationship;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.commons.collections.CollectionUtils;
 import org.slf4j.Logger;
@@ -65,9 +64,7 @@ public class AtlasGlossaryTermDTO extends AbstractGlossaryDTO<AtlasGlossaryTerm>
         Object anchor = entity.getRelationshipAttribute("anchor");
         if (anchor instanceof AtlasRelatedObjectId) {
             LOG.debug("Processing anchor");
-            if (((AtlasRelatedObjectId) anchor).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
-                ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor));
-            }
+            ret.setAnchor(constructGlossaryId((AtlasRelatedObjectId) anchor));
         }
 
         Object categories = entity.getRelationshipAttribute("categories");
@@ -75,9 +72,7 @@ public class AtlasGlossaryTermDTO extends AbstractGlossaryDTO<AtlasGlossaryTerm>
             LOG.debug("Processing categories");
             for (Object category : (Collection) categories) {
                 if (category instanceof AtlasRelatedObjectId) {
-                    if (((AtlasRelatedObjectId) category).getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
-                        ret.addCategory(constructTermCategorizationId((AtlasRelatedObjectId) category));
-                    }
+                    ret.addCategory(constructTermCategorizationId((AtlasRelatedObjectId) category));
                 }
             }
         }
@@ -90,9 +85,7 @@ public class AtlasGlossaryTermDTO extends AbstractGlossaryDTO<AtlasGlossaryTerm>
             for (Object assignedEntity : (Collection) assignedEntities) {
                 if (assignedEntity instanceof AtlasRelatedObjectId) {
                     AtlasRelatedObjectId id = (AtlasRelatedObjectId) assignedEntity;
-                    if (id.getRelationshipStatus() == AtlasRelationship.Status.ACTIVE) {
-                        ret.addAssignedEntity(id);
-                    }
+                    ret.addAssignedEntity(id);
                 }
             }
         }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java

@@ -30,6 +30,8 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.EntityMutationResponse;
 import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.v1.model.instance.Referenceable;
 import org.apache.atlas.v1.model.instance.Struct;
 import org.apache.atlas.repository.Constants;
@@ -48,6 +50,7 @@ import javax.inject.Inject;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 
 import static org.apache.atlas.util.AtlasRepositoryConfiguration.isV2EntityNotificationEnabled;
@@ -61,15 +64,20 @@ public class AtlasEntityChangeNotifier {
     private final Set<EntityChangeListenerV2> entityChangeListenersV2;
     private final AtlasInstanceConverter      instanceConverter;
     private final FullTextMapperV2            fullTextMapperV2;
+    private final AtlasTypeRegistry           atlasTypeRegistry;
 
 
     @Inject
-    public AtlasEntityChangeNotifier(Set<EntityChangeListener> entityChangeListeners, Set<EntityChangeListenerV2> entityChangeListenersV2,
-                                     AtlasInstanceConverter instanceConverter, final FullTextMapperV2 fullTextMapperV2) {
+    public AtlasEntityChangeNotifier(Set<EntityChangeListener> entityChangeListeners,
+                                     Set<EntityChangeListenerV2> entityChangeListenersV2,
+                                     AtlasInstanceConverter instanceConverter,
+                                     FullTextMapperV2 fullTextMapperV2,
+                                     AtlasTypeRegistry atlasTypeRegistry) {
         this.entityChangeListeners   = entityChangeListeners;
         this.entityChangeListenersV2 = entityChangeListenersV2;
         this.instanceConverter       = instanceConverter;
         this.fullTextMapperV2 = fullTextMapperV2;
+        this.atlasTypeRegistry = atlasTypeRegistry;
     }
 
     public void onEntitiesMutated(EntityMutationResponse entityMutationResponse, boolean isImport) throws AtlasBaseException {
@@ -282,6 +290,17 @@ public class AtlasEntityChangeNotifier {
         if (CollectionUtils.isNotEmpty(entityHeaders)) {
             for (AtlasEntityHeader entityHeader : entityHeaders) {
                 String                 entityGuid        = entityHeader.getGuid();
+                String                 typeName          = entityHeader.getTypeName();
+
+                // Skip all internal types as the HARD DELETE will cause lookup errors
+                AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName(typeName);
+                if (Objects.nonNull(entityType) && entityType.isInternalType()) {
+                    if (LOG.isDebugEnabled()) {
+                        LOG.debug("Skipping internal type = {}", typeName);
+                    }
+                    continue;
+                }
+
                 AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid);
 
                 if (entityWithExtInfo != null) {

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java

@@ -683,6 +683,13 @@ public final class EntityGraphRetriever {
         String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
 
         for (Object element : arrayElements) {
+            // When internal types are deleted, sometimes the collection type attribute will contain a null value
+            // Graph layer does erroneous mapping of the null element, hence avoiding the processing of the null element
+            if (element == null) {
+                LOG.debug("Skipping null arrayElement");
+                continue;
+            }
+
             Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel,
                                                          entityExtInfo, isOwnedAttribute, edgeDirection);
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java

@@ -20,6 +20,7 @@ package org.apache.atlas.repository.store.graph.v1;
 
 import org.apache.atlas.annotation.ConditionalOnAtlasProperty;
 import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graph.GraphHelper;
 import org.apache.atlas.repository.graphdb.AtlasEdge;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.type.AtlasTypeRegistry;
@@ -38,11 +39,19 @@ public class HardDeleteHandlerV1 extends DeleteHandlerV1 {
 
     @Override
     protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> SoftDeleteHandlerV1._deleteVertex({}, {})", GraphHelper.string(instanceVertex), force);
+        }
+
         graphHelper.removeVertex(instanceVertex);
     }
 
     @Override
     protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseException {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> HardDeleteHandlerV1.deleteEdge({}, {})", GraphHelper.string(edge), force);
+        }
+
         graphHelper.removeEdge(edge);
     }
 }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java

@@ -45,6 +45,10 @@ public class SoftDeleteHandlerV1 extends DeleteHandlerV1 {
 
     @Override
     protected void _deleteVertex(AtlasVertex instanceVertex, boolean force) {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> SoftDeleteHandlerV1._deleteVertex({}, {})", GraphHelper.string(instanceVertex), force);
+        }
+
         if (force) {
             graphHelper.removeVertex(instanceVertex);
         } else {
@@ -60,6 +64,10 @@ public class SoftDeleteHandlerV1 extends DeleteHandlerV1 {
 
     @Override
     protected void deleteEdge(AtlasEdge edge, boolean force) throws AtlasBaseException {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> SoftDeleteHandlerV1.deleteEdge({}, {})",GraphHelper.string(edge), force);
+        }
+
         if (force) {
             graphHelper.removeEdge(edge);
         } else {

repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java

@@ -332,7 +332,7 @@ public class GlossaryServiceTest {
             try {
                 glossaryService.getGlossary(bankGlossary.getGuid());
             } catch (AtlasBaseException e) {
-                assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INSTANCE_GUID_DELETED);
+                assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INSTANCE_GUID_NOT_FOUND);
             }
         } catch (AtlasBaseException e) {
             fail("Glossary delete should've succeeded", e);