ATLAS-3327-2 Create internal vertex property for classification names and…

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -180,6 +180,7 @@ public final class Constants {
     public static final String CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isPropagated");
     public static final String CLASSIFICATION_EDGE_STATE_PROPERTY_KEY         = STATE_PROPERTY_KEY;
     public static final String CLASSIFICATION_LABEL                           = "classifiedAs";
+    public static final String CLASSIFICATION_NAME_DELIMITER                  = "|";
     public static final String TERM_ASSIGNMENT_LABEL                          = "r:AtlasGlossarySemanticAssignment";
     public static final String ATTRIBUTE_INDEX_PROPERTY_KEY                   = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "index");
     public static final String ATTRIBUTE_KEY_PROPERTY_KEY                     = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "key");

repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java

@@ -319,6 +319,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
             createCommonVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
             createCommonVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
             createCommonVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
+            createCommonVertexIndex(management, CLASSIFICATION_NAMES_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
+            createCommonVertexIndex(management, PROPAGATED_CLASSIFICATION_NAMES_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
             createCommonVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
             createCommonVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
 

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -85,6 +85,8 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
+
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_NAME_KEY;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY;
@@ -92,8 +94,6 @@ import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
 import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
-import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
-import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_BLOCKED_PROPAGATED_CLASSIFICATIONS_KEY;
 import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
 import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
@@ -659,18 +659,6 @@ public final class GraphHelper {
         return element.toString();
     }
 
-    public static void addToPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName, string(entityVertex));
-        }
-
-        entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
-
-        String propClsNames = entityVertex.getProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, String.class);
-        propClsNames        = StringUtils.isEmpty(propClsNames)? classificationName : propClsNames + "|" + classificationName;
-        entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
-    }
-
     /**
      * Remove the specified edge from the graph.
      *
@@ -1849,4 +1837,14 @@ public final class GraphHelper {
     public static boolean getDefaultRemovePropagations() {
         return removePropagations;
     }
+
+    public static String getDelimitedClassificationNames(Collection<String> classificationNames) {
+        String ret = null;
+
+        if (CollectionUtils.isNotEmpty(classificationNames)) {
+            ret = CLASSIFICATION_NAME_DELIMITER + StringUtils.join(classificationNames, CLASSIFICATION_NAME_DELIMITER)
+                + CLASSIFICATION_NAME_DELIMITER;
+        }
+        return ret;
+    }
 }
\ No newline at end of file

repository/src/main/java/org/apache/atlas/repository/patches/ClassificationTextPatch.java

@@ -71,8 +71,7 @@ public class ClassificationTextPatch extends AtlasPatchHandler {
                 LOG.debug("processItem(typeName={}, vertexId={})", typeName, vertexId);
             }
 
-            getEntityGraphMapper().updateClassificationText(vertex);
-            getEntityGraphMapper().setClassificationNames(vertex);
+            getEntityGraphMapper().updateClassificationTextAndNames(vertex);
 
             if(LOG.isDebugEnabled()) {
                 LOG.debug("processItem(typeName={}, vertexId={}): Done!", typeName, vertexId);

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java

@@ -62,6 +62,7 @@ import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
 import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
 import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
@@ -450,7 +451,7 @@ public abstract class DeleteHandlerV1 {
 
                 graphHelper.addClassificationEdge(propagatedEntityVertex, classificationVertex, true);
 
-                addToPropagatedClassificationAndTraitNames(propagatedEntityVertex, classificationName);
+                addToPropagatedClassificationNames(propagatedEntityVertex, classificationName);
 
                 // record add propagation details to send notifications at the end
                 RequestContext      context        = RequestContext.get();
@@ -614,7 +615,7 @@ public abstract class DeleteHandlerV1 {
 
                         graphHelper.removeEdge(propagatedEdge);
 
-                        removeFromPropagatedClassificationAndTraitNames(impactedEntityVertex, classificationName);
+                        removeFromPropagatedClassificationNames(impactedEntityVertex, classificationName);
                     } else {
                         if (LOG.isDebugEnabled()) {
                             LOG.debug(" --> Not removing propagated classification edge from [{}] --> [{}][{}] using edge label: [{}], since edge doesn't exist",
@@ -666,7 +667,7 @@ public abstract class DeleteHandlerV1 {
                        getTypeName(entityVertex), GraphHelper.getGuid(entityVertex), CLASSIFICATION_LABEL);
         }
 
-        removeFromPropagatedClassificationAndTraitNames(entityVertex, classificationName);
+        removeFromPropagatedClassificationNames(entityVertex, classificationName);
 
         deleteEdge(edge, true);
 
@@ -987,18 +988,47 @@ public abstract class DeleteHandlerV1 {
         return Objects.nonNull(entityType) && entityType.isInternalType();
     }
 
-    private void removeFromPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
+    private void addToPropagatedClassificationNames(AtlasVertex entityVertex, String classificationName) {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName, string(entityVertex));
+        }
+        entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
+
+        String propClsNames = entityVertex.getProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, String.class);
+
+        propClsNames = StringUtils.isEmpty(propClsNames) ? CLASSIFICATION_NAME_DELIMITER + classificationName
+                                                         : (propClsNames + classificationName);
+
+        propClsNames = propClsNames + CLASSIFICATION_NAME_DELIMITER;
+        entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
+    }
+
+    private void removeFromPropagatedClassificationNames(AtlasVertex entityVertex, String classificationName) {
         if (entityVertex != null && StringUtils.isNotEmpty(classificationName)) {
             List<String> propagatedTraitNames = getTraitNames(entityVertex, true);
 
             propagatedTraitNames.remove(classificationName);
 
+            setPropagatedClassificationNames(entityVertex, propagatedTraitNames);
+        }
+    }
+
+    private void setPropagatedClassificationNames(AtlasVertex entityVertex, List<String> classificationNames) {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationNames, string(entityVertex));
+        }
+
         entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
         entityVertex.removeProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY);
 
-            for (String propagatedTraitName : propagatedTraitNames) {
-                addToPropagatedClassificationAndTraitNames(entityVertex, propagatedTraitName);
+        if (CollectionUtils.isNotEmpty(classificationNames)) {
+            for (String classificationName : classificationNames) {
+                entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
             }
+
+            String propClsName = CLASSIFICATION_NAME_DELIMITER + StringUtils.join(classificationNames, CLASSIFICATION_NAME_DELIMITER) + CLASSIFICATION_NAME_DELIMITER;
+
+            entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsName);
         }
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java

@@ -81,11 +81,11 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PA
 import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
 import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
 import static org.apache.atlas.repository.Constants.*;
-import static org.apache.atlas.repository.graph.GraphHelper.DEFAULT_REMOVE_PROPAGATIONS_ON_ENTITY_DELETE;
 import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
 import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
 import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
 import static org.apache.atlas.repository.graph.GraphHelper.getDefaultRemovePropagations;
+import static org.apache.atlas.repository.graph.GraphHelper.getDelimitedClassificationNames;
 import static org.apache.atlas.repository.graph.GraphHelper.getMapElementsProperty;
 import static org.apache.atlas.repository.graph.GraphHelper.getStatus;
 import static org.apache.atlas.repository.graph.GraphHelper.getTraitLabel;
@@ -1512,7 +1512,7 @@ public class EntityGraphMapper {
                     LOG.debug("Adding classification [{}] to [{}] using edge label: [{}]", classificationName, entityType.getTypeName(), getTraitLabel(classificationName));
                 }
 
-                addClassificationAndTraitNames(entityVertex, classificationName);
+                addToClassificationNames(entityVertex, classificationName);
 
                 // add a new AtlasVertex for the struct or trait instance
                 AtlasVertex classificationVertex = createClassificationVertex(classification);
@@ -1676,20 +1676,19 @@ public class EntityGraphMapper {
 
         traitNames.remove(classificationName);
 
-        updateClassificationAndTraitNames(entityVertex, traitNames);
+        setClassificationNames(entityVertex, traitNames);
 
         updateModificationMetadata(entityVertex);
 
         for (Map.Entry<AtlasVertex, List<AtlasClassification>> entry : removedClassifications.entrySet()) {
             AtlasEntity entity = updateClassificationText(entry.getKey());
 
-
             List<AtlasClassification> deletedClassificationNames = entry.getValue();
             entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassificationNames);
         }
     }
 
-    public AtlasEntity updateClassificationText(AtlasVertex vertex) throws AtlasBaseException {
+    private AtlasEntity updateClassificationText(AtlasVertex vertex) throws AtlasBaseException {
         String guid        = GraphHelper.getGuid(vertex);
         AtlasEntity entity = instanceConverter.getAndCacheEntity(guid);
 
@@ -1697,26 +1696,47 @@ public class EntityGraphMapper {
         return entity;
     }
 
-    public void setClassificationNames(AtlasVertex vertex) {
-        List<String> clsNamesList     = getTraitNames(vertex, false);
-        List<String> propClsNamesList = getTraitNames(vertex, true);
-        String clsNames               = StringUtils.join(clsNamesList, CLASSIFICATION_NAME_DELIMITER);
-        String propClsNames           = StringUtils.join(propClsNamesList, CLASSIFICATION_NAME_DELIMITER);
+    public void updateClassificationTextAndNames(AtlasVertex vertex) throws AtlasBaseException {
+        String        guid         = GraphHelper.getGuid(vertex);
+        AtlasEntity   entity       = instanceConverter.getAndCacheEntity(guid);
+
+        if (CollectionUtils.isEmpty(entity.getClassifications())) return;
+
+        List<String> classificationNames           = new ArrayList<>();
+        List<String> propagatedClassificationNames = new ArrayList<>();
 
-        vertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
-        vertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
+        for (AtlasClassification classification : entity.getClassifications()) {
+            if (isPropagatedClassification(classification, guid)) {
+                propagatedClassificationNames.add(classification.getTypeName());
+            } else {
+                classificationNames.add(classification.getTypeName());
+            }
         }
 
-    private void addClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
+        vertex.setProperty(CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(classificationNames));
+        vertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(propagatedClassificationNames));
+        vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
+    }
 
+    private boolean isPropagatedClassification(AtlasClassification classification, String guid) {
+        String classificationEntityGuid = classification.getEntityGuid();
+
+        return StringUtils.isNotEmpty(classificationEntityGuid) && !StringUtils.equals(classificationEntityGuid, guid);
+    }
+
+    private void addToClassificationNames(AtlasVertex entityVertex, String classificationName) {
         AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
 
         String clsNames = entityVertex.getProperty(CLASSIFICATION_NAMES_KEY, String.class);
-        clsNames = StringUtils.isEmpty(clsNames) ? classificationName : clsNames + CLASSIFICATION_NAME_DELIMITER + classificationName;
+
+        clsNames = StringUtils.isEmpty(clsNames) ? CLASSIFICATION_NAME_DELIMITER + classificationName : clsNames + classificationName;
+
+        clsNames = clsNames + CLASSIFICATION_NAME_DELIMITER;
+
         entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
     }
 
-    private void updateClassificationAndTraitNames(AtlasVertex entityVertex, List<String> traitNames) {
+    private void setClassificationNames(AtlasVertex entityVertex, List<String> traitNames) {
         if (entityVertex != null) {
             entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
             entityVertex.removeProperty(CLASSIFICATION_NAMES_KEY);
@@ -1725,7 +1745,11 @@ public class EntityGraphMapper {
                 AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
             }
 
-            entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, StringUtils.join(traitNames, CLASSIFICATION_NAME_DELIMITER));
+            String clsNames = StringUtils.join(traitNames, CLASSIFICATION_NAME_DELIMITER);
+
+            clsNames = StringUtils.isEmpty(clsNames) ? clsNames : CLASSIFICATION_NAME_DELIMITER + clsNames + CLASSIFICATION_NAME_DELIMITER;
+
+            entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
         }
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java

@@ -34,6 +34,7 @@ import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
@@ -48,6 +49,7 @@ import java.util.List;
 import java.util.Map;
 
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
+import static org.apache.atlas.repository.graph.GraphHelper.getDelimitedClassificationNames;
 
 @Component
 public final class EntityStateChecker {
@@ -266,10 +268,13 @@ public final class EntityStateChecker {
                     }
 
                     entityVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
+                    entityVertex.removeProperty(Constants.CLASSIFICATION_NAMES_KEY);
 
                     for (String classificationName : traitVertexNames) {
                         AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, classificationName);
                     }
+
+                    entityVertex.setProperty(Constants.CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(traitVertexNames));
                 }
 
                 if (propagatedTraitNamesToAdd != null || propagatedTraitNamesToRemove != null) {
@@ -282,10 +287,13 @@ public final class EntityStateChecker {
                     }
 
                     entityVertex.removeProperty(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
+                    entityVertex.removeProperty(Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY);
 
                     for (String classificationName : propagatedTraitVertexNames) {
                         AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
                     }
+
+                    entityVertex.setProperty(Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY,getDelimitedClassificationNames(propagatedTraitVertexNames));
                 }
 
                 AtlasGraphUtilsV2.setEncodedProperty(entityVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());

webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java

@@ -132,10 +132,11 @@ public class TestEntityREST {
 
         // For ATLAS-3327 to test internal properties are added properly.
         AtlasVertex vertex         = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
+        String expectedClsName     = Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.CLASSIFICATION + Constants.CLASSIFICATION_NAME_DELIMITER ;
         String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
 
         Assert.assertNotNull(classificationNames);
-        Assert.assertEquals(classificationNames, TestUtilsV2.CLASSIFICATION);
+        Assert.assertEquals(classificationNames, expectedClsName);
     }
 
     @Test(dependsOnMethods = "testGetEntityById")
@@ -167,7 +168,8 @@ public class TestEntityREST {
         }
 
         // For ATLAS-3327 to test internal properties are added properly.
-        String expectedClsNames = TestUtilsV2.CLASSIFICATION + "|" + TestUtilsV2.PHI;
+        String expectedClsNames    = Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.CLASSIFICATION +
+                                     Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.PHI + Constants.CLASSIFICATION_NAME_DELIMITER;
         AtlasVertex vertex         = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
         String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
 
@@ -204,7 +206,7 @@ public class TestEntityREST {
 
         deleteClassification(dbEntity.getGuid(), TestUtilsV2.PHI);
 
-        expectedClsNames = TestUtilsV2.CLASSIFICATION;
+        expectedClsNames    = Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.CLASSIFICATION + Constants.CLASSIFICATION_NAME_DELIMITER;
         vertex              = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
         classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
 
@@ -257,7 +259,7 @@ public class TestEntityREST {
 
         Assert.assertEquals(classifications, retrievedClassificationsList);
 
-        String expectedClsNames    = TestUtilsV2.CLASSIFICATION;
+        String expectedClsNames    = Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.CLASSIFICATION + Constants.CLASSIFICATION_NAME_DELIMITER;
         AtlasVertex vertex         = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
         String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);