ATLAS-3327 Create internal vertex property for classification names and…

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -39,8 +39,8 @@ public final class Constants {
     public static final String GUID_PROPERTY_KEY                = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "guid");
     public static final String RELATIONSHIP_GUID_PROPERTY_KEY   = encodePropertyKey(RELATIONSHIP_PROPERTY_KEY_PREFIX + GUID_PROPERTY_KEY);
     public static final String HISTORICAL_GUID_PROPERTY_KEY     = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids");
-    public static final String FREETEXT_REQUEST_HANDLER = "/freetext";
-    public static final String TERMS_REQUEST_HANDLER = "/terms";
+    public static final String FREETEXT_REQUEST_HANDLER         = "/freetext";
+    public static final String TERMS_REQUEST_HANDLER            = "/terms";
 
     /**
      * Entity type name property key.
@@ -82,15 +82,17 @@ public final class Constants {
     /**
      * Trait names property key and index name.
      */
-    public static final String TRAIT_NAMES_PROPERTY_KEY            = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
-    public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
+    public static final String TRAIT_NAMES_PROPERTY_KEY             = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
+    public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY  = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
 
-    public static final String VERSION_PROPERTY_KEY          = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version");
-    public static final String STATE_PROPERTY_KEY            = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
-    public static final String CREATED_BY_KEY                = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
-    public static final String CLASSIFICATION_TEXT_KEY       = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
+    public static final String VERSION_PROPERTY_KEY                 = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version");
+    public static final String STATE_PROPERTY_KEY                   = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
+    public static final String CREATED_BY_KEY                       = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
+    public static final String CLASSIFICATION_TEXT_KEY              = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
+    public static final String CLASSIFICATION_NAMES_KEY             = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames");
+    public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY  = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames");
 
-    public static final String MODIFIED_BY_KEY      = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
+    public static final String MODIFIED_BY_KEY                      = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
 
     /**
      * Patch vertices property keys.

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -92,6 +92,7 @@ import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
 import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
 import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_BLOCKED_PROPAGATED_CLASSIFICATIONS_KEY;
 import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
@@ -658,12 +659,16 @@ public final class GraphHelper {
         return element.toString();
     }
 
-    public static void addToPropagatedTraitNames(AtlasVertex entityVertex, String classificationName) {
+    public static void addToPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
         if (LOG.isDebugEnabled()) {
             LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName, string(entityVertex));
         }
 
         entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
+
+        String propClsNames = entityVertex.getProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, String.class);
+        propClsNames        = StringUtils.isEmpty(propClsNames)? classificationName : propClsNames + "|" + classificationName;
+        entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
     }
 
     /**

repository/src/main/java/org/apache/atlas/repository/patches/ClassificationTextPatch.java

@@ -72,6 +72,7 @@ public class ClassificationTextPatch extends AtlasPatchHandler {
             }
 
             getEntityGraphMapper().updateClassificationText(vertex);
+            getEntityGraphMapper().setClassificationNames(vertex);
 
             if(LOG.isDebugEnabled()) {
                 LOG.debug("processItem(typeName={}, vertexId={}): Done!", typeName, vertexId);

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java

@@ -64,6 +64,7 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
 import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
 import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
 import static org.apache.atlas.repository.graph.GraphHelper.*;
@@ -449,7 +450,7 @@ public abstract class DeleteHandlerV1 {
 
                 graphHelper.addClassificationEdge(propagatedEntityVertex, classificationVertex, true);
 
-                addToPropagatedTraitNames(propagatedEntityVertex, classificationName);
+                addToPropagatedClassificationAndTraitNames(propagatedEntityVertex, classificationName);
 
                 // record add propagation details to send notifications at the end
                 RequestContext      context        = RequestContext.get();
@@ -613,7 +614,7 @@ public abstract class DeleteHandlerV1 {
 
                         graphHelper.removeEdge(propagatedEdge);
 
-                        removeFromPropagatedTraitNames(impactedEntityVertex, classificationName);
+                        removeFromPropagatedClassificationAndTraitNames(impactedEntityVertex, classificationName);
                     } else {
                         if (LOG.isDebugEnabled()) {
                             LOG.debug(" --> Not removing propagated classification edge from [{}] --> [{}][{}] using edge label: [{}], since edge doesn't exist",
@@ -665,7 +666,7 @@ public abstract class DeleteHandlerV1 {
                        getTypeName(entityVertex), GraphHelper.getGuid(entityVertex), CLASSIFICATION_LABEL);
         }
 
-        removeFromPropagatedTraitNames(entityVertex, classificationName);
+        removeFromPropagatedClassificationAndTraitNames(entityVertex, classificationName);
 
         deleteEdge(edge, true);
 
@@ -986,16 +987,17 @@ public abstract class DeleteHandlerV1 {
         return Objects.nonNull(entityType) && entityType.isInternalType();
     }
 
-    private void removeFromPropagatedTraitNames(AtlasVertex entityVertex, String classificationName) {
+    private void removeFromPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
         if (entityVertex != null && StringUtils.isNotEmpty(classificationName)) {
             List<String> propagatedTraitNames = getTraitNames(entityVertex, true);
 
             propagatedTraitNames.remove(classificationName);
 
             entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
+            entityVertex.removeProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY);
 
             for (String propagatedTraitName : propagatedTraitNames) {
-                addToPropagatedTraitNames(entityVertex, propagatedTraitName);
+                addToPropagatedClassificationAndTraitNames(entityVertex, propagatedTraitName);
             }
         }
     }

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java

@@ -81,6 +81,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PA
 import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
 import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
 import static org.apache.atlas.repository.Constants.*;
+import static org.apache.atlas.repository.graph.GraphHelper.DEFAULT_REMOVE_PROPAGATIONS_ON_ENTITY_DELETE;
 import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
 import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
 import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
@@ -105,8 +106,9 @@ import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelation
 public class EntityGraphMapper {
     private static final Logger LOG = LoggerFactory.getLogger(EntityGraphMapper.class);
 
-    private static final String SOFT_REF_FORMAT      = "%s:%s";
-    private static final int INDEXED_STR_SAFE_LEN = AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
+    private static final String SOFT_REF_FORMAT               = "%s:%s";
+    private static final int INDEXED_STR_SAFE_LEN             = AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
+    private static final String CLASSIFICATION_NAME_DELIMITER = "|";
 
     private final GraphHelper               graphHelper = GraphHelper.getInstance();
     private final AtlasGraph                graph;
@@ -116,7 +118,7 @@ public class EntityGraphMapper {
     private final AtlasEntityChangeNotifier entityChangeNotifier;
     private final AtlasInstanceConverter    instanceConverter;
     private final EntityGraphRetriever      entityRetriever;
-    private final FullTextMapperV2 fullTextMapperV2;
+    private final FullTextMapperV2          fullTextMapperV2;
 
     @Inject
     public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph,
@@ -1510,7 +1512,7 @@ public class EntityGraphMapper {
                     LOG.debug("Adding classification [{}] to [{}] using edge label: [{}]", classificationName, entityType.getTypeName(), getTraitLabel(classificationName));
                 }
 
-                AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
+                addClassificationAndTraitNames(entityVertex, classificationName);
 
                 // add a new AtlasVertex for the struct or trait instance
                 AtlasVertex classificationVertex = createClassificationVertex(classification);
@@ -1575,7 +1577,6 @@ public class EntityGraphMapper {
                 AtlasEntity               entity               = instanceConverter.getAndCacheEntity(entityGuid);
                 List<AtlasClassification> addedClassifications = StringUtils.equals(entityGuid, guid) ? addClassifications : propagations.get(vertex);
 
-
                 vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
                 if (CollectionUtils.isNotEmpty(addedClassifications)) {
                     entityChangeNotifier.onClassificationAddedToEntity(entity, addedClassifications);
@@ -1675,7 +1676,7 @@ public class EntityGraphMapper {
 
         traitNames.remove(classificationName);
 
-        updateTraitNamesProperty(entityVertex, traitNames);
+        updateClassificationAndTraitNames(entityVertex, traitNames);
 
         updateModificationMetadata(entityVertex);
 
@@ -1696,6 +1697,38 @@ public class EntityGraphMapper {
         return entity;
     }
 
+    public void setClassificationNames(AtlasVertex vertex) {
+        List<String> clsNamesList     = getTraitNames(vertex, false);
+        List<String> propClsNamesList = getTraitNames(vertex, true);
+        String clsNames               = StringUtils.join(clsNamesList, CLASSIFICATION_NAME_DELIMITER);
+        String propClsNames           = StringUtils.join(propClsNamesList, CLASSIFICATION_NAME_DELIMITER);
+
+        vertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
+        vertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
+    }
+
+    private void addClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
+
+        AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
+
+        String clsNames = entityVertex.getProperty(CLASSIFICATION_NAMES_KEY, String.class);
+        clsNames = StringUtils.isEmpty(clsNames) ? classificationName : clsNames + CLASSIFICATION_NAME_DELIMITER + classificationName;
+        entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
+    }
+
+    private void updateClassificationAndTraitNames(AtlasVertex entityVertex, List<String> traitNames) {
+        if (entityVertex != null) {
+            entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
+            entityVertex.removeProperty(CLASSIFICATION_NAMES_KEY);
+
+            for (String traitName : traitNames) {
+                AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
+            }
+
+            entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, StringUtils.join(traitNames, CLASSIFICATION_NAME_DELIMITER));
+        }
+    }
+
     public void updateClassifications(EntityMutationContext context, String guid, List<AtlasClassification> classifications) throws AtlasBaseException {
         if (CollectionUtils.isEmpty(classifications)) {
             throw new AtlasBaseException(AtlasErrorCode.INVALID_CLASSIFICATION_PARAMS, "update", guid);
@@ -1918,16 +1951,6 @@ public class EntityGraphMapper {
         }
     }
 
-    private void updateTraitNamesProperty(AtlasVertex entityVertex, List<String> traitNames) {
-        if (entityVertex != null) {
-            entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
-
-            for (String traitName : traitNames) {
-                AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
-            }
-        }
-    }
-
     private void validateClassificationExists(List<String> existingClassifications, List<String> suppliedClassifications) throws AtlasBaseException {
         Set<String> existingNames = new HashSet<>(existingClassifications);
         for (String classificationName : suppliedClassifications) {

webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java

@@ -30,6 +30,9 @@ import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.EntityMutationResponse;
 import org.apache.atlas.model.instance.EntityMutations;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
 import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.type.AtlasTypeUtil;
 import org.apache.atlas.web.rest.EntityREST;
@@ -62,7 +65,7 @@ public class TestEntityREST {
     private AtlasEntity dbEntity;
 
     private AtlasClassification testClassification;
-    
+
     private AtlasClassification phiClassification;
 
     @BeforeClass
@@ -108,7 +111,7 @@ public class TestEntityREST {
     }
 
     @Test(dependsOnMethods = "testGetEntityById")
-    public void  testAddAndGetClassification() throws Exception {
+    public void testAddAndGetClassification() throws Exception {
 
         List<AtlasClassification> classifications = new ArrayList<>();
         testClassification = new AtlasClassification(TestUtilsV2.CLASSIFICATION, new HashMap<String, Object>() {{ put("tag", "tagName"); }});
@@ -126,6 +129,13 @@ public class TestEntityREST {
 
         Assert.assertNotNull(retrievedClassification);
         Assert.assertEquals(retrievedClassification, testClassification);
+
+        // For ATLAS-3327 to test internal properties are added properly.
+        AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
+        String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
+
+        Assert.assertNotNull(classificationNames);
+        Assert.assertEquals(classificationNames, TestUtilsV2.CLASSIFICATION);
     }
 
     @Test(dependsOnMethods = "testGetEntityById")
@@ -156,6 +166,14 @@ public class TestEntityREST {
             Assert.assertEquals(retrievedClassification.getAttribute(attrName), phiClassification.getAttribute(attrName));
         }
 
+        // For ATLAS-3327 to test internal properties are added properly.
+        String expectedClsNames = TestUtilsV2.CLASSIFICATION + "|" + TestUtilsV2.PHI;
+        AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
+        String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
+
+        Assert.assertNotNull(classificationNames);
+        Assert.assertEquals(classificationNames, expectedClsNames);
+
         // update multiple tags attributes
         phiClassification = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
             put("stringAttr", "sample_string_v2");
@@ -178,11 +196,24 @@ public class TestEntityREST {
         Assert.assertNotNull(updatedClassification);
         Assert.assertEquals(updatedClassification.getAttribute("tag"), testClassification.getAttribute("tag"));
 
+        vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
+        classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
+
+        Assert.assertNotNull(classificationNames);
+        Assert.assertEquals(classificationNames, expectedClsNames);
+
         deleteClassification(dbEntity.getGuid(), TestUtilsV2.PHI);
+
+        expectedClsNames = TestUtilsV2.CLASSIFICATION;
+        vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
+        classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
+
+        Assert.assertNotNull(classificationNames);
+        Assert.assertEquals(classificationNames, expectedClsNames);
     }
 
     @Test(dependsOnMethods = "testAddAndGetClassification")
-    public void  testGetEntityWithAssociations() throws Exception {
+    public void testGetEntityWithAssociations() throws Exception {
 
         AtlasEntityWithExtInfo entity = entityREST.getById(dbEntity.getGuid(), false, false);
         final List<AtlasClassification> retrievedClassifications = entity.getEntity().getClassifications();
@@ -192,11 +223,18 @@ public class TestEntityREST {
     }
 
     @Test(dependsOnMethods = "testGetEntityWithAssociations")
-    public void  testDeleteClassification() throws Exception {
+    public void testDeleteClassification() throws Exception {
 
         deleteClassification(dbEntity.getGuid(), TestUtilsV2.CLASSIFICATION);
         final AtlasClassification.AtlasClassifications retrievedClassifications = entityREST.getClassifications(dbEntity.getGuid());
 
+        String expectedClsNames    = "";
+        AtlasVertex vertex         = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
+        String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
+
+        Assert.assertNotNull(classificationNames);
+        Assert.assertEquals(classificationNames, expectedClsNames);
+
         Assert.assertNotNull(retrievedClassifications);
         Assert.assertEquals(retrievedClassifications.getList().size(), 0);
     }
@@ -219,6 +257,13 @@ public class TestEntityREST {
 
         Assert.assertEquals(classifications, retrievedClassificationsList);
 
+        String expectedClsNames    = TestUtilsV2.CLASSIFICATION;
+        AtlasVertex vertex         = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
+        String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
+
+        Assert.assertNotNull(classificationNames);
+        Assert.assertEquals(classificationNames, expectedClsNames);
+
         final AtlasClassification retrievedClassification = entityREST.getClassification(dbEntity.getGuid(), TestUtilsV2.CLASSIFICATION);
 
         Assert.assertNotNull(retrievedClassification);