ATLAS-3112: Allow Indexing of array attributes (LIST or SET) in indexing store

graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java

@@ -155,6 +155,7 @@ public interface AtlasGraphManagement {
      *
      * @param vertexIndex
      * @param propertyKey
+     * @param propertyClass
      */
-    void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
+    void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey, Class propertyClass);
 }

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java

@@ -195,12 +195,16 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
     }
 
     @Override
-    public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
+    public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey, Class propertyClass) {
         PropertyKey     janusKey    = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
         JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
 
+        if (propertyClass == String.class) {
+            management.addIndexKey(vertexIndex, janusKey, Mapping.STRING.asParameter());
+        } else {
             management.addIndexKey(vertexIndex, janusKey);
         }
+    }
 
     @Override
     public AtlasGraphIndex getGraphIndex(String indexName) {

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java

@@ -99,7 +99,7 @@ public abstract class AbstractGraphDatabaseTest {
         AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
         try {
             if (propertyClass != Integer.class) {
-                management.addMixedIndex(BACKING_INDEX_NAME, key);
+                management.addMixedIndex(BACKING_INDEX_NAME, key, propertyClass);
             }
         } catch(Throwable t) {
             //ok

repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java

@@ -271,12 +271,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
             createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, UniqueKind.NONE, Long.class, SINGLE, true, false);
 
             createVertexIndex(management, ENTITY_TYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
-            createVertexIndex(management, SUPER_TYPES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false);
             createVertexIndex(management, TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
             createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
             createVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
             createVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
             createVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
+
+            createVertexIndex(management, SUPER_TYPES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false);
             createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
             createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
 
@@ -351,10 +352,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
                 createLabelIfNeeded(management, propertyName, attribTypeName);
 
                 AtlasArrayType arrayType   = (AtlasArrayType) attributeType;
-                boolean        isReference = isReference(arrayType.getElementType());
+                AtlasType      elementType = arrayType.getElementType();
+                boolean        isReference = isReference(elementType);
 
                 if (!isReference) {
-                    createPropertyKey(management, propertyName, ArrayList.class, SINGLE);
+                    createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(elementType.getTypeName()), cardinality, isIndexable, false);
                 }
             }
 
@@ -508,12 +510,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
             if (propertyKey == null) {
                 propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
 
-                if (isIndexApplicable(propertyClass, cardinality)) {
+                if (isIndexApplicable(propertyClass)) {
                     if (LOG.isDebugEnabled()) {
                         LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
                     }
 
-                    management.addMixedIndex(VERTEX_INDEX, propertyKey);
+                    management.addMixedIndex(VERTEX_INDEX, propertyKey, propertyClass);
 
                     LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
                 }
@@ -591,12 +593,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
             if (propertyKey == null) {
                 propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
 
-                if (isIndexApplicable(propertyClass, cardinality)) {
+                if (isIndexApplicable(propertyClass)) {
                     if (LOG.isDebugEnabled()) {
                         LOG.debug("Creating backing index for edge property {} of type {} ", propertyName, propertyClass.getName());
                     }
 
-                    management.addMixedIndex(EDGE_INDEX, propertyKey);
+                    management.addMixedIndex(EDGE_INDEX, propertyKey, propertyClass);
 
                     LOG.info("Created backing index for edge property {} of type {} ", propertyName, propertyClass.getName());
                 }
@@ -619,12 +621,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
         if (propertyKey == null) {
             propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
 
-            if (isIndexApplicable(propertyClass, cardinality)) {
+            if (isIndexApplicable(propertyClass)) {
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
                 }
 
-                management.addMixedIndex(FULLTEXT_INDEX, propertyKey);
+                management.addMixedIndex(FULLTEXT_INDEX, propertyKey, propertyClass);
 
                 LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
             }
@@ -700,8 +702,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
         }
     }
 
-    private boolean isIndexApplicable(Class propertyClass, AtlasCardinality cardinality) {
-        return !(INDEX_EXCLUSION_CLASSES.contains(propertyClass) || cardinality.isMany());
+    private boolean isIndexApplicable(Class propertyClass) {
+        return !INDEX_EXCLUSION_CLASSES.contains(propertyClass);
     }
     
     public void commit(AtlasGraphManagement management) throws IndexException {

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -199,7 +199,7 @@ public final class GraphHelper {
         AtlasGraphUtilsV2.setEncodedProperty(ret, MODIFIED_BY_KEY, RequestContext.get().getUser());
 
         for (String superTypeName : superTypeNames) {
-            AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
+            AtlasGraphUtilsV2.addToEncodedSetProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
         }
 
         return ret;
@@ -1545,7 +1545,7 @@ public final class GraphHelper {
         if (isReference(elementType)) {
             return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute);
         } else {
-            return (List) instanceVertex.getListProperty(propertyName);
+            return (List) instanceVertex.getPropertyValues(propertyName, List.class);
         }
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java

@@ -32,6 +32,7 @@ import org.apache.atlas.model.patches.AtlasPatch;
 import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches;
 import org.apache.atlas.model.patches.AtlasPatch.PatchStatus;
 import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graph.AtlasGraphProvider;
 import org.apache.atlas.repository.graph.GraphHelper;
@@ -64,6 +65,8 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.LIST;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
 import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
 import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
@@ -187,15 +190,23 @@ public class AtlasGraphUtilsV2 {
      * @param propertyName
      * @param value
      */
-    public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Object value) {
-        return addProperty(vertex, propertyName, value, false);
+    public static AtlasVertex addToListProperty(AtlasVertex vertex, String propertyName, Object value) {
+        return addProperty(vertex, propertyName, LIST, false, value);
     }
 
-    public static AtlasVertex addEncodedProperty(AtlasVertex vertex, String propertyName, Object value) {
-        return addProperty(vertex, propertyName, value, true);
+    public static AtlasVertex addToSetProperty(AtlasVertex vertex, String propertyName, Object value) {
+        return addProperty(vertex, propertyName, SET, false, value);
     }
 
-    public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Object value, boolean isEncoded) {
+    public static AtlasVertex addToEncodedListProperty(AtlasVertex vertex, String propertyName, Object value) {
+        return addProperty(vertex, propertyName, LIST, true, value);
+    }
+
+    public static AtlasVertex addToEncodedSetProperty(AtlasVertex vertex, String propertyName, Object value) {
+        return addProperty(vertex, propertyName, SET, true, value);
+    }
+
+    public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Cardinality cardinality, boolean isEncoded, Object value) {
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> addProperty({}, {}, {})", toString(vertex), propertyName, value);
         }
@@ -204,7 +215,11 @@ public class AtlasGraphUtilsV2 {
             propertyName = encodePropertyKey(propertyName);
         }
 
+        if (cardinality == LIST) {
+            vertex.addListProperty(propertyName, value);
+        } else {
             vertex.addProperty(propertyName, value);
+        }
 
         return vertex;
     }

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java

@@ -78,6 +78,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CR
 import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE;
 import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE;
 import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.LIST;
 import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
 import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
@@ -116,6 +117,8 @@ import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled
 import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
 import static org.apache.atlas.repository.graph.GraphHelper.string;
 import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata;
+import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.addToListProperty;
+import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.addToSetProperty;
 import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex;
 import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isReference;
 import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
@@ -165,7 +168,7 @@ public class EntityGraphMapper {
         AtlasVertex ret = createStructVertex(entity);
 
         for (String superTypeName : entityType.getAllSuperTypes()) {
-            AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
+            AtlasGraphUtilsV2.addToEncodedSetProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
         }
 
         AtlasGraphUtilsV2.setEncodedProperty(ret, GUID_PROPERTY_KEY, guid);
@@ -310,7 +313,10 @@ public class EntityGraphMapper {
 
         AtlasVertex ret = createStructVertex(classification);
 
-        AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, classificationType.getAllSuperTypes());
+        for (String superTypeName : classificationType.getAllSuperTypes()) {
+            AtlasGraphUtilsV2.addToEncodedSetProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
+        }
+
         AtlasGraphUtilsV2.setEncodedProperty(ret, CLASSIFICATION_ENTITY_GUID, classification.getEntityGuid());
         AtlasGraphUtilsV2.setEncodedProperty(ret, CLASSIFICATION_ENTITY_STATUS, classification.getEntityStatus().name());
 
@@ -1080,9 +1086,9 @@ public class EntityGraphMapper {
         }
 
         if (isNewElementsNull) {
-            setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), null);
+            setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), null, cardinality);
         } else {
-            setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated);
+            setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated, cardinality);
         }
 
         if (LOG.isDebugEnabled()) {
@@ -1392,7 +1398,7 @@ public class EntityGraphMapper {
             return (List)vertex.getListProperty(vertexPropertyName, AtlasEdge.class);
         }
         else {
-            return (List)vertex.getListProperty(vertexPropertyName);
+            return (List) vertex.getPropertyValues(vertexPropertyName, List.class);
         }
     }
 
@@ -1436,9 +1442,17 @@ public class EntityGraphMapper {
 
         return Collections.emptyList();
     }
-    private void setArrayElementsProperty(AtlasType elementType, boolean isSoftReference, AtlasVertex vertex, String vertexPropertyName, List<Object> values) {
+
+    private void setArrayElementsProperty(AtlasType elementType, boolean isSoftReference, AtlasVertex vertex, String propertyName, List<Object> values, Cardinality cardinality) {
         if (!isReference(elementType) || isSoftReference) {
-            AtlasGraphUtilsV2.setEncodedProperty(vertex, vertexPropertyName, values);
+            //remove existing array values before setting new values
+            vertex.removeProperty(propertyName);
+
+            if (cardinality == LIST) {
+                values.forEach(value -> addToListProperty(vertex, propertyName, value));
+            } else {
+                values.forEach(value -> addToSetProperty(vertex, propertyName, value));
+            }
         }
     }
 
@@ -1523,7 +1537,7 @@ public class EntityGraphMapper {
                     LOG.debug("Adding classification [{}] to [{}] using edge label: [{}]", classificationName, entityType.getTypeName(), getTraitLabel(classificationName));
                 }
 
-                AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
+                AtlasGraphUtilsV2.addToEncodedSetProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
 
                 // add a new AtlasVertex for the struct or trait instance
                 AtlasVertex classificationVertex = createClassificationVertex(classification);
@@ -1924,7 +1938,7 @@ public class EntityGraphMapper {
             entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
 
             for (String traitName : traitNames) {
-                AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
+                AtlasGraphUtilsV2.addToEncodedSetProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
             }
         }
     }

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java

@@ -268,7 +268,7 @@ public final class EntityStateChecker {
                     entityVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
 
                     for (String classificationName : traitVertexNames) {
-                        AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, classificationName);
+                        AtlasGraphUtilsV2.addToEncodedSetProperty(entityVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, classificationName);
                     }
                 }
 
@@ -284,7 +284,7 @@ public final class EntityStateChecker {
                     entityVertex.removeProperty(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
 
                     for (String classificationName : propagatedTraitVertexNames) {
-                        AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
+                        AtlasGraphUtilsV2.addToEncodedListProperty(entityVertex, Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
                     }
                 }