ATLAS-2662: Remove complex array and map attribute's edge information from entity vertex

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -126,6 +126,8 @@ public final class Constants {
     public static final String CLASSIFICATION_EDGE_STATE_PROPERTY_KEY         = STATE_PROPERTY_KEY;
     public static final String CLASSIFICATION_LABEL                           = "classifiedAs";
     public static final String TERM_ASSIGNMENT_LABEL                          = "r:AtlasGlossarySemanticAssignment";
+    public static final String ATTRIBUTE_INDEX_PROPERTY_KEY                   = INTERNAL_PROPERTY_KEY_PREFIX + "index";
+    public static final String ATTRIBUTE_KEY_PROPERTY_KEY                     = INTERNAL_PROPERTY_KEY_PREFIX + "key";
 
     public static final String VERTEX_ID_IN_IMPORT_KEY = "__vIdInImport";
     public static final String EDGE_ID_IN_IMPORT_KEY   = "__eIdInImport";

intg/src/main/java/org/apache/atlas/AtlasErrorCode.java

@@ -142,6 +142,7 @@ public enum AtlasErrorCode {
     RELATIONSHIP_END_IS_NULL(400, "ATLAS-400-00-07D", "Relationship end is invalid. Expected {0} but is NULL"),
     INVALID_TERM_RELATION_TO_SELF(400, "ATLAS-400-00-07E", "Invalid Term relationship: Term can't have a relationship with self"),
     INVALID_CHILD_CATEGORY_DIFFERENT_GLOSSARY(400, "ATLAS-400-00-07F", "Invalid child category relationship: Child category (guid = {0}) belongs to different glossary"),
+    ATTRIBUTE_TYPE_INVALID(400, "ATLAS-400-00-080", "{0}.{1}: invalid attribute type. Attribute cannot be of type classification"),
 
     UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
 

intg/src/main/java/org/apache/atlas/model/instance/AtlasObjectId.java

@@ -34,6 +34,8 @@ import javax.xml.bind.annotation.XmlSeeAlso;
 import org.apache.atlas.model.PList;
 import org.apache.atlas.model.SearchFilter.SortType;
 import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.commons.lang.StringUtils;
+
 import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
 import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
 
@@ -163,13 +165,12 @@ public class AtlasObjectId  implements Serializable {
 
         AtlasObjectId that = (AtlasObjectId) o;
 
-        // if guid is null, equality should be based on typeName/uniqueAttributes
-        if (guid != null && Objects.equals(guid, that.guid)) {
-            return true;
+        // if guid is empty/null, equality should be based on typeName/uniqueAttributes
+        if (StringUtils.isEmpty(guid) && StringUtils.isEmpty(that.guid)) {
+            return Objects.equals(typeName, that.typeName) && Objects.equals(uniqueAttributes, that.uniqueAttributes);
+        } else {
+            return Objects.equals(guid, that.guid);
         }
-
-        return Objects.equals(typeName, that.typeName) &&
-               Objects.equals(uniqueAttributes, that.uniqueAttributes);
     }
 
     @Override

intg/src/main/java/org/apache/atlas/type/AtlasStructType.java

@@ -98,6 +98,17 @@ public class AtlasStructType extends AtlasType {
                 arrayType.setMaxCount(attributeDef.getValuesMaxCount());
             }
 
+            //check if attribute type is not classification
+            if (attrType instanceof AtlasArrayType) {
+                attrType = ((AtlasArrayType) attrType).getElementType();
+            } else if (attrType instanceof AtlasMapType) {
+                attrType = ((AtlasMapType) attrType).getValueType();
+            }
+
+            if (attrType instanceof AtlasClassificationType) {
+                throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_TYPE_INVALID, getTypeName(), attributeDef.getName());
+            }
+
             a.put(attributeDef.getName(), attribute);
         }
 

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -26,12 +26,13 @@ import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.RequestContextV1;
 import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.instance.AtlasClassification;
 import org.apache.atlas.model.instance.AtlasEntity.Status;
 import org.apache.atlas.model.instance.AtlasObjectId;
 import org.apache.atlas.model.instance.AtlasRelationship;
 import org.apache.atlas.model.typedef.AtlasRelationshipDef;
 import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
+import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasMapType;
 import org.apache.atlas.util.AtlasGremlinQueryProvider;
 import org.apache.atlas.v1.model.instance.Id;
 import org.apache.atlas.v1.model.instance.Referenceable;
@@ -54,6 +55,7 @@ import org.apache.atlas.exception.EntityNotFoundException;
 import org.apache.atlas.util.AttributeValueMap;
 import org.apache.atlas.util.IndexedInstance;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.IteratorUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -75,6 +77,8 @@ import java.util.Set;
 import java.util.UUID;
 
 import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
@@ -82,6 +86,7 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PRO
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_NAME_KEY;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
 import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
 import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
 import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
 import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
@@ -1170,6 +1175,11 @@ public final class GraphHelper {
         return AtlasGraphUtilsV1.getProperty(classificationVertex, CLASSIFICATION_ENTITY_GUID, String.class);
     }
 
+    public static Integer getIndexValue(AtlasEdge edge) {
+        Integer index = edge.getProperty(ATTRIBUTE_INDEX_PROPERTY_KEY, Integer.class);
+        return (index == null) ? 0: index;
+    }
+
     public static boolean isPropagatedClassificationEdge(AtlasEdge edge) {
         boolean ret = false;
 
@@ -1557,44 +1567,100 @@ public final class GraphHelper {
         return typeName != null && typeName.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX);
     }
 
-    public static void setArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, List<Object> values) {
-        String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
-        if(AtlasGraphUtilsV1.isReference(elementType)) {
-            setListPropertyFromElementIds(instanceVertex, actualPropertyName, (List)values);
+    public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
+        String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName);
+
+        if (!AtlasGraphUtilsV1.isReference(elementType)) {
+            return instanceVertex.getProperty(vertexPropertyName, Object.class);
         }
-        else {
-            setProperty(instanceVertex, actualPropertyName, values);
+
+        return null;
+    }
+
+    public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) {
+        String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
+
+        if (isReference(elementType)) {
+            return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute);
+        } else {
+            return (List) instanceVertex.getListProperty(encodedPropertyName);
         }
     }
 
-    public static void setMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, Object value) {
-        String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
-        if(AtlasGraphUtilsV1.isReference(elementType)) {
-            instanceVertex.setPropertyFromElementId(actualPropertyName, (AtlasEdge)value);
+    public static Map<String, Object> getMapElementsProperty(AtlasMapType mapType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) {
+        AtlasType mapValueType = mapType.getValueType();
+
+        if (isReference(mapValueType)) {
+            return getReferenceMap(instanceVertex, attribute);
+        } else {
+            return getPrimitiveMap(instanceVertex, propertyName, mapValueType);
         }
-        else {
-            instanceVertex.setProperty(actualPropertyName, value);
+    }
+
+    // map elements for reference types - AtlasObjectId, AtlasStruct
+    public static Map<String, Object> getReferenceMap(AtlasVertex instanceVertex, AtlasAttribute attribute) {
+        Map<String, Object> ret            = new HashMap<>();
+        List<AtlasEdge>     referenceEdges = getCollectionElementsUsingRelationship(instanceVertex, attribute);
+
+        for (AtlasEdge edge : referenceEdges) {
+            String key = edge.getProperty(ATTRIBUTE_KEY_PROPERTY_KEY, String.class);
+
+            if (StringUtils.isNotEmpty(key)) {
+                ret.put(key, edge);
             }
         }
 
-    public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
-        String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName);
+        return ret;
+    }
 
-        if (AtlasGraphUtilsV1.isReference(elementType)) {
-            return instanceVertex.getProperty(vertexPropertyName, AtlasEdge.class);
-        } else {
-            return instanceVertex.getProperty(vertexPropertyName, Object.class);
+    public static List<AtlasEdge> getMapValuesUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute) {
+        String                         edgeLabel     = attribute.getRelationshipEdgeLabel();
+        AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
+        Iterator<AtlasEdge>            edgesForLabel = getEdgesForLabel(vertex, edgeLabel, edgeDirection);
+
+        return (List<AtlasEdge>) IteratorUtils.toList(edgesForLabel);
+    }
+
+    // map elements for primitive types
+    public static Map<String,Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName, AtlasType mapValueType) {
+        String              encodedPropertyName = encodePropertyKey(propertyName);
+        List<String>        currentKeys         = getListProperty(instanceVertex, encodedPropertyName);
+        Map<String, Object> ret                 = new HashMap<>();
+
+        if (CollectionUtils.isNotEmpty(currentKeys)) {
+            for (String key : currentKeys) {
+                String propertyNameForKey  = getQualifiedNameForMapKey(encodedPropertyName, encodePropertyKey(key));
+                Object propertyValueForKey = getMapValueProperty(mapValueType, instanceVertex, propertyNameForKey);
+
+                ret.put(key, propertyValueForKey);
             }
         }
 
-    // newly added
-    public static List<Object>  getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
-        String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
-        if(AtlasGraphUtilsV1.isReference(elementType)) {
-            return (List)instanceVertex.getListProperty(encodedPropertyName, AtlasEdge.class);
+        return ret;
     }
-        else {
-            return (List)instanceVertex.getListProperty(encodedPropertyName);
+
+    public static List<AtlasEdge> getCollectionElementsUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute) {
+        List<AtlasEdge>                ret;
+        String                         edgeLabel     = attribute.getRelationshipEdgeLabel();
+        AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
+        Iterator<AtlasEdge>            edgesForLabel = getEdgesForLabel(vertex, edgeLabel, edgeDirection);
+
+        ret = IteratorUtils.toList(edgesForLabel);
+
+        sortCollectionElements(attribute, ret);
+
+        return ret;
+    }
+
+    private static void sortCollectionElements(AtlasAttribute attribute, List<AtlasEdge> edges) {
+        // sort array elements based on edge index
+        if (attribute.getAttributeType() instanceof AtlasArrayType && CollectionUtils.isNotEmpty(edges)) {
+            Collections.sort(edges, (e1, e2) -> {
+                Integer e1Index = getIndexValue(e1);
+                Integer e2Index = getIndexValue(e2);
+
+                return e1Index.compareTo(e2Index);
+            });
         }
     }
 
@@ -1730,7 +1796,7 @@ public final class GraphHelper {
 
     }
 
-    public static void setListProperty(AtlasVertex instanceVertex, String propertyName, ArrayList<String> value) throws AtlasException {
+    public static void setListProperty(AtlasVertex instanceVertex, String propertyName, ArrayList<String> value) {
         String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
         instanceVertex.setListProperty(actualPropertyName, value);
     }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java

@@ -147,7 +147,6 @@ public class AtlasGraphUtilsV1 {
     public static boolean isReference(TypeCategory typeCategory) {
         return typeCategory == TypeCategory.STRUCT ||
                typeCategory == TypeCategory.ENTITY ||
-            typeCategory == TypeCategory.CLASSIFICATION ||
                typeCategory == TypeCategory.OBJECT_ID_TYPE;
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java

@@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.type.AtlasTypeUtil;
 import org.apache.atlas.utils.AtlasJson;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -97,15 +98,19 @@ import static org.apache.atlas.repository.graph.GraphHelper.getAdjacentEdgesByLa
 import static org.apache.atlas.repository.graph.GraphHelper.getAllClassificationEdges;
 import static org.apache.atlas.repository.graph.GraphHelper.getAllTraitNames;
 import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds;
+import static org.apache.atlas.repository.graph.GraphHelper.getArrayElementsProperty;
 import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
 import static org.apache.atlas.repository.graph.GraphHelper.getGuid;
 import static org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel;
+import static org.apache.atlas.repository.graph.GraphHelper.getPrimitiveMap;
+import static org.apache.atlas.repository.graph.GraphHelper.getReferenceMap;
 import static org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel;
 import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
 import static org.apache.atlas.repository.graph.GraphHelper.getRelationshipGuid;
 import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
 import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled;
 import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
+import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
 import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
 import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH;
 import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
@@ -614,10 +619,10 @@ public final class EntityGraphRetriever {
                 ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection);
                 break;
             case ARRAY:
-                ret = mapVertexToArray(entityVertex, (AtlasArrayType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, edgeDirection);
+                ret = mapVertexToArray(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
                 break;
             case MAP:
-                ret = mapVertexToMap(entityVertex, (AtlasMapType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, edgeDirection);
+                ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
                 break;
             case CLASSIFICATION:
                 // do nothing
@@ -627,45 +632,50 @@ public final class EntityGraphRetriever {
         return ret;
     }
 
-    private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasMapType atlasMapType, final String propertyName,
-                                               AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
-                                               AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
+    private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, final String propertyName, AtlasEntityExtInfo entityExtInfo,
+                                               boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
 
-        List<String> mapKeys = GraphHelper.getListProperty(entityVertex, propertyName);
-
-        if (CollectionUtils.isEmpty(mapKeys)) {
-            return null;
-        }
+        Map<String, Object> ret          = null;
+        AtlasMapType        mapType      = (AtlasMapType) attribute.getAttributeType();
+        AtlasType           mapValueType = mapType.getValueType();
 
         if (LOG.isDebugEnabled()) {
-            LOG.debug("Mapping map attribute {} for vertex {}", atlasMapType.getTypeName(), entityVertex);
+            LOG.debug("Mapping map attribute {} for vertex {}", mapType.getTypeName(), entityVertex);
         }
 
-        Map<String, Object> ret          = new HashMap<>(mapKeys.size());
-        AtlasType           mapValueType = atlasMapType.getValueType();
-
-        for (String mapKey : mapKeys) {
-            final String keyPropertyName = propertyName + "." + mapKey;
-            final String edgeLabel       = EDGE_LABEL_PREFIX + keyPropertyName;
-            final Object keyValue        = GraphHelper.getMapValueProperty(mapValueType, entityVertex, keyPropertyName);
+        if (isReference(mapValueType)) {
+            Map<String, Object> currentMap = getReferenceMap(entityVertex, attribute);
 
-            Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, edgeLabel,
-                                                         entityExtInfo, isOwnedAttribute, edgeDirection);
+            if (MapUtils.isNotEmpty(currentMap)) {
+                ret = new HashMap<>();
 
+                for (Map.Entry<String, Object> entry : currentMap.entrySet()) {
+                    String mapKey    = entry.getKey();
+                    Object keyValue  = entry.getValue();
+                    Object mapValue  = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, attribute.getRelationshipEdgeLabel(),
+                                                                  entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection());
                     if (mapValue != null) {
                         ret.put(mapKey, mapValue);
                     }
                 }
+            }
+        } else {
+            ret = getPrimitiveMap(entityVertex, propertyName, mapValueType);
+        }
+
+        if (MapUtils.isEmpty(ret)) {
+            ret = null;
+        }
 
         return ret;
     }
 
-    private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasArrayType arrayType, String propertyName,
-                                          AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
-                                          AtlasRelationshipEdgeDirection edgeDirection)  throws AtlasBaseException {
+    private List<Object> mapVertexToArray(AtlasVertex entityVertex, String propertyName, AtlasEntityExtInfo entityExtInfo,
+                                          boolean isOwnedAttribute, AtlasAttribute attribute)  throws AtlasBaseException {
 
+        AtlasArrayType arrayType        = (AtlasArrayType) attribute.getAttributeType();
         AtlasType      arrayElementType = arrayType.getElementType();
-        List<Object> arrayElements    = GraphHelper.getArrayElementsProperty(arrayElementType, entityVertex, propertyName);
+        List<Object>   arrayElements    = getArrayElementsProperty(arrayElementType, entityVertex, propertyName, attribute);
 
         if (CollectionUtils.isEmpty(arrayElements)) {
             return null;
@@ -676,7 +686,8 @@ public final class EntityGraphRetriever {
         }
 
         List                           arrValues     = new ArrayList(arrayElements.size());
-        String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
+        String                         edgeLabel     = attribute.getRelationshipEdgeLabel();
+        AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
 
         for (Object element : arrayElements) {
             // When internal types are deleted, sometimes the collection type attribute will contain a null value