ATLAS-2332: support for attributes having nested collection datatype

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java

@@ -24,12 +24,16 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.graphdb.GraphDatabase;
 import org.apache.atlas.repository.graphdb.janus.serializer.BigDecimalSerializer;
 import org.apache.atlas.repository.graphdb.janus.serializer.BigIntegerSerializer;
-import org.apache.atlas.repository.graphdb.janus.serializer.StringListSerializer;
 import org.apache.atlas.repository.graphdb.janus.serializer.TypeCategorySerializer;
 import org.apache.atlas.runner.LocalSolrRunner;
 import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.apache.commons.configuration.Configuration;
 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
+import org.janusgraph.graphdb.database.serialize.attribute.SerializableSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.janusgraph.core.JanusGraphFactory;
 import org.janusgraph.core.JanusGraph;
 import org.janusgraph.core.JanusGraphFactory;
 import org.janusgraph.core.schema.JanusGraphManagement;
@@ -81,7 +85,7 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
 
         //not ideal, but avoids making large changes to Atlas
         janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
-        janusConfig.addProperty("attributes.custom.attribute2.serializer-class", StringListSerializer.class.getName());
+        janusConfig.addProperty("attributes.custom.attribute2.serializer-class", SerializableSerializer.class.getName());
 
         janusConfig.addProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName());
         janusConfig.addProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName());

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/serializer/StringListSerializer.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.repository.graphdb.janus.serializer;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.janusgraph.core.attribute.AttributeSerializer;
-import org.janusgraph.diskstorage.ScanBuffer;
-import org.janusgraph.diskstorage.WriteBuffer;
-import org.janusgraph.graphdb.database.idhandling.VariableLong;
-import org.janusgraph.graphdb.database.serialize.attribute.StringSerializer;
-
-/**
- * Serializer for String lists.
- */
-public class StringListSerializer implements AttributeSerializer<List<String>> {
-
-    private final StringSerializer stringSerializer = new StringSerializer();
-
-    @Override
-    public List<String> read(ScanBuffer buffer) {
-        int length = (int)VariableLong.readPositive(buffer);
-        List<String> result = new ArrayList<String>(length);
-        for(int i = 0; i < length; i++) {
-            result.add(stringSerializer.read(buffer));
-        }
-        return result;
-    }
-
-    @Override
-    public void write(WriteBuffer buffer, List<String> attributes) {
-        VariableLong.writePositive(buffer, attributes.size());
-        for(String attr : attributes) {
-            stringSerializer.write(buffer, attr);
-        }
-    }
-
-}

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -1070,7 +1070,7 @@ public final class GraphHelper {
         if (AtlasGraphUtilsV1.isReference(elementType)) {
             return instanceVertex.getProperty(vertexPropertyName, AtlasEdge.class);
         } else {
-            return instanceVertex.getProperty(vertexPropertyName, Object.class).toString();
+            return instanceVertex.getProperty(vertexPropertyName, Object.class);
         }
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java

@@ -918,6 +918,8 @@ public class EntityGraphMapper {
         switch(ctx.getAttrType().getTypeCategory()) {
         case PRIMITIVE:
         case ENUM:
+        case MAP:
+        case ARRAY:
             return ctx.getValue();
 
         case STRUCT:
@@ -928,8 +930,6 @@ public class EntityGraphMapper {
             ctx.setElementType(instanceType);
             return mapObjectIdValueUsingRelationship(ctx, context);
 
-        case MAP:
-        case ARRAY:
         default:
                 throw new AtlasBaseException(AtlasErrorCode.TYPE_CATEGORY_INVALID, ctx.getAttrType().getTypeCategory().name());
         }
@@ -1025,6 +1025,10 @@ public class EntityGraphMapper {
     public static Object getMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) {
         if (AtlasGraphUtilsV1.isReference(elementType)) {
             return vertex.getProperty(vertexPropertyName, AtlasEdge.class);
+        } else if (elementType instanceof AtlasArrayType) {
+            return vertex.getProperty(vertexPropertyName, List.class);
+        } else if (elementType instanceof AtlasMapType) {
+            return vertex.getProperty(vertexPropertyName, Map.class);
         }
         else {
             return vertex.getProperty(vertexPropertyName, String.class).toString();

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java

@@ -505,11 +505,11 @@ public final class EntityGraphRetriever {
         switch (arrayElement.getTypeCategory()) {
             case PRIMITIVE:
             case ENUM:
+            case ARRAY:
+            case MAP:
                 ret = value;
                 break;
 
-            case ARRAY:
-            case MAP:
             case CLASSIFICATION:
                 break;
 

repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java

@@ -333,6 +333,21 @@ public class AtlasTypeDefGraphStoreTest {
         }
     }
 
+    @Test(dependsOnMethods = "testGet")
+    public void testCreateWithNestedContainerAttributes() {
+        AtlasTypesDef typesDef = TestUtilsV2.defineTypeWithNestedCollectionAttributes();
+
+        try {
+            AtlasTypesDef createdTypes = typeDefStore.createTypesDef(typesDef);
+            assertEquals(typesDef.getEnumDefs(), createdTypes.getEnumDefs(), "Data integrity issue while persisting");
+            assertEquals(typesDef.getStructDefs(), createdTypes.getStructDefs(), "Data integrity issue while persisting");
+            assertEquals(typesDef.getClassificationDefs(), createdTypes.getClassificationDefs(), "Data integrity issue while persisting");
+            assertEquals(typesDef.getEntityDefs(), createdTypes.getEntityDefs(), "Data integrity issue while persisting");
+        } catch (AtlasBaseException e) {
+            fail("creation of type with nested-container attributes should've succeeded");
+        }
+    }
+
     @Test(enabled = false)
     public void testCreateWithInvalidAttributes(){
     }

repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java

@@ -100,6 +100,7 @@ public class AtlasEntityStoreV1Test {
     private AtlasEntitiesWithExtInfo deptEntity;
     private AtlasEntityWithExtInfo   dbEntity;
     private AtlasEntityWithExtInfo   tblEntity;
+    private AtlasEntityWithExtInfo   nestedCollectionAttrEntity;
     private AtlasEntityWithExtInfo   primitiveEntity;
 
     AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
@@ -115,7 +116,8 @@ public class AtlasEntityStoreV1Test {
         new GraphBackedSearchIndexer(typeRegistry);
 
         AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(),
-                                                              TestUtilsV2.defineHiveTypes()
+                                                              TestUtilsV2.defineHiveTypes(),
+                                                              TestUtilsV2.defineTypeWithNestedCollectionAttributes(),
                                                             };
 
         for (AtlasTypesDef typesDef : testTypesDefs) {
@@ -130,6 +132,8 @@ public class AtlasEntityStoreV1Test {
         dbEntity   = TestUtilsV2.createDBEntityV2();
         tblEntity  = TestUtilsV2.createTableEntityV2(dbEntity.getEntity());
 
+        nestedCollectionAttrEntity = TestUtilsV2.createNestedCollectionAttrEntity();
+
         AtlasTypesDef typesDef11 = new  AtlasTypesDef();
         List primitiveEntityDef = new ArrayList<AtlasEntityDef>();
         primitiveEntityDef.add(TestUtilsV2.createPrimitiveEntityDef());
@@ -229,6 +233,14 @@ public class AtlasEntityStoreV1Test {
 
         AtlasEntityHeader tableEntity = tableCreationResponse.getFirstCreatedEntityByTypeName(TABLE_TYPE);
         validateEntity(tblEntity, getEntityFromStore(tableEntity));
+
+        //Create nested-collection attribute entity
+        init();
+        EntityMutationResponse entityMutationResponse = entityStore.createOrUpdate(new AtlasEntityStream(nestedCollectionAttrEntity), false);
+        validateMutationResponse(entityMutationResponse, EntityOperation.CREATE, 1);
+
+        AtlasEntityHeader createdEntity = entityMutationResponse.getFirstCreatedEntityByTypeName(TestUtilsV2.ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR);
+        validateEntity(nestedCollectionAttrEntity, getEntityFromStore(createdEntity));
     }
 
     @Test(dependsOnMethods = "testCreate")