ATLAS-2332: support for attributes having nested collection datatype

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java

@@ -24,12 +24,16 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.graphdb.GraphDatabase;
 import org.apache.atlas.repository.graphdb.janus.serializer.BigDecimalSerializer;
 import org.apache.atlas.repository.graphdb.janus.serializer.BigIntegerSerializer;
-import org.apache.atlas.repository.graphdb.janus.serializer.StringListSerializer;
 import org.apache.atlas.repository.graphdb.janus.serializer.TypeCategorySerializer;
 import org.apache.atlas.runner.LocalSolrRunner;
 import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.apache.commons.configuration.Configuration;
 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
+import org.janusgraph.graphdb.database.serialize.attribute.SerializableSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.janusgraph.core.JanusGraphFactory;
 import org.janusgraph.core.JanusGraph;
 import org.janusgraph.core.JanusGraphFactory;
 import org.janusgraph.core.schema.JanusGraphManagement;
@@ -81,7 +85,7 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
 
         //not ideal, but avoids making large changes to Atlas
         janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
-        janusConfig.addProperty("attributes.custom.attribute2.serializer-class", StringListSerializer.class.getName());
+        janusConfig.addProperty("attributes.custom.attribute2.serializer-class", SerializableSerializer.class.getName());
 
         janusConfig.addProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName());
         janusConfig.addProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName());

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/serializer/StringListSerializer.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.repository.graphdb.janus.serializer;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.janusgraph.core.attribute.AttributeSerializer;
-import org.janusgraph.diskstorage.ScanBuffer;
-import org.janusgraph.diskstorage.WriteBuffer;
-import org.janusgraph.graphdb.database.idhandling.VariableLong;
-import org.janusgraph.graphdb.database.serialize.attribute.StringSerializer;
-
-/**
- * Serializer for String lists.
- */
-public class StringListSerializer implements AttributeSerializer<List<String>> {
-
-    private final StringSerializer stringSerializer = new StringSerializer();
-
-    @Override
-    public List<String> read(ScanBuffer buffer) {
-        int length = (int)VariableLong.readPositive(buffer);
-        List<String> result = new ArrayList<String>(length);
-        for(int i = 0; i < length; i++) {
-            result.add(stringSerializer.read(buffer));
-        }
-        return result;
-    }
-
-    @Override
-    public void write(WriteBuffer buffer, List<String> attributes) {
-        VariableLong.writePositive(buffer, attributes.size());
-        for(String attr : attributes) {
-            stringSerializer.write(buffer, attr);
-        }
-    }
-
-}

intg/src/test/java/org/apache/atlas/TestUtilsV2.java

@@ -541,6 +541,7 @@ public final class TestUtilsV2 {
     public static final String SERDE_TYPE = "serdeType";
     public static final String COLUMNS_MAP = "columnsMap";
     public static final String COLUMNS_ATTR_NAME = "columns";
+    public static final String ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR = "entity_with_nested_collection_attr";
 
     public static final String NAME = "name";
 
@@ -832,6 +833,150 @@ public final class TestUtilsV2 {
         return ret;
     }
 
+    public static AtlasTypesDef defineTypeWithNestedCollectionAttributes() {
+        AtlasEntityDef nestedCollectionAttributesEntityType =
+                AtlasTypeUtil.createClassTypeDef(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR, ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR + "_description", null,
+                        AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
+
+                        new AtlasAttributeDef("mapOfArrayOfStrings", "map<string,array<string>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("mapOfArrayOfBooleans", "map<string,array<boolean>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("mapOfArrayOfInts", "map<string,array<int>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("mapOfArrayOfFloats", "map<string,array<float>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("mapOfArrayOfDates", "map<string,array<date>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+
+                        new AtlasAttributeDef("mapOfMapOfStrings", "map<string,map<string,string>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("mapOfMapOfBooleans", "map<string,map<string,boolean>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("mapOfMapOfInts", "map<string,map<string,int>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("mapOfMapOfFloats", "map<string,map<string,float>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("mapOfMapOfDates", "map<string,map<string,date>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+
+                        new AtlasAttributeDef("arrayOfArrayOfStrings", "array<array<string>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("arrayOfArrayOfBooleans", "array<array<boolean>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("arrayOfArrayOfInts", "array<array<int>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("arrayOfArrayOfFloats", "array<array<float>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("arrayOfArrayOfDates", "array<array<date>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+
+                        new AtlasAttributeDef("arrayOfMapOfStrings", "array<map<string,string>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("arrayOfMapOfBooleans", "array<map<string,boolean>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("arrayOfMapOfInts", "array<map<string,int>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("arrayOfMapOfFloats", "array<map<string,float>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList()),
+                        new AtlasAttributeDef("arrayOfMapOfDates", "array<map<string,date>>", false,
+                                AtlasAttributeDef.Cardinality.SINGLE, 1, 1,
+                                false, false,
+                                Collections.<AtlasConstraintDef>emptyList())
+                );
+
+        AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.emptyList(),
+                                                      Collections.emptyList(),
+                                                      Collections.emptyList(),
+                                                      Arrays.asList(nestedCollectionAttributesEntityType));
+
+        populateSystemAttributes(ret);
+
+        return ret;
+    }
+
+    public static AtlasEntityWithExtInfo createNestedCollectionAttrEntity() {
+        AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR);
+
+        String[]  arrayOfStrings  = new String[] { "one", "two", "three" };
+        boolean[] arrayOfBooleans = new boolean[] { false, true };
+        int[]     arrayOfInts     = new int[] { 1, 2, 3 };
+        float[]   arrayOfFloats   = new float[] { 1.1f, 2.2f, 3.3f };
+        Date[]    arrayOfDates    = new Date[] { new Date() };
+
+        Map<String, String>  mapOfStrings  = Collections.singletonMap("one", "one");
+        Map<String, Boolean> mapOfBooleans = Collections.singletonMap("one", true);
+        Map<String, Integer> mapOfInts     = Collections.singletonMap("one", 1);
+        Map<String, Float>   mapOfFloats   = Collections.singletonMap("one", 1.1f);
+        Map<String, Date>    mapOfDates    = Collections.singletonMap("now", new Date());
+
+        entity.setAttribute("name", randomString() + "_" + System.currentTimeMillis());
+
+        entity.setAttribute("mapOfArrayOfStrings", Collections.singletonMap("one", arrayOfStrings));
+        entity.setAttribute("mapOfArrayOfBooleans", Collections.singletonMap("one", arrayOfBooleans));
+        entity.setAttribute("mapOfArrayOfInts", Collections.singletonMap("one", arrayOfInts));
+        entity.setAttribute("mapOfArrayOfFloats", Collections.singletonMap("one", arrayOfFloats));
+        entity.setAttribute("mapOfArrayOfDates", Collections.singletonMap("one", arrayOfDates));
+
+        entity.setAttribute("mapOfMapOfStrings", Collections.singletonMap("one", mapOfStrings));
+        entity.setAttribute("mapOfMapOfBooleans", Collections.singletonMap("one", mapOfBooleans));
+        entity.setAttribute("mapOfMapOfInts", Collections.singletonMap("one", mapOfInts));
+        entity.setAttribute("mapOfMapOfFloats", Collections.singletonMap("one", mapOfFloats));
+        entity.setAttribute("mapOfMapOfDates", Collections.singletonMap("one", mapOfDates));
+
+        entity.setAttribute("arrayOfArrayOfStrings", Collections.singletonList(arrayOfStrings));
+        entity.setAttribute("arrayOfArrayOfBooleans", Collections.singletonList(arrayOfBooleans));
+        entity.setAttribute("arrayOfArrayOfInts", Collections.singletonList(arrayOfInts));
+        entity.setAttribute("arrayOfArrayOfFloats", Collections.singletonList(arrayOfFloats));
+        entity.setAttribute("arrayOfArrayOfDates", Collections.singletonList(arrayOfDates));
+
+        entity.setAttribute("arrayOfMapOfStrings", Collections.singletonList(mapOfStrings));
+        entity.setAttribute("arrayOfMapOfBooleans", Collections.singletonList(mapOfBooleans));
+        entity.setAttribute("arrayOfMapOfInts", Collections.singletonList(mapOfInts));
+        entity.setAttribute("arrayOfMapOfFloats", Collections.singletonList(mapOfFloats));
+        entity.setAttribute("arrayOfMapOfDates", Collections.singletonList(mapOfDates));
+
+        return new AtlasEntityWithExtInfo(entity);
+    }
+
     public static final String randomString() {
         return RandomStringUtils.randomAlphanumeric(10);
     }

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -1070,7 +1070,7 @@ public final class GraphHelper {
         if (AtlasGraphUtilsV1.isReference(elementType)) {
             return instanceVertex.getProperty(vertexPropertyName, AtlasEdge.class);
         } else {
-            return instanceVertex.getProperty(vertexPropertyName, Object.class).toString();
+            return instanceVertex.getProperty(vertexPropertyName, Object.class);
         }
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java

@@ -918,6 +918,8 @@ public class EntityGraphMapper {
         switch(ctx.getAttrType().getTypeCategory()) {
         case PRIMITIVE:
         case ENUM:
+        case MAP:
+        case ARRAY:
             return ctx.getValue();
 
         case STRUCT:
@@ -928,8 +930,6 @@ public class EntityGraphMapper {
             ctx.setElementType(instanceType);
             return mapObjectIdValueUsingRelationship(ctx, context);
 
-        case MAP:
-        case ARRAY:
         default:
                 throw new AtlasBaseException(AtlasErrorCode.TYPE_CATEGORY_INVALID, ctx.getAttrType().getTypeCategory().name());
         }
@@ -1025,6 +1025,10 @@ public class EntityGraphMapper {
     public static Object getMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) {
         if (AtlasGraphUtilsV1.isReference(elementType)) {
             return vertex.getProperty(vertexPropertyName, AtlasEdge.class);
+        } else if (elementType instanceof AtlasArrayType) {
+            return vertex.getProperty(vertexPropertyName, List.class);
+        } else if (elementType instanceof AtlasMapType) {
+            return vertex.getProperty(vertexPropertyName, Map.class);
         }
         else {
             return vertex.getProperty(vertexPropertyName, String.class).toString();

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java

@@ -505,11 +505,11 @@ public final class EntityGraphRetriever {
         switch (arrayElement.getTypeCategory()) {
             case PRIMITIVE:
             case ENUM:
+            case ARRAY:
+            case MAP:
                 ret = value;
                 break;
 
-            case ARRAY:
-            case MAP:
             case CLASSIFICATION:
                 break;
 

repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java

@@ -333,6 +333,21 @@ public class AtlasTypeDefGraphStoreTest {
         }
     }
 
+    @Test(dependsOnMethods = "testGet")
+    public void testCreateWithNestedContainerAttributes() {
+        AtlasTypesDef typesDef = TestUtilsV2.defineTypeWithNestedCollectionAttributes();
+
+        try {
+            AtlasTypesDef createdTypes = typeDefStore.createTypesDef(typesDef);
+            assertEquals(typesDef.getEnumDefs(), createdTypes.getEnumDefs(), "Data integrity issue while persisting");
+            assertEquals(typesDef.getStructDefs(), createdTypes.getStructDefs(), "Data integrity issue while persisting");
+            assertEquals(typesDef.getClassificationDefs(), createdTypes.getClassificationDefs(), "Data integrity issue while persisting");
+            assertEquals(typesDef.getEntityDefs(), createdTypes.getEntityDefs(), "Data integrity issue while persisting");
+        } catch (AtlasBaseException e) {
+            fail("creation of type with nested-container attributes should've succeeded");
+        }
+    }
+
     @Test(enabled = false)
     public void testCreateWithInvalidAttributes(){
     }

repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java

@@ -100,6 +100,7 @@ public class AtlasEntityStoreV1Test {
     private AtlasEntitiesWithExtInfo deptEntity;
     private AtlasEntityWithExtInfo   dbEntity;
     private AtlasEntityWithExtInfo   tblEntity;
+    private AtlasEntityWithExtInfo   nestedCollectionAttrEntity;
     private AtlasEntityWithExtInfo   primitiveEntity;
 
     AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
@@ -115,7 +116,8 @@ public class AtlasEntityStoreV1Test {
         new GraphBackedSearchIndexer(typeRegistry);
 
         AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(),
-                                                              TestUtilsV2.defineHiveTypes()
+                                                              TestUtilsV2.defineHiveTypes(),
+                                                              TestUtilsV2.defineTypeWithNestedCollectionAttributes(),
                                                             };
 
         for (AtlasTypesDef typesDef : testTypesDefs) {
@@ -130,6 +132,8 @@ public class AtlasEntityStoreV1Test {
         dbEntity   = TestUtilsV2.createDBEntityV2();
         tblEntity  = TestUtilsV2.createTableEntityV2(dbEntity.getEntity());
 
+        nestedCollectionAttrEntity = TestUtilsV2.createNestedCollectionAttrEntity();
+
         AtlasTypesDef typesDef11 = new  AtlasTypesDef();
         List primitiveEntityDef = new ArrayList<AtlasEntityDef>();
         primitiveEntityDef.add(TestUtilsV2.createPrimitiveEntityDef());
@@ -229,6 +233,14 @@ public class AtlasEntityStoreV1Test {
 
         AtlasEntityHeader tableEntity = tableCreationResponse.getFirstCreatedEntityByTypeName(TABLE_TYPE);
         validateEntity(tblEntity, getEntityFromStore(tableEntity));
+
+        //Create nested-collection attribute entity
+        init();
+        EntityMutationResponse entityMutationResponse = entityStore.createOrUpdate(new AtlasEntityStream(nestedCollectionAttrEntity), false);
+        validateMutationResponse(entityMutationResponse, EntityOperation.CREATE, 1);
+
+        AtlasEntityHeader createdEntity = entityMutationResponse.getFirstCreatedEntityByTypeName(TestUtilsV2.ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR);
+        validateEntity(nestedCollectionAttrEntity, getEntityFromStore(createdEntity));
     }
 
     @Test(dependsOnMethods = "testCreate")