ATLAS-2637: migration-import updates for changes in collection attribute storage

common/src/main/java/org/apache/atlas/AtlasConstants.java

@@ -35,6 +35,6 @@ public final class AtlasConstants {
     public static final int ATLAS_SHUTDOWN_HOOK_PRIORITY = 30;
     public static final String DEFAULT_TYPE_VERSION = "1.0";
 
-    public static final String ATLAS_MIGRATION_MODE_FILENAME = "atlas.migration.mode.filename";
+    public static final String ATLAS_MIGRATION_MODE_FILENAME = "atlas.migration.data.filename";
     public static final String ATLAS_SERVICES_ENABLED        = "atlas.services.enabled";
 }

graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java

@@ -30,6 +30,7 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.groovy.GroovyExpression;
 import org.apache.atlas.model.impexp.MigrationStatus;
 import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
 
 /**
  * Represents a graph.
@@ -320,7 +321,7 @@ public interface AtlasGraph<V, E> {
      */
     boolean isMultiProperty(String name);
 
-    void loadLegacyGraphSON(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException;
+    void importLegacyGraphSON(AtlasTypeRegistry typeRegistry, InputStream fs) throws AtlasBaseException;
 
     MigrationStatus getMigrationStatus();
 }

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java

@@ -34,10 +34,10 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
 import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.graphdb.GremlinVersion;
-import org.apache.atlas.repository.graphdb.janus.migration.ReaderStatusManager;
 import org.apache.atlas.repository.graphdb.janus.query.AtlasJanusGraphQuery;
 import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
 import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.commons.configuration.Configuration;
 import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider;
 import org.apache.tinkerpop.gremlin.groovy.DefaultImportCustomizerProvider;
@@ -84,26 +84,20 @@ import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_P
  */
 public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> {
 
-    private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
-
     private static Configuration APPLICATION_PROPERTIES = null;
 
-    private final class ConvertGremlinValueFunction implements Function<Object, Object> {
-        @Override
-        public Object apply(Object input) {
-            return convertGremlinValue(input);
-        }
-    }
-
-    private final Set<String> multiProperties;
+    private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
+    private final Set<String>                 multiProperties                   = new HashSet<>();
 
     public AtlasJanusGraph() {
         //determine multi-properties once at startup
         JanusGraphManagement mgmt = null;
+
         try {
             mgmt = AtlasJanusGraphDatabase.getGraphInstance().openManagement();
+
             Iterable<PropertyKey> keys = mgmt.getRelationTypes(PropertyKey.class);
-            multiProperties = new HashSet<>();
+
             for (PropertyKey key : keys) {
                 if (key.cardinality() != Cardinality.SINGLE) {
                     multiProperties.add(key.name());
@@ -120,11 +114,11 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
     public AtlasEdge<AtlasJanusVertex, AtlasJanusEdge> addEdge(AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> outVertex,
                                                                AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> inVertex,
                                                                String edgeLabel) {
-
         try {
             Vertex oV   = outVertex.getV().getWrappedElement();
             Vertex iV   = inVertex.getV().getWrappedElement();
             Edge   edge = oV.addEdge(edgeLabel, iV);
+
             return GraphDbObjectFactory.createEdge(this, edge);
         } catch (SchemaViolationException e) {
             throw new AtlasSchemaViolationException(e);
@@ -140,41 +134,42 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
     public AtlasEdge<AtlasJanusVertex, AtlasJanusEdge> getEdge(String edgeId) {
         Iterator<Edge> it = getGraph().edges(edgeId);
         Edge           e  = getSingleElement(it, edgeId);
+
         return GraphDbObjectFactory.createEdge(this, e);
     }
 
     @Override
     public void removeEdge(AtlasEdge<AtlasJanusVertex, AtlasJanusEdge> edge) {
-
         Edge wrapped = edge.getE().getWrappedElement();
-        wrapped.remove();
 
+        wrapped.remove();
     }
 
     @Override
     public void removeVertex(AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> vertex) {
         Vertex wrapped = vertex.getV().getWrappedElement();
+
         wrapped.remove();
     }
 
     @Override
     public Iterable<AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>> getEdges() {
-
         Iterator<Edge> edges = getGraph().edges();
-        return wrapEdges(edges);
 
+        return wrapEdges(edges);
     }
 
     @Override
     public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> getVertices() {
-
         Iterator<Vertex> vertices = getGraph().vertices();
+
         return wrapVertices(vertices);
     }
 
     @Override
     public AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> addVertex() {
         Vertex result = getGraph().addVertex();
+
         return GraphDbObjectFactory.createVertex(this, result);
     }
 
@@ -221,73 +216,32 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
         return getIndexKeys(Vertex.class);
     }
 
-    private Set<String> getIndexKeys(Class<? extends Element> janusGraphElementClass) {
-
-        JanusGraphManagement mgmt = getGraph().openManagement();
-        Iterable<JanusGraphIndex> indices = mgmt.getGraphIndexes(janusGraphElementClass);
-        Set<String> result = new HashSet<String>();
-        for (JanusGraphIndex index : indices) {
-            result.add(index.name());
-        }
-        mgmt.commit();
-        return result;
-
-    }
-
     @Override
     public AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> getVertex(String vertexId) {
         Iterator<Vertex> it     = getGraph().vertices(vertexId);
         Vertex           vertex = getSingleElement(it, vertexId);
-        return GraphDbObjectFactory.createVertex(this, vertex);
-    }
 
-    public static <T> T getSingleElement(Iterator<T> it, String id) {
-        if (!it.hasNext()) {
-            return null;
-        }
-        T element = it.next();
-        if (it.hasNext()) {
-            throw new RuntimeException("Multiple items were found with the id " + id);
-        }
-        return element;
+        return GraphDbObjectFactory.createVertex(this, vertex);
     }
 
     @Override
     public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> getVertices(String key, Object value) {
         AtlasGraphQuery<AtlasJanusVertex, AtlasJanusEdge> query = query();
-        query.has(key, value);
-        return query.vertices();
-    }
 
-    private Object convertGremlinValue(Object rawValue) {
+        query.has(key, value);
 
-        if (rawValue instanceof Vertex) {
-            return GraphDbObjectFactory.createVertex(this, (Vertex) rawValue);
-        } else if (rawValue instanceof Edge) {
-            return GraphDbObjectFactory.createEdge(this, (Edge) rawValue);
-        } else if (rawValue instanceof Map) {
-            Map<String,Object> rowValue = (Map<String,Object>)rawValue;
-            return Maps.transformValues(rowValue, GREMLIN_VALUE_CONVERSION_FUNCTION);
-        } else if (rawValue instanceof ImmutablePath) {
-            ImmutablePath path = (ImmutablePath) rawValue;
-            return convertGremlinValue(path.objects());
-        }
-        else if (rawValue instanceof List) {
-            return Lists.transform((List)rawValue, GREMLIN_VALUE_CONVERSION_FUNCTION);
-        } else if (rawValue instanceof Collection) {
-            throw new UnsupportedOperationException("Unhandled collection type: " + rawValue.getClass());
-        }
-        return rawValue;
+        return query.vertices();
     }
 
     @Override
     public GremlinVersion getSupportedGremlinVersion() {
-
         return GremlinVersion.THREE;
     }
+
     @Override
     public void clear() {
         JanusGraph graph = getGraph();
+
         if (graph.isOpen()) {
             // only a shut down graph can be cleared
             graph.close();
@@ -295,7 +249,8 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
 
         try {
             JanusGraphFactory.drop(graph);
-        } catch (BackendException ignoreEx) {}
+        } catch (BackendException ignoreEx) {
+        }
     }
 
     private JanusGraph getGraph() {
@@ -304,24 +259,26 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
 
     @Override
     public void exportToGson(OutputStream os) throws IOException {
-
         GraphSONMapper         mapper  = getGraph().io(IoCore.graphson()).mapper().create();
         GraphSONWriter.Builder builder = GraphSONWriter.build();
+
         builder.mapper(mapper);
+
         GraphSONWriter writer = builder.create();
+
         writer.writeGraph(os, getGraph());
     }
 
     @Override
     public GremlinGroovyScriptEngine getGremlinScriptEngine() {
         Set<String> extraImports       = new HashSet<String>();
-        extraImports.add(java.util.function.Function.class.getName());
-
         Set<String> extraStaticImports = new HashSet<String>();
+
+        extraImports.add(java.util.function.Function.class.getName());
         extraStaticImports.add(P.class.getName() + ".*");
         extraStaticImports.add(__.class.getName() + ".*");
-        CompilerCustomizerProvider provider = new DefaultImportCustomizerProvider(extraImports, extraStaticImports);
 
+        CompilerCustomizerProvider provider     = new DefaultImportCustomizerProvider(extraImports, extraStaticImports);
         GremlinGroovyScriptEngine  scriptEngine = new GremlinGroovyScriptEngine(provider);
 
         return scriptEngine;
@@ -341,38 +298,20 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
     @Override
     public Object executeGremlinScript(String query, boolean isPath) throws AtlasBaseException {
         Object result = executeGremlinScript(query);
-        return convertGremlinValue(result);
-    }
-
-    private Object executeGremlinScript(String gremlinQuery) throws AtlasBaseException {
-        GremlinGroovyScriptEngine scriptEngine = getGremlinScriptEngine();
-
-        try {
-            Bindings bindings = scriptEngine.createBindings();
-
-            bindings.put("graph", getGraph());
-            bindings.put("g", getGraph().traversal());
 
-            Object result = scriptEngine.eval(gremlinQuery, bindings);
-
-            return result;
-        } catch (ScriptException e) {
-            throw new AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, e, gremlinQuery);
-        } finally {
-            releaseGremlinScriptEngine(scriptEngine);
-        }
+        return convertGremlinValue(result);
     }
 
     @Override
-    public Object executeGremlinScript(ScriptEngine scriptEngine,
-            Map<? extends  String, ? extends  Object> userBindings, String query, boolean isPath)
-            throws ScriptException {
+    public Object executeGremlinScript(ScriptEngine scriptEngine, Map<? extends String, ? extends Object> userBindings,
+                                       String query, boolean isPath) throws ScriptException {
         Bindings bindings = scriptEngine.createBindings();
 
         bindings.putAll(userBindings);
         bindings.put("g", getGraph().traversal());
 
         Object result = scriptEngine.eval(query, bindings);
+
         return convertGremlinValue(result);
     }
 
@@ -402,39 +341,42 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
         return expr;
     }
 
-    public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> wrapVertices(Iterator<? extends Vertex> it) {
-        Iterable<? extends Vertex> iterable = new IteratorToIterableAdapter<>(it);
-        return wrapVertices(iterable);
+    @Override
+    public boolean isMultiProperty(String propertyName) {
+        return multiProperties.contains(propertyName);
     }
 
-    public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> wrapVertices(Iterable<? extends Vertex> it) {
+    @Override
+    public void importLegacyGraphSON(AtlasTypeRegistry typeRegistry, InputStream fs) throws AtlasBaseException {
+        AtlasJanusGraphDatabase.loadLegacyGraphSON(typeRegistry, fs);
+    }
 
-        return StreamSupport.stream(it.spliterator(), false).map(input -> GraphDbObjectFactory.createVertex(AtlasJanusGraph.this, input)).collect(Collectors.toList());
+    @Override
+    public MigrationStatus getMigrationStatus() {
+        return AtlasJanusGraphDatabase.getMigrationStatus();
+    }
 
+    public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> wrapVertices(Iterable<? extends Vertex> it) {
+        return StreamSupport.stream(it.spliterator(), false).map(input -> GraphDbObjectFactory.createVertex(AtlasJanusGraph.this, input)).collect(Collectors.toList());
     }
 
     public Iterable<AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>> wrapEdges(Iterator<? extends Edge> it) {
         Iterable<? extends Edge> iterable = new IteratorToIterableAdapter<>(it);
+
         return wrapEdges(iterable);
     }
 
     public Iterable<AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>> wrapEdges(Iterable<? extends Edge> it) {
-
         return StreamSupport.stream(it.spliterator(), false).map(input -> GraphDbObjectFactory.createEdge(AtlasJanusGraph.this, input)).collect(Collectors.toList());
-
-    }
-
-    @Override
-    public boolean isMultiProperty(String propertyName) {
-        return multiProperties.contains(propertyName);
     }
 
     public void addMultiProperties(Set<String> names) {
         multiProperties.addAll(names);
     }
 
-    public String getIndexQueryPrefix() {
-        String ret;
+
+    private String getIndexQueryPrefix() {
+        final String ret;
 
         initApplicationProperties();
 
@@ -447,6 +389,82 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
         return ret;
     }
 
+    private Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> wrapVertices(Iterator<? extends Vertex> it) {
+        Iterable<? extends Vertex> iterable = new IteratorToIterableAdapter<>(it);
+
+        return wrapVertices(iterable);
+    }
+
+    private static <T> T getSingleElement(Iterator<T> it, String id) {
+        if (!it.hasNext()) {
+            return null;
+        }
+
+        T element = it.next();
+
+        if (it.hasNext()) {
+            throw new RuntimeException("Multiple items were found with the id " + id);
+        }
+
+        return element;
+    }
+
+    private Object convertGremlinValue(Object rawValue) {
+        if (rawValue instanceof Vertex) {
+            return GraphDbObjectFactory.createVertex(this, (Vertex) rawValue);
+        } else if (rawValue instanceof Edge) {
+            return GraphDbObjectFactory.createEdge(this, (Edge) rawValue);
+        } else if (rawValue instanceof Map) {
+            Map<String,Object> rowValue = (Map<String,Object>)rawValue;
+
+            return Maps.transformValues(rowValue, GREMLIN_VALUE_CONVERSION_FUNCTION);
+        } else if (rawValue instanceof ImmutablePath) {
+            ImmutablePath path = (ImmutablePath) rawValue;
+
+            return convertGremlinValue(path.objects());
+        } else if (rawValue instanceof List) {
+            return Lists.transform((List)rawValue, GREMLIN_VALUE_CONVERSION_FUNCTION);
+        } else if (rawValue instanceof Collection) {
+            throw new UnsupportedOperationException("Unhandled collection type: " + rawValue.getClass());
+        }
+
+        return rawValue;
+    }
+
+    private Set<String> getIndexKeys(Class<? extends Element> janusGraphElementClass) {
+        JanusGraphManagement      mgmt    = getGraph().openManagement();
+        Iterable<JanusGraphIndex> indices = mgmt.getGraphIndexes(janusGraphElementClass);
+        Set<String>               result  = new HashSet<String>();
+
+        for (JanusGraphIndex index : indices) {
+            result.add(index.name());
+        }
+
+        mgmt.commit();
+
+        return result;
+
+    }
+
+    private Object executeGremlinScript(String gremlinQuery) throws AtlasBaseException {
+        GremlinGroovyScriptEngine scriptEngine = getGremlinScriptEngine();
+
+        try {
+            Bindings bindings = scriptEngine.createBindings();
+
+            bindings.put("graph", getGraph());
+            bindings.put("g", getGraph().traversal());
+
+            Object result = scriptEngine.eval(gremlinQuery, bindings);
+
+            return result;
+        } catch (ScriptException e) {
+            throw new AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, e, gremlinQuery);
+        } finally {
+            releaseGremlinScriptEngine(scriptEngine);
+        }
+    }
+
     private void initApplicationProperties() {
         if (APPLICATION_PROPERTIES == null) {
             try {
@@ -457,13 +475,11 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
         }
     }
 
-    @Override
-    public void loadLegacyGraphSON(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
-        AtlasJanusGraphDatabase.loadLegacyGraphSON(relationshipCache, fs);
-    }
 
+    private final class ConvertGremlinValueFunction implements Function<Object, Object> {
         @Override
-    public MigrationStatus getMigrationStatus() {
-        return AtlasJanusGraphDatabase.getMigrationStatus();
+        public Object apply(Object input) {
+            return convertGremlinValue(input);
+        }
     }
 }

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java

@@ -26,10 +26,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.graphdb.GraphDatabase;
 import org.apache.atlas.repository.graphdb.janus.migration.AtlasGraphSONReader;
 import org.apache.atlas.repository.graphdb.janus.migration.ReaderStatusManager;
+import org.apache.atlas.repository.graphdb.janus.migration.ElementProcessors;
 import org.apache.atlas.repository.graphdb.janus.serializer.BigDecimalSerializer;
 import org.apache.atlas.repository.graphdb.janus.serializer.BigIntegerSerializer;
 import org.apache.atlas.repository.graphdb.janus.serializer.TypeCategorySerializer;
 import org.apache.atlas.runner.LocalSolrRunner;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.commons.configuration.Configuration;
@@ -47,7 +49,6 @@ import java.io.InputStream;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.util.ArrayList;
-import java.util.Map;
 
 /**
  * Default implementation for Graph Provider that doles out JanusGraph.
@@ -231,7 +232,7 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
         return ret;
     }
 
-    public static void loadLegacyGraphSON(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
+    public static void loadLegacyGraphSON(AtlasTypeRegistry typeRegistry, InputStream fs) throws AtlasBaseException {
         AtlasPerfTracer perf = null;
 
         try {
@@ -242,9 +243,10 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
             }
 
             AtlasGraphSONReader legacyGraphSONReader = AtlasGraphSONReader.build().
-                    relationshipCache(relationshipCache).
+                    relationshipCache(new ElementProcessors(typeRegistry)).
                     schemaDB(getGraphInstance()).
-                    bulkLoadingDB(getBulkLoadingGraphInstance()).create();
+                    bulkLoadingDB(getBulkLoadingGraphInstance()).
+                    create();
 
             legacyGraphSONReader.readGraph(fs);
         } catch (Exception ex) {

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java

@@ -37,44 +37,46 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
 public final class AtlasGraphSONReader {
     private static final Logger LOG = LoggerFactory.getLogger(AtlasGraphSONReader.class);
 
+    private static String APPLICATION_PROPERTY_MIGRATION_START_INDEX      = "atlas.migration.mode.start.index";
+    private static String APPLICATION_PROPERTY_MIGRATION_NUMER_OF_WORKERS = "atlas.migration.mode.workers";
+    private static String APPLICATION_PROPERTY_MIGRATION_BATCH_SIZE       = "atlas.migration.mode.batch.size";
+
     private final ObjectMapper        mapper;
-    private final RelationshipTypeCache relationshipCache;
+    private final ElementProcessors   relationshipCache;
     private final Graph               graph;
     private final Graph               bulkLoadGraph;
     private final int                 numWorkers;
     private final int                 batchSize;
     private final long                suppliedStartIndex;
-    private final String[]              propertiesToPostProcess;
-    private       GraphSONUtility       graphSONUtility;
+    private final GraphSONUtility     graphSONUtility;
     private       ReaderStatusManager readerStatusManager;
     private       AtomicLong          counter;
 
-    private AtlasGraphSONReader(ObjectMapper mapper, Map<String, String> relationshipLookup, Graph graph,
-                                Graph bulkLoadGraph, String[] propertiesToPostProcess, int numWorkers, int batchSize, long suppliedStartIndex) {
+    private AtlasGraphSONReader(ObjectMapper mapper, ElementProcessors relationshipLookup, Graph graph,
+                                Graph bulkLoadGraph, int numWorkers, int batchSize, long suppliedStartIndex) {
         this.mapper                 = mapper;
-        this.relationshipCache      = new RelationshipTypeCache(relationshipLookup);
+        this.relationshipCache      = relationshipLookup;
         this.graph                  = graph;
         this.bulkLoadGraph          = bulkLoadGraph;
         this.numWorkers             = numWorkers;
         this.batchSize              = batchSize;
         this.suppliedStartIndex     = suppliedStartIndex;
-        this.propertiesToPostProcess = propertiesToPostProcess;
+        this.graphSONUtility        = new GraphSONUtility(relationshipCache);
     }
 
     public void readGraph(final InputStream inputStream) throws IOException {
         counter = new AtomicLong(0);
-        graphSONUtility = new GraphSONUtility(relationshipCache);
 
         final long        startIndex = initStatusManager();
         final JsonFactory factory    = mapper.getFactory();
 
         LOG.info("AtlasGraphSONReader.readGraph: numWorkers: {}: batchSize: {}: startIndex: {}", numWorkers, batchSize, startIndex);
+
         try (JsonParser parser = factory.createParser(inputStream)) {
             if (parser.nextToken() != JsonToken.START_OBJECT) {
                 throw new IOException("Expected data to start with an Object");
@@ -88,6 +90,7 @@ public final class AtlasGraphSONReader {
                 switch (fieldName) {
                     case GraphSONTokensTP2.MODE:
                         parser.nextToken();
+
                         final String mode = parser.getText();
 
                         if (!mode.equals("EXTENDED")) {
@@ -180,8 +183,7 @@ public final class AtlasGraphSONReader {
         LOG.info("postProcess: Starting... : counter at: {}", counter.get());
 
         try {
-            PostProcessManager.WorkItemsManager wim   = PostProcessManager.create(bulkLoadGraph, graphSONUtility,
-                                                                    propertiesToPostProcess, batchSize, numWorkers);
+            PostProcessManager.WorkItemsManager wim   = PostProcessManager.create(bulkLoadGraph, relationshipCache.getPropertiesToPostProcess(), batchSize, numWorkers);
             GraphTraversal                      query = bulkLoadGraph.traversal().V();
 
             while (query.hasNext()) {
@@ -228,30 +230,32 @@ public final class AtlasGraphSONReader {
         }
     }
 
-    public static Builder build() throws AtlasException {
+    public static Builder build() {
         return new Builder();
     }
 
     public final static class Builder {
         private int               batchSize = 500;
-        private Map<String, String> relationshipCache;
+        private ElementProcessors relationshipCache;
         private Graph             graph;
         private Graph             bulkLoadGraph;
         private int               numWorkers;
         private long              suppliedStartIndex;
-        private String[]            propertiesToPostProcess;
 
         private Builder() {
         }
 
-        private void setDefaults() throws AtlasException {
-            this.startIndex(ApplicationProperties.get().getLong("atlas.migration.mode.start.index", 0L))
-                .numWorkers(ApplicationProperties.get().getInt("atlas.migration.mode.workers", 4))
-                .batchSize(ApplicationProperties.get().getInt("atlas.migration.mode.batch.size", 3000))
-                .propertiesToPostProcess(getPropertiesToPostProcess("atlas.migration.mode.postprocess.properties"));
+        private void setDefaults() {
+            try {
+                this.startIndex(ApplicationProperties.get().getLong(APPLICATION_PROPERTY_MIGRATION_START_INDEX, 0L))
+                        .numWorkers(ApplicationProperties.get().getInt(APPLICATION_PROPERTY_MIGRATION_NUMER_OF_WORKERS, 4))
+                        .batchSize(ApplicationProperties.get().getInt(APPLICATION_PROPERTY_MIGRATION_BATCH_SIZE, 3000));
+            } catch (AtlasException ex) {
+                LOG.error("setDefaults: failed!", ex);
+            }
         }
 
-        public AtlasGraphSONReader create() throws AtlasException {
+        public AtlasGraphSONReader create() {
             setDefaults();
             if(bulkLoadGraph == null) {
                 bulkLoadGraph = graph;
@@ -261,10 +265,10 @@ public final class AtlasGraphSONReader {
             final GraphSONMapper         mapper  = builder.typeInfo(TypeInfo.NO_TYPES).create();
 
             return new AtlasGraphSONReader(mapper.createMapper(), relationshipCache, graph, bulkLoadGraph,
-                                           propertiesToPostProcess, numWorkers, batchSize, suppliedStartIndex);
+                                                                    numWorkers, batchSize, suppliedStartIndex);
         }
 
-        public Builder relationshipCache(Map<String, String> relationshipCache) {
+        public Builder relationshipCache(ElementProcessors relationshipCache) {
             this.relationshipCache = relationshipCache;
 
             return this;
@@ -305,27 +309,5 @@ public final class AtlasGraphSONReader {
 
             return this;
         }
-
-        public Builder propertiesToPostProcess(String[] list) {
-            this.propertiesToPostProcess = list;
-            return this;
-        }
-
-        private static String[] getPropertiesToPostProcess(String applicationPropertyKey) throws AtlasException {
-            final String HIVE_COLUMNS_PROPERTY        = "hive_table.columns";
-            final String HIVE_PARTITION_KEYS_PROPERTY = "hive_table.partitionKeys";
-            final String PROCESS_INPUT_PROPERTY       = "Process.inputs";
-            final String PROCESS_OUTPUT_PROPERTY      = "Process.outputs";
-            final String USER_PROFILE_OUTPUT_PROPERTY = "__AtlasUserProfile.savedSearches";
-
-            String[] defaultProperties = new String[] { HIVE_COLUMNS_PROPERTY, HIVE_PARTITION_KEYS_PROPERTY,
-                                                        PROCESS_INPUT_PROPERTY, PROCESS_OUTPUT_PROPERTY,
-                                                        USER_PROFILE_OUTPUT_PROPERTY
-                                                      };
-
-            String[] userDefinedList = ApplicationProperties.get().getStringArray(applicationPropertyKey);
-
-            return (userDefinedList == null || userDefinedList.length == 0) ? defaultProperties : userDefinedList;
-        }
     }
 }

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration;
+
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.lang.StringUtils;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
+import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
+
+public class ElementProcessors {
+    private static final Logger LOG = LoggerFactory.getLogger(ElementProcessors.class);
+
+    public  static final String   PRIMITIVE_MAP_CATEGORY       = "MAP_PRIMITIVE";
+    public  static final String   NON_PRIMITIVE_MAP_CATEGORY   = "MAP";
+    public  static final String   NON_PRIMITIVE_ARRAY_CATEGORY = "ARRAY";
+    private static final String[] NON_PRIMITIVE_KEYS           = { ElementProcessors.NON_PRIMITIVE_ARRAY_CATEGORY };
+
+    private final Map<String, RelationshipCacheGenerator.TypeInfo> relationshipLookup;
+    private final Map<String, Map<String, List<String>>>        postProcessMap;
+
+    private final NonPrimitiveListPropertyProcessor nonPrimitiveListPropertyProcessor = new NonPrimitiveListPropertyProcessor();
+    private final NonPrimitiveMapPropertyProcessor  nonPrimitiveMapPropertyProcessor  = new NonPrimitiveMapPropertyProcessor();
+    private final PrimitiveMapPropertyProcessor     primitiveMapPropertyProcessor     = new PrimitiveMapPropertyProcessor();
+    private final EdgeCollectionPropertyProcessor   edgeCollectionPropertyProcessor   = new EdgeCollectionPropertyProcessor();
+    private final EdgeRelationshipPropertyProcessor edgeRelationshipPropertyProcessor = new EdgeRelationshipPropertyProcessor();
+
+    public ElementProcessors(AtlasTypeRegistry typeRegistry) {
+        this(RelationshipCacheGenerator.get(typeRegistry), TypesWithCollectionsFinder.getVertexPropertiesForCollectionAttributes(typeRegistry));
+    }
+
+    ElementProcessors(Map<String, RelationshipCacheGenerator.TypeInfo> lookup, Map<String, Map<String, List<String>>> postProcessMap) {
+        this.relationshipLookup = lookup;
+        this.postProcessMap     = postProcessMap;
+    }
+
+    public static String[] getNonPrimitiveCategoryKeys() {
+        return NON_PRIMITIVE_KEYS;
+    }
+
+    public Map<String,Map<String, List<String>>> getPropertiesToPostProcess() {
+        return postProcessMap;
+    }
+
+    public String addIndexKeysForCollections(Vertex out, Object edgeId, String label, Map<String, Object> edgeProperties) {
+        return edgeCollectionPropertyProcessor.update(out, edgeId, label, edgeProperties);
+    }
+
+    public void processCollections(String typeNameKey, Map<String,Object> vertexProperties) {
+        if (!vertexProperties.containsKey(typeNameKey)) {
+            return;
+        }
+
+        String typeName = (String) vertexProperties.get(typeNameKey);
+
+        if (!postProcessMap.containsKey(typeName)) {
+            return;
+        }
+
+        primitiveMapPropertyProcessor.update(typeName, vertexProperties);
+        nonPrimitiveMapPropertyProcessor.update(typeName, vertexProperties);
+        nonPrimitiveListPropertyProcessor.update(typeName, vertexProperties);
+    }
+
+    public String updateEdge(Vertex in, Vertex out, Object edgeId, String label, Map<String,Object> props) {
+        return edgeRelationshipPropertyProcessor.update(in, out, edgeId, label, props);
+    }
+
+    private class NonPrimitiveMapPropertyProcessor {
+        final String category = NON_PRIMITIVE_MAP_CATEGORY;
+
+        public void update(String typeName, Map<String,Object> vertexProperties) {
+            if (!postProcessMap.containsKey(typeName)) {
+                return;
+            }
+
+            if (!postProcessMap.get(typeName).containsKey(category)) {
+                return;
+            }
+
+            List<String> propertyTypeList = postProcessMap.get(typeName).get(category);
+
+            for (String property : propertyTypeList) {
+                if (!vertexProperties.containsKey(property)) {
+                    continue;
+                }
+
+                List<Object> list = (List<Object>) vertexProperties.get(property);
+
+                if (list == null) {
+                    continue;
+                }
+
+                for (Object listEntry : list) {
+                    String key      = (String) listEntry;
+                    String valueKey = getMapKey(property, key);
+
+                    if (vertexProperties.containsKey(valueKey)) {
+                        vertexProperties.remove(valueKey);
+                    }
+                }
+
+                vertexProperties.remove(property);
+            }
+        }
+
+        private String getMapKey(String property, String key) {
+            return String.format("%s.%s", property, key);
+        }
+    }
+
+    private class PrimitiveMapPropertyProcessor {
+        final String category = PRIMITIVE_MAP_CATEGORY;
+
+        public void update(String typeName, Map<String, Object> vertexProperties) {
+            if (!postProcessMap.get(typeName).containsKey(category)) {
+                return;
+            }
+
+            List<String> propertyTypeList = postProcessMap.get(typeName).get(category);
+
+            for (String property : propertyTypeList) {
+                if (!vertexProperties.containsKey(property)) {
+                    continue;
+                }
+
+                List<Object> list = (List<Object>) vertexProperties.get(property);
+
+                if (list == null) {
+                    continue;
+                }
+
+                Map<String, Object> map = getAggregatedMap(vertexProperties, property, list);
+
+                vertexProperties.put(property, map);
+            }
+        }
+
+        private Map<String, Object> getAggregatedMap(Map<String, Object> vertexProperties, String property, List<Object> list) {
+            Map<String, Object> map = new HashMap<>();
+
+            for (Object listEntry : list) {
+                String key      = (String) listEntry;
+                String valueKey = getMapKey(property, key);
+
+                if (vertexProperties.containsKey(valueKey)) {
+                    Object value = getValueFromProperties(valueKey, vertexProperties);
+
+                    vertexProperties.remove(valueKey);
+
+                    map.put(key, value);
+                }
+            }
+
+            return map;
+        }
+
+        private String getMapKey(String property, String key) {
+            return String.format("%s.%s", property, key);
+        }
+
+        private Object getValueFromProperties(String key, Map<String, Object> vertexProperties) {
+            if (!vertexProperties.containsKey(key)) {
+                return null;
+            }
+
+            return vertexProperties.get(key);
+        }
+    }
+
+    private class NonPrimitiveListPropertyProcessor {
+        private final String category = NON_PRIMITIVE_ARRAY_CATEGORY;
+
+        private void update(String typeName, Map<String,Object> props) {
+            if(!postProcessMap.get(typeName).containsKey(category)) {
+                return;
+            }
+
+            List<String> propertyTypeList = postProcessMap.get(typeName).get(category);
+            for (String property : propertyTypeList) {
+                if(!props.containsKey(property)) {
+                    continue;
+                }
+
+                Map<String, String> listMap = getUpdatedEdgeList(props.get(property));
+
+                if(listMap == null) {
+                    continue;
+                }
+
+                props.put(property, listMap);
+            }
+        }
+
+        private Map<String, String> getUpdatedEdgeList(Object o) {
+            Map<String, String> listMap = new HashMap<>();
+
+            if(!(o instanceof List)) {
+                return null;
+            }
+
+            List list = (List) o;
+
+            for (int i = 0; i < list.size(); i++) {
+                listMap.put((String) list.get(i), Integer.toString(i));
+            }
+
+            return listMap;
+        }
+    }
+
+    private class EdgeRelationshipPropertyProcessor {
+        public String update(Vertex in, Vertex out, Object edgeId, String label, Map<String, Object> props) {
+            if(addRelationshipTypeForClassification(in, out, label, props)) {
+                label = Constants.CLASSIFICATION_LABEL;
+            } else {
+                addRelationshipTypeName(label, props);
+
+                label = addIndexKeysForCollections(out, edgeId, label, props);
+            }
+
+            addMandatoryRelationshipProperties(label, props);
+
+            return label;
+        }
+
+        private String getRelationshipTypeName(String label) {
+            return relationshipLookup.containsKey(label) ? relationshipLookup.get(label).getTypeName() : "";
+        }
+
+        private PropagateTags getDefaultPropagateValue(String label) {
+            return relationshipLookup.containsKey(label) ?
+                    relationshipLookup.get(label).getPropagateTags() :
+                    AtlasRelationshipDef.PropagateTags.NONE;
+        }
+
+        private boolean addRelationshipTypeForClassification(Vertex in, Vertex out, String label, Map<String, Object> props) {
+            if (in.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
+                String inTypeName  = (String) in.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
+
+                if (StringUtils.isNotEmpty(inTypeName)) {
+                    if (inTypeName.equals(label)) {
+                        props.put(Constants.ENTITY_TYPE_PROPERTY_KEY, inTypeName);
+
+                        addEntityGuidToTrait(in, out);
+
+                        return true;
+                    }
+                } else {
+                    LOG.info("Could not find typeName for trait: {}", label);
+                }
+            }
+
+            return false;
+        }
+
+        private void addEntityGuidToTrait(Vertex in, Vertex out) {
+            String entityGuid = "";
+
+            if (out.property(Constants.GUID_PROPERTY_KEY).isPresent()) {
+                entityGuid = (String) out.property(Constants.GUID_PROPERTY_KEY).value();
+            }
+
+            if(StringUtils.isNotEmpty(entityGuid)) {
+                in.property(CLASSIFICATION_ENTITY_GUID, entityGuid);
+                in.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY, false);
+            }
+        }
+
+        private void addRelationshipTypeName(String edgeLabel, Map<String, Object> props) {
+            String typeName = getRelationshipTypeName(edgeLabel);
+
+            if (StringUtils.isNotEmpty(typeName)) {
+                props.put(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+            } else {
+                if (LOG.isDebugEnabled()) {
+                    LOG.debug("Could not find relationship type for: {}", edgeLabel);
+                }
+            }
+        }
+
+        private void addMandatoryRelationshipProperties(String label, Map<String, Object> props) {
+            props.put(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, UUID.randomUUID().toString());
+
+            props.put(RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, String.valueOf(getDefaultPropagateValue(label)));
+            props.put(STATE_PROPERTY_KEY, "ACTIVE");
+        }
+    }
+
+    private class EdgeCollectionPropertyProcessor {
+        private static final int LABEL_INDEX = 0;
+        private static final int KEY_INDEX   = 1;
+
+        public String update(Vertex out, Object edgeId, String label, Map<String, Object> edgeProperties) {
+            String[] labelKeyPair = getNonPrimitiveArrayFromLabel(out, (String) edgeId, label);
+
+            if (labelKeyPair != null) {
+                edgeProperties.put(ATTRIBUTE_INDEX_PROPERTY_KEY, Integer.valueOf(labelKeyPair[KEY_INDEX]));
+
+                return label;
+            }
+
+            labelKeyPair = getNonPrimitiveMapKeyFromLabel(out, label);
+
+            if (labelKeyPair != null) {
+                label = labelKeyPair[LABEL_INDEX];
+
+                edgeProperties.put(ATTRIBUTE_KEY_PROPERTY_KEY, labelKeyPair[KEY_INDEX]);
+            }
+
+            return label;
+        }
+
+        private String[] getNonPrimitiveArrayFromLabel(Vertex v, String edgeId, String label) {
+            if (!v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
+                return null;
+            }
+
+            String typeName     = (String) v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
+            String propertyName = StringUtils.remove(label, Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+
+            if(!containsNonPrimitiveCollectionProperty(typeName, propertyName, NON_PRIMITIVE_ARRAY_CATEGORY)) {
+                return null;
+            }
+
+            Map<String, String> edgeIdIndexList = (Map<String, String>) v.property(propertyName).value();
+
+            if (edgeIdIndexList.containsKey(edgeId)) {
+                return getLabelKeyPair(label, edgeIdIndexList.get(edgeId));
+            }
+
+            return null;
+        }
+
+        // legacy edge label is in format: __<type name>.<key>
+        //      label: in new format which is type name
+        // this method extracts:
+        //      key: what remains of the legacy label string when '__' and type name are removed
+        private String[] getNonPrimitiveMapKeyFromLabel(Vertex v, String label) {
+            if (!v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
+                return null;
+            }
+
+            String typeName = (String) v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
+
+            if(!postProcessMap.containsKey(typeName)) {
+                return null;
+            }
+
+            if(!postProcessMap.get(typeName).containsKey(NON_PRIMITIVE_MAP_CATEGORY)) {
+                return null;
+            }
+
+            String       propertyName = StringUtils.remove(label, Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+            List<String> properties   = postProcessMap.get(typeName).get(NON_PRIMITIVE_MAP_CATEGORY);
+
+            for (String p : properties) {
+                if (propertyName.startsWith(p)) {
+                    return getLabelKeyPair(
+                            String.format("%s%s", Constants.INTERNAL_PROPERTY_KEY_PREFIX, p),
+                            StringUtils.remove(propertyName, p).substring(1).trim());
+                }
+            }
+
+            return null;
+        }
+
+        private boolean containsNonPrimitiveCollectionProperty(String typeName, String propertyName, String categoryType) {
+            if (!postProcessMap.containsKey(typeName)) {
+                return false;
+            }
+
+            if (!postProcessMap.get(typeName).containsKey(categoryType)) {
+                return false;
+            }
+
+            List<String> properties = postProcessMap.get(typeName).get(categoryType);
+
+            for (String p : properties) {
+                if (p.equals(propertyName)) {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        private String[] getLabelKeyPair(String label, String value) {
+            return new String[] { label, value };
+        }
+    }
+}

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtility.java

@@ -18,10 +18,9 @@
 
 package org.apache.atlas.repository.graphdb.janus.migration;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.apache.atlas.repository.Constants;
-import org.apache.atlas.type.AtlasBuiltInTypes;
-import org.apache.commons.lang.StringUtils;
+import org.apache.atlas.type.AtlasBuiltInTypes.AtlasBigDecimalType;
+import org.apache.atlas.type.AtlasBuiltInTypes.AtlasBigIntegerType;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Graph;
 import org.apache.tinkerpop.gremlin.structure.Graph.Features.EdgeFeatures;
@@ -34,21 +33,17 @@ import org.slf4j.LoggerFactory;
 
 import java.util.*;
 
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_STATE_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
-import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
-
 class GraphSONUtility {
     private static final Logger LOG = LoggerFactory.getLogger(GraphSONUtility.class);
 
     private static final String              EMPTY_STRING   = "";
+    private static final AtlasBigIntegerType bigIntegerType = new AtlasBigIntegerType();
+    private static final AtlasBigDecimalType bigDecimalType = new AtlasBigDecimalType();
 
-    private final RelationshipTypeCache relationshipTypeCache;
-    private static AtlasBuiltInTypes.AtlasBigIntegerType bigIntegerType = new AtlasBuiltInTypes.AtlasBigIntegerType();
-    private static AtlasBuiltInTypes.AtlasBigDecimalType bigDecimalType = new AtlasBuiltInTypes.AtlasBigDecimalType();
+    private final ElementProcessors elementProcessors;
 
-    public GraphSONUtility(final RelationshipTypeCache relationshipTypeCache) {
-        this.relationshipTypeCache = relationshipTypeCache;
+    public GraphSONUtility(final ElementProcessors elementProcessors) {
+        this.elementProcessors = elementProcessors;
     }
 
     public Map<String, Object> vertexFromJson(Graph g, final JsonNode json) {
@@ -64,6 +59,7 @@ class GraphSONUtility {
         Vertex              vertex         = vertexFeatures.willAllowId(vertexId) ? g.addVertex(T.id, vertexId) : g.addVertex();
 
         props.put(Constants.VERTEX_ID_IN_IMPORT_KEY, vertexId);
+        elementProcessors.processCollections(Constants.ENTITY_TYPE_PROPERTY_KEY, props);
 
         for (Map.Entry<String, Object> entry : props.entrySet()) {
             try {
@@ -107,17 +103,11 @@ class GraphSONUtility {
 
             props.put(Constants.EDGE_ID_IN_IMPORT_KEY, edgeId.toString());
 
-            if(addRelationshipTypeForClassification(in, out, label, props)) {
-                label = Constants.CLASSIFICATION_LABEL;
-            } else {
-                addRelationshipTypeName(label, props);
-            }
+            label = elementProcessors.updateEdge(in, out, edgeId, label, props);
 
             EdgeFeatures  edgeFeatures = g.features().edge();
             final Edge    edge         = edgeFeatures.willAllowId(edgeId) ? out.addEdge(label, in, T.id, edgeId) : out.addEdge(label, in);
 
-            addMandatoryRelationshipProperties(props);
-
             for (Map.Entry<String, Object> entry : props.entrySet()) {
                 try {
                     edge.property(entry.getKey(), entry.getValue());
@@ -153,82 +143,6 @@ class GraphSONUtility {
         return cache.getMappedVertex(gr, inVId);
     }
 
-    private boolean addRelationshipTypeForClassification(Vertex in, Vertex out, String label, Map<String, Object> props) {
-        if (in.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
-            String inTypeName  = (String) in.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
-
-            if (inTypeName.equals(label)) {
-                if (StringUtils.isNotEmpty(inTypeName)) {
-                    props.put(Constants.ENTITY_TYPE_PROPERTY_KEY, inTypeName);
-
-                    addEntityGuidToTrait(in, out);
-
-                    return true;
-                } else {
-                    LOG.info("Could not find typeName for trait: {}", label);
-                }
-            }
-        }
-
-        return false;
-    }
-
-    private void addEntityGuidToTrait(Vertex in, Vertex out) {
-        String entityGuid = "";
-        if (out.property(Constants.GUID_PROPERTY_KEY).isPresent()) {
-            entityGuid = (String) out.property(Constants.GUID_PROPERTY_KEY).value();
-        }
-
-        if(StringUtils.isNotEmpty(entityGuid)) {
-            in.property(CLASSIFICATION_ENTITY_GUID, entityGuid);
-        }
-    }
-
-    private void addRelationshipTypeName(String edgeLabel, Map<String, Object> props) {
-        String typeName = relationshipTypeCache.get(edgeLabel);
-
-        if (StringUtils.isNotEmpty(typeName)) {
-            props.put(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
-        } else {
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("Could not find relationship type for: {}", edgeLabel);
-            }
-        }
-    }
-
-    private void addMandatoryRelationshipProperties(Map<String, Object> props) {
-        props.put(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, UUID.randomUUID().toString());
-        props.put(RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, "NONE");
-        props.put(CLASSIFICATION_EDGE_STATE_PROPERTY_KEY, "ACTIVE");
-    }
-
-    public void replaceReferencedEdgeIdForList(Graph g, MappedElementCache cache, Vertex v, String propertyName) {
-        try {
-            if (v.property(Constants.TYPENAME_PROPERTY_KEY).isPresent() || !v.property(propertyName).isPresent()) {
-                return;
-            }
-
-            List list = (List) v.property(propertyName).value();
-            for (int i = 0; i < list.size(); i++) {
-                String id    = list.get(i).toString();
-                Object newId = cache.getMappedEdge(g, id);
-
-                if (newId == null) {
-                    continue;
-                }
-
-                list.set(i, newId.toString());
-            }
-
-            v.property(propertyName, list);
-        } catch (IllegalArgumentException ex) {
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("processItem: IllegalArgumentException: v[{}] error!", v.id(), ex);
-            }
-        }
-    }
-
-    @VisibleForTesting
     static Map<String, Object> readProperties(final JsonNode node) {
         final Map<String, Object>                   map      = new HashMap<>();
         final Iterator<Map.Entry<String, JsonNode>> iterator = node.fields();
@@ -303,7 +217,6 @@ class GraphSONUtility {
         return array;
     }
 
-    @VisibleForTesting
     static Object getTypedValueFromJsonNode(final JsonNode node) {
         Object theValue = null;
 

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeProcessManager.java

@@ -101,6 +101,7 @@ public class JsonNodeProcessManager {
 
         private void commitRegular() {
             commit(graph, nodes.size());
+            cache.clearAll();
         }
 
         private void commit(Graph g, int size) {

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCache.java

@@ -18,9 +18,7 @@
 
 package org.apache.atlas.repository.graphdb.janus.migration;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.apache.atlas.utils.LruCache;
-import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Graph;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.slf4j.Logger;
@@ -28,18 +26,13 @@ import org.slf4j.LoggerFactory;
 
 import java.util.Map;
 
-import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY;
 import static org.apache.atlas.repository.Constants.VERTEX_ID_IN_IMPORT_KEY;
 
 public class MappedElementCache {
     private static final Logger LOG = LoggerFactory.getLogger(MappedElementCache.class);
 
-    @VisibleForTesting
     final Map<Object, Vertex> lruVertexCache = new LruCache<>(500, 100000);
 
-    @VisibleForTesting
-    final Map<String, String> lruEdgeCache   = new LruCache<>(500, 100000);
-
     public Vertex getMappedVertex(Graph gr, Object key) {
         try {
             Vertex ret = lruVertexCache.get(key);
@@ -62,32 +55,6 @@ public class MappedElementCache {
         }
     }
 
-    public String getMappedEdge(Graph gr, String key) {
-        try {
-            String ret = lruEdgeCache.get(key);
-
-            if (ret == null) {
-                synchronized (lruEdgeCache) {
-                    ret = lruEdgeCache.get(key);
-
-                    if (ret == null) {
-                        Edge e = fetchEdge(gr, key);
-
-                        ret = e.id().toString();
-
-                        lruEdgeCache.put(key, ret);
-                    }
-                }
-            }
-
-            return ret;
-        } catch (Exception ex) {
-            LOG.error("getMappedEdge: {}", key, ex);
-            return null;
-        }
-    }
-
-    @VisibleForTesting
     Vertex fetchVertex(Graph gr, Object key) {
         try {
             return gr.traversal().V().has(VERTEX_ID_IN_IMPORT_KEY, key).next();
@@ -97,18 +64,7 @@ public class MappedElementCache {
         }
     }
 
-    @VisibleForTesting
-    Edge fetchEdge(Graph gr, String key) {
-        try {
-            return gr.traversal().E().has(EDGE_ID_IN_IMPORT_KEY, key).next();
-        } catch (Exception ex) {
-            LOG.error("fetchEdge: fetchFromDB failed: {}", key);
-            return null;
-        }
-    }
-
     public void clearAll() {
         lruVertexCache.clear();
-        lruEdgeCache.clear();
     }
 }

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessManager.java

@@ -21,36 +21,42 @@ package org.apache.atlas.repository.graphdb.janus.migration;
 import org.apache.atlas.repository.graphdb.janus.migration.pc.WorkItemBuilder;
 import org.apache.atlas.repository.graphdb.janus.migration.pc.WorkItemConsumer;
 import org.apache.atlas.repository.graphdb.janus.migration.pc.WorkItemManager;
+import org.apache.atlas.repository.graphdb.janus.migration.postProcess.PostProcessListProperty;
 import org.apache.tinkerpop.gremlin.structure.Graph;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 
+import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
+
 public class PostProcessManager {
     static class Consumer extends WorkItemConsumer<Object> {
         private static final Logger LOG = LoggerFactory.getLogger(Consumer.class);
 
         private final Graph                                  bulkLoadGraph;
-        private final GraphSONUtility    utility;
-        private final String[]           properties;
-        private final MappedElementCache cache;
+        private final Map<String, Map<String, List<String>>> typePropertiesMap;
         private final int                                    batchSize;
         private       long                                   counter;
         private       long                                   batchCounter;
+        private final PostProcessListProperty                processor;
+        private final String[]                               nonPrimitiveCategoryKeys;
 
-        public Consumer(BlockingQueue<Object> queue, Graph bulkLoadGraph, GraphSONUtility utility,
-                        String[] properties, MappedElementCache cache, int batchSize) {
+        public Consumer(BlockingQueue<Object> queue, Graph bulkLoadGraph, Map<String, Map<String, List<String>>> typePropertiesMap, int batchSize) {
             super(queue);
 
             this.bulkLoadGraph            = bulkLoadGraph;
-            this.utility         = utility;
-            this.properties      = properties;
-            this.cache           = cache;
+            this.typePropertiesMap        = typePropertiesMap;
             this.batchSize                = batchSize;
             this.counter                  = 0;
             this.batchCounter             = 0;
+            this.processor                = new PostProcessListProperty();
+            this.nonPrimitiveCategoryKeys = ElementProcessors.getNonPrimitiveCategoryKeys();
         }
 
         @Override
@@ -59,22 +65,43 @@ public class PostProcessManager {
             counter++;
 
             try {
-                Vertex v = bulkLoadGraph.traversal().V(vertexId).next();
+                Vertex         vertex           = bulkLoadGraph.traversal().V(vertexId).next();
+                boolean        isTypeVertex     = vertex.property(TYPENAME_PROPERTY_KEY).isPresent();
+                VertexProperty typeNameProperty = vertex.property(ENTITY_TYPE_PROPERTY_KEY);
+
+                if (!isTypeVertex && typeNameProperty.isPresent()) {
+                    String typeName = (String) typeNameProperty.value();
+                    if (!typePropertiesMap.containsKey(typeName)) {
+                        return;
+                    }
+
+                    Map<String, List<String>> collectionTypeProperties = typePropertiesMap.get(typeName);
+                    for (String key : nonPrimitiveCategoryKeys) {
+                        if (!collectionTypeProperties.containsKey(key)) {
+                            continue;
+                        }
 
-                for (String p : properties) {
-                    utility.replaceReferencedEdgeIdForList(bulkLoadGraph, cache, v, p);
+                        for(String propertyName : collectionTypeProperties.get(key)) {
+                            processor.process(vertex, typeName, propertyName);
+                        }
+                    }
                 }
 
+                commitBatch();
+            } catch (Exception ex) {
+                LOG.error("processItem: v[{}] error!", vertexId, ex);
+            }
+        }
+
+        private void commitBatch() {
             if (batchCounter >= batchSize) {
                 LOG.info("[{}]: batch: {}: commit", counter, batchCounter);
+
                 commit();
+
                 batchCounter = 0;
             }
         }
-            catch (Exception ex) {
-                LOG.error("processItem: v[{}] error!", vertexId, ex);
-            }
-        }
 
         @Override
         protected void doCommit() {
@@ -84,22 +111,18 @@ public class PostProcessManager {
 
     private static class ConsumerBuilder implements WorkItemBuilder<Consumer, Object> {
         private final Graph                                  bulkLoadGraph;
-        private final GraphSONUtility    utility;
         private final int                                    batchSize;
-        private final MappedElementCache cache;
-        private final String[]           vertexPropertiesToPostProcess;
+        private final Map<String, Map<String, List<String>>> vertexPropertiesToPostProcess;
 
-        public ConsumerBuilder(Graph bulkLoadGraph, GraphSONUtility utility, String[] propertiesToPostProcess, int batchSize) {
+        public ConsumerBuilder(Graph bulkLoadGraph, Map<String, Map<String, List<String>>> propertiesToPostProcess, int batchSize) {
             this.bulkLoadGraph                 = bulkLoadGraph;
-            this.utility                       = utility;
             this.batchSize                     = batchSize;
-            this.cache                         = new MappedElementCache();
             this.vertexPropertiesToPostProcess = propertiesToPostProcess;
         }
 
         @Override
         public Consumer build(BlockingQueue<Object> queue) {
-            return new Consumer(queue, bulkLoadGraph, utility, vertexPropertiesToPostProcess, cache, batchSize);
+            return new Consumer(queue, bulkLoadGraph, vertexPropertiesToPostProcess, batchSize);
         }
     }
 
@@ -109,8 +132,9 @@ public class PostProcessManager {
         }
     }
 
-    public static WorkItemsManager create(Graph bGraph, GraphSONUtility utility, String[] propertiesToPostProcess, int batchSize, int numWorkers) {
-        ConsumerBuilder cb = new ConsumerBuilder(bGraph, utility, propertiesToPostProcess, batchSize);
+    public static WorkItemsManager create(Graph bGraph, Map<String, Map<String, List<String>>> propertiesToPostProcess,
+                                          int batchSize, int numWorkers) {
+        ConsumerBuilder cb = new ConsumerBuilder(bGraph, propertiesToPostProcess, batchSize);
 
         return new WorkItemsManager(cb, batchSize, numWorkers);
     }

repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java → graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipCacheGenerator.java

@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
  * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,58 +16,83 @@
  * limitations under the License.
  */
 
-package org.apache.atlas.repository.migration;
+package org.apache.atlas.repository.graphdb.janus.migration;
 
 import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
 import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
 import org.apache.atlas.repository.Constants;
-import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasRelationshipType;
 import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.v1.typesystem.types.utils.TypesUtil;
 import org.apache.commons.lang.StringUtils;
 
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 
 public class RelationshipCacheGenerator {
 
-    public static Map<String, String> get(AtlasTypeRegistry typeRegistry) {
-        Map<String, String>               ret               = new HashMap<>();
-        Collection<AtlasRelationshipType> relationshipTypes = typeRegistry.getAllRelationshipTypes();
+    public static class TypeInfo extends TypesUtil.Pair<String, PropagateTags> {
 
-        for (AtlasRelationshipType rt : relationshipTypes) {
-            AtlasRelationshipDef rd          = rt.getRelationshipDef();
-            String               relTypeName = rt.getTypeName();
+        public TypeInfo(String typeName, PropagateTags propagateTags) {
+            super(typeName, propagateTags);
+        }
+
+        public String getTypeName() {
+            return left;
+        }
+
+        public PropagateTags getPropagateTags() {
+            return right;
+        }
+    }
 
-            add(ret, getKey(rd.getEndDef1(), rt.getEnd1Type()), relTypeName);
-            add(ret, getKey(rd.getEndDef2(), rt.getEnd2Type()), relTypeName);
+
+    public static Map<String, TypeInfo> get(AtlasTypeRegistry typeRegistry) {
+        Map<String, TypeInfo> ret = new HashMap<>();
+
+        for (AtlasRelationshipType relType : typeRegistry.getAllRelationshipTypes()) {
+            AtlasRelationshipDef relDef      = relType.getRelationshipDef();
+            String               relTypeName = relType.getTypeName();
+
+            add(ret, getKey(relDef.getEndDef1()), relTypeName, relDef.getPropagateTags());
+            add(ret, getKey(relDef.getEndDef2()), relTypeName, getEnd2PropagateTag(relDef.getPropagateTags()));
         }
 
         return ret;
     }
 
-    private static String getKey(AtlasRelationshipEndDef ed, AtlasEntityType rt) {
-        return getKey(ed.getIsLegacyAttribute(), rt.getTypeName(), ed.getName());
+    private static String getKey(AtlasRelationshipEndDef endDef) {
+        return getKey(endDef.getIsLegacyAttribute(), endDef.getType(), endDef.getName());
     }
 
     private static String getKey(String lhs, String rhs) {
         return String.format("%s%s.%s", Constants.INTERNAL_PROPERTY_KEY_PREFIX, lhs, rhs);
     }
 
-    private static String getKey(boolean isLegacy, String typeName, String name) {
-        if(!isLegacy) {
+    private static String getKey(boolean isLegacy, String entityTypeName, String relEndName) {
+        if (!isLegacy) {
             return "";
         }
 
-        return getKey(typeName, name);
+        return getKey(entityTypeName, relEndName);
     }
 
-    private static void add(Map<String, String> map, String key, String value) {
-        if(StringUtils.isEmpty(key) || map.containsKey(key)) {
+    private static void add(Map<String, TypeInfo> map, String key, String relationTypeName, PropagateTags propagateTags) {
+        if (StringUtils.isEmpty(key) || map.containsKey(key)) {
             return;
         }
 
-        map.put(key, value);
+        map.put(key, new TypeInfo(relationTypeName, propagateTags));
+    }
+
+    private static PropagateTags getEnd2PropagateTag(PropagateTags end1PropagateTags) {
+        if (end1PropagateTags == PropagateTags.ONE_TO_TWO) {
+            return PropagateTags.TWO_TO_ONE;
+        } else if (end1PropagateTags == PropagateTags.TWO_TO_ONE) {
+            return PropagateTags.ONE_TO_TWO;
+        } else {
+            return end1PropagateTags;
+        }
     }
 }

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipTypeCache.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.graphdb.janus.migration;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-public class RelationshipTypeCache {
-    private static final Logger LOG = LoggerFactory.getLogger(RelationshipTypeCache.class);
-    private final Map<String, String> relationshipLookup;
-
-    public RelationshipTypeCache(Map<String, String> lookup) {
-        relationshipLookup = lookup;
-    }
-
-    public String get(String label) {
-        return relationshipLookup.get(label);
-    }
-}

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration;
+
+import org.apache.atlas.model.TypeCategory;
+import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasMapType;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.atlas.model.TypeCategory.*;
+
+public class TypesWithCollectionsFinder {
+    private static final Logger LOG = LoggerFactory.getLogger(TypesWithCollectionsFinder.class);
+
+    static final EnumSet<TypeCategory> nonPrimitives = EnumSet.of(ENTITY, STRUCT, OBJECT_ID_TYPE);
+
+    public static Map<String, Map<String, List<String>>> getVertexPropertiesForCollectionAttributes(AtlasTypeRegistry typeRegistry) {
+        Map<String, Map<String, List<String>>> ret = new HashMap<>();
+
+        addVertexPropertiesForCollectionAttributes(typeRegistry.getAllEntityTypes(), ret);
+        addVertexPropertiesForCollectionAttributes(typeRegistry.getAllStructTypes(), ret);
+
+        displayInfo("types with properties: ", ret);
+
+        return ret;
+    }
+
+    private static void addVertexPropertiesForCollectionAttributes(Collection<? extends AtlasStructType> types, Map<String, Map<String, List<String>>> typeAttrMap) {
+        for (AtlasStructType type : types) {
+            Map<String, List<String>> collectionProperties = getVertexPropertiesForCollectionAttributes(type);
+
+            if(collectionProperties != null && collectionProperties.size() > 0) {
+                typeAttrMap.put(type.getTypeName(), collectionProperties);
+            }
+        }
+    }
+
+    static Map<String, List<String>> getVertexPropertiesForCollectionAttributes(AtlasStructType type) {
+        try {
+            Map<String, List<String>> collectionProperties = new HashMap<>();
+
+            for (AtlasAttribute attr : type.getAllAttributes().values()) {
+                addIfCollectionAttribute(attr, collectionProperties);
+            }
+
+            return collectionProperties;
+        } catch (Exception e) {
+            LOG.error("addVertexPropertiesForCollectionAttributes", e);
+        }
+
+        return null;
+    }
+
+    private static void addIfCollectionAttribute(AtlasAttribute attr, Map<String, List<String>> collectionProperties) {
+        AtlasType    attrType         = attr.getAttributeType();
+        TypeCategory attrTypeCategory = attrType.getTypeCategory();
+
+        switch (attrTypeCategory) {
+            case ARRAY: {
+                TypeCategory arrayElementType = ((AtlasArrayType) attrType).getElementType().getTypeCategory();
+
+                if (nonPrimitives.contains(arrayElementType)) {
+                    addVertexProperty(attrTypeCategory.toString(), attr.getVertexPropertyName(), collectionProperties);
+                }
+            }
+            break;
+
+            case MAP: {
+                TypeCategory mapValueType = ((AtlasMapType) attrType).getValueType().getTypeCategory();
+
+                if (nonPrimitives.contains(mapValueType)) {
+                    addVertexProperty(attrTypeCategory.toString(), attr.getVertexPropertyName(), collectionProperties);
+                } else {
+                    addVertexProperty(attrTypeCategory.toString() + "_PRIMITIVE", attr.getVertexPropertyName(), collectionProperties);
+                }
+            }
+            break;
+        }
+    }
+
+    private static void addVertexProperty(String collectionType, String propertyName, Map<String, List<String>> collectionProperties) {
+        if(!collectionProperties.containsKey(collectionType)) {
+            collectionProperties.put(collectionType, new ArrayList<>());
+        }
+
+        collectionProperties.get(collectionType).add(propertyName);
+    }
+
+    static void displayInfo(String message, Map<String, Map<String, List<String>>> map) {
+        LOG.info(message);
+        for (Map.Entry<String, Map<String, List<String>>> e : map.entrySet()) {
+            LOG.info("  type: {} : {}", e.getKey(), e.getValue());
+        }
+    }
+}

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/postProcess/PostProcessListProperty.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration.postProcess;
+
+import org.apache.atlas.repository.Constants;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PostProcessListProperty {
+    private static final Logger LOG = LoggerFactory.getLogger(PostProcessListProperty.class);
+
+    public void process(Vertex v, String typeName, String propertyName) {
+        try {
+            if (doesNotHaveProperty(v, typeName) || !hasProperty(v, propertyName)) {
+                return;
+            }
+
+            removeProperty(v, propertyName);
+        } catch (IllegalArgumentException ex) {
+            LOG.error("process: IllegalArgumentException: v[{}] error!", v.id(), ex);
+        }
+    }
+
+    protected void removeProperty(Vertex v, String propertyName) {
+        v.property(propertyName).remove();
+    }
+
+    protected boolean doesNotHaveProperty(Vertex v, String typeName) {
+        return v.property(Constants.TYPENAME_PROPERTY_KEY).isPresent() || !isInstanceVertexOfType(v, typeName);
+    }
+
+    private boolean hasProperty(Vertex v, String propertyName) {
+        try {
+            return v.property(propertyName).isPresent();
+        } catch(Exception ex) {
+            // ...
+        }
+
+        return false;
+    }
+
+    private boolean isInstanceVertexOfType(Vertex v, String typeName) {
+        if(v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
+            String s = (String) v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
+
+            return s.equals(typeName);
+        }
+
+        return false;
+    }
+}

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java

@@ -26,14 +26,16 @@ import org.apache.tinkerpop.gremlin.structure.io.graphson.TypeInfo;
 import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
 import org.apache.tinkerpop.shaded.jackson.databind.JsonNode;
 import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper;
-import org.testng.ITestContext;
+import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import static org.testng.AssertJUnit.assertTrue;
 
@@ -41,12 +43,16 @@ public class BaseUtils {
     private static final String resourcesDirRelativePath = "/src/test/resources/";
     private String resourceDir;
 
-    protected final RelationshipTypeCache emptyRelationshipCache = new RelationshipTypeCache(new HashMap<>());
+    protected final ElementProcessors emptyRelationshipCache = new ElementProcessors(new HashMap<>(), new HashMap<>());
     protected GraphSONUtility graphSONUtility;
 
-    protected Object[][] getJsonNodeFromFile(String s) throws IOException {
+    protected JsonNode getJsonNodeFromFile(String s) {
         File f = new File(getFilePath(s));
-        return new Object[][]{{getEntityNode(FileUtils.readFileToString(f))}};
+        try {
+            return getEntityNode(FileUtils.readFileToString(f));
+        } catch (IOException e) {
+            throw new SkipException("getJsonNodeFromFile: " + s, e);
+        }
     }
 
     protected String getFilePath(String fileName) {
@@ -76,12 +82,18 @@ public class BaseUtils {
         utility.vertexFromJson(tg, node);
     }
 
-    protected void addEdge(TinkerGraph tg, MappedElementCache cache) throws IOException {
+    protected void addEdge(TinkerGraph tg, MappedElementCache cache) {
+        GraphSONUtility gu = graphSONUtility;
+
+        addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+        addEdgeToGraph(tg, gu, cache, getEdge(), getEdgeCol(), getEdgeCol2());
+    }
+
+    protected void addEdgesForMap(TinkerGraph tg, MappedElementCache cache) {
         GraphSONUtility gu = graphSONUtility;
 
-        gu.vertexFromJson(tg, (JsonNode) (getDBV(null)[0][0]));
-        gu.vertexFromJson(tg, (JsonNode) (getTableV(null))[0][0]);
-        gu.edgeFromJson(tg, cache, (JsonNode) getEdge(null)[0][0]);
+        addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+        addEdgeToGraph(tg, gu, cache, getEdgeCol3(), getEdgeCol4());
     }
 
     protected Vertex fetchTableVertex(TinkerGraph tg) {
@@ -91,29 +103,84 @@ public class BaseUtils {
         return (Vertex) query.next();
     }
 
-    @DataProvider(name = "col1")
-    public Object[][] getCol1(ITestContext context) throws IOException {
+    protected Map<String, Map<String, List<String>>> getTypePropertyMap(String type, String property, String category) {
+        Map<String, Map<String, List<String>>> map = new HashMap<>();
+        map.put(type, new HashMap<>());
+        map.get(type).put(category, new ArrayList<>());
+        map.get(type).get(category).add(property);
+        return map;
+    }
+
+    protected void addVertexToGraph(TinkerGraph tg, GraphSONUtility gu, JsonNode... nodes) {
+        for(JsonNode n : nodes) {
+            gu.vertexFromJson(tg, n);
+        }
+    }
+
+    protected void addEdgeToGraph(TinkerGraph tg, GraphSONUtility gu, MappedElementCache cache, JsonNode... nodes) {
+
+        for(JsonNode n : nodes) {
+            gu.edgeFromJson(tg, cache, n);
+        }
+    }
+
+    public JsonNode getCol1() {
         return getJsonNodeFromFile("col-legacy.json");
     }
 
-    @DataProvider(name = "dbType")
-    public Object[][] getDbType(ITestContext context) throws IOException {
+    public JsonNode getCol2() {
+        return getJsonNodeFromFile("col-2-legacy.json");
+    }
+
+    public JsonNode getCol3() {
+        return getJsonNodeFromFile("col-3-legacy.json");
+    }
+
+    public JsonNode getDbType() {
         return getJsonNodeFromFile("db-type-legacy.json");
     }
 
-    @DataProvider(name = "edge")
-    public Object[][] getEdge(ITestContext context) throws IOException {
+    public JsonNode getEdge() {
         return getJsonNodeFromFile("edge-legacy.json");
     }
 
-    @DataProvider(name = "dbV")
-    public Object[][] getDBV(ITestContext context) throws IOException {
-        return getJsonNodeFromFile("db-v-65544.json");
+    public JsonNode getEdgeCol() {
+        return getJsonNodeFromFile("edge-legacy-col.json");
     }
 
+    public JsonNode getEdgeCol2() {
+        return getJsonNodeFromFile("edge-legacy-col2.json");
+    }
 
-    @DataProvider(name = "tableV")
-    public Object[][] getTableV(ITestContext context) throws IOException {
+    public JsonNode getEdgeCol3() {
+        return getJsonNodeFromFile("edge-legacy-col3.json");
+    }
+
+    public JsonNode getEdgeCol4() {
+        return getJsonNodeFromFile("edge-legacy-col4.json");
+    }
+
+    public JsonNode getEdgeTag() {
+        return getJsonNodeFromFile("edge-legacy-tag.json");
+    }
+
+    public JsonNode getDBV() {
+        return getJsonNodeFromFile("db-v-65544.json");
+    }
+
+    public JsonNode getTableV() {
         return getJsonNodeFromFile("table-v-147504.json");
     }
+
+    public JsonNode getTagV() {
+        return getJsonNodeFromFile("tag-163856752.json");
+    }
+
+    public JsonNode getProcessV() {
+        return getJsonNodeFromFile("lineage-v-98312.json");
+    }
+
+    public JsonNode getEdgeProcess() {
+        return getJsonNodeFromFile("edge-legacy-process.json");
+    }
 }

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.graphdb.janus.migration;
-
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import org.eclipse.jetty.util.BlockingArrayQueue;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.AssertJUnit.assertTrue;
-
-public class GraphSONUtilityPostProcessTest extends BaseUtils {
-    final String HIVE_COLUMNS_PROPERTY = "hive_table.columns";
-    final String edgeId1 = "816u-35tc-ao0l-47so";
-    final String edgeId2 = "82rq-35tc-ao0l-2glc";
-
-    final String edgeId1x = "816u-35tc-ao0l-xxxx";
-    final String edgeId2x = "82rq-35tc-ao0l-xxxx";
-
-    private TinkerGraph tg;
-    private MappedElementCache cache = new MappedElementCache();
-    private Vertex tableV;
-
-    @Test
-    public void noRefNoUpdate() throws IOException {
-        tg = TinkerGraph.open();
-        graphSONUtility = new GraphSONUtility(emptyRelationshipCache);
-
-        addEdge(tg, cache);
-
-        tableV = fetchTableVertex(tg);
-        assertNotNull(tableV);
-
-        assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV);
-
-        graphSONUtility.replaceReferencedEdgeIdForList(tg, cache, tableV, HIVE_COLUMNS_PROPERTY);
-        assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV);
-    }
-
-    @Test(dependsOnMethods = "noRefNoUpdate")
-    public void refFoundVertexUpdated() throws IOException {
-
-        cache.lruEdgeCache.put(edgeId1, edgeId1x);
-        cache.lruEdgeCache.put(edgeId2, edgeId2x);
-
-        graphSONUtility.replaceReferencedEdgeIdForList(tg, cache, tableV, HIVE_COLUMNS_PROPERTY);
-        assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1x, edgeId2x, tableV);
-    }
-
-    @Test(dependsOnMethods = "refFoundVertexUpdated")
-    public void updateUsingPostProcessConsumer() throws IOException {
-        MappedElementCache cache = new MappedElementCache();
-        BlockingQueue<Object> bc = new BlockingArrayQueue<>();
-        PostProcessManager.Consumer consumer = new PostProcessManager.Consumer(bc, tg, graphSONUtility,
-                new String[] {HIVE_COLUMNS_PROPERTY}, cache, 5);
-
-        cache.lruEdgeCache.put(edgeId1x, edgeId1);
-        cache.lruEdgeCache.put(edgeId2x, edgeId2);
-        consumer.processItem(tableV.id());
-
-        assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV);
-    }
-
-    private void assertListProperty(String HIVE_COLUMNS_PROPERTY, String edgeId1, String edgeId2, Vertex tableV) {
-        assertTrue(tableV.property(HIVE_COLUMNS_PROPERTY).isPresent());
-        List list = (List) tableV.property(HIVE_COLUMNS_PROPERTY).value();
-
-        assertEquals(list.size(), 2);
-        assertEquals(list.get(0), edgeId1);
-        assertEquals(list.get(1), edgeId2);
-    }
-}

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java

@@ -18,36 +18,58 @@
 
 package org.apache.atlas.repository.graphdb.janus.migration;
 
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.commons.lang.StringUtils;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
 import org.apache.tinkerpop.shaded.jackson.databind.JsonNode;
 import org.testng.Assert;
-import org.testng.ITestContext;
-import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
 import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.VERTEX_ID_IN_IMPORT_KEY;
 import static org.testng.Assert.*;
 
 public class GraphSONUtilityTest extends BaseUtils {
 
-    @Test(dataProvider = "col1")
-    public void idFetch(JsonNode node) {
+    private final String HIVE_TABLE_NAME_PROPERTY = "Asset.name";
+    private final String HIVE_TABLE_NAME = "test_table_view";
+    private final String HIVE_TABLE_COLUMNS_RELATIONSHIP = "hive_table.columns";
+    private final String HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP = "hive_table.columnsMap";
+    private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP = "hive_table.parameters";
+    private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY = "transient_lastDdlTime";
+    private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP_VALUE_KEY = String.format("%s.%s", HIVE_TABLE_COLUMNS_PARAMETERS_MAP, HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY);
+    private final String HIVE_TABLE_TYPE = "hive_table";
+    
+    @Test
+    public void idFetch() {
+        JsonNode node = getCol1();
+        final int EXPECTED_ID = 98336;
         Object o = GraphSONUtility.getTypedValueFromJsonNode(node.get(GraphSONTokensTP2._ID));
 
         assertNotNull(o);
-        assertEquals((int) o, 98336);
+        assertEquals((int) o, EXPECTED_ID);
     }
 
-    @Test(dataProvider = "col1")
-    public void verifyReadProperties(JsonNode node) {
+    @Test
+    public void verifyReadProperties() {
+        JsonNode node = getCol1();
         Map<String, Object> props = GraphSONUtility.readProperties(node);
 
         assertEquals(props.get("__superTypeNames").getClass(), ArrayList.class);
@@ -58,8 +80,9 @@ public class GraphSONUtilityTest extends BaseUtils {
         assertNotNull(props);
     }
 
-    @Test(dataProvider = "col1")
-    public void dataNodeReadAndVertexAddedToGraph(JsonNode entityNode) throws IOException {
+    @Test
+    public void dataNodeReadAndVertexAddedToGraph() throws IOException {
+        JsonNode entityNode = getCol1();
         TinkerGraph tg = TinkerGraph.open();
         GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
         Map<String, Object> map = gu.vertexFromJson(tg, entityNode);
@@ -71,8 +94,9 @@ public class GraphSONUtilityTest extends BaseUtils {
         assertTrue(v.property(VERTEX_ID_IN_IMPORT_KEY).isPresent());
     }
 
-    @Test(dataProvider = "dbType")
-    public void typeNodeReadAndVertexNotAddedToGraph(JsonNode entityNode) throws IOException {
+    @Test
+    public void typeNodeReadAndVertexNotAddedToGraph() throws IOException {
+        JsonNode entityNode = getDbType();
         TinkerGraph tg = TinkerGraph.open();
         GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
         gu.vertexFromJson(tg, entityNode);
@@ -81,19 +105,46 @@ public class GraphSONUtilityTest extends BaseUtils {
     }
 
     @Test
-    public void edgeReadAndAddedToGraph() throws IOException {
+    public void updateNonPrimitiveArrayProperty() throws IOException {
+        ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getNonPrimitiveArray());
+
         TinkerGraph tg = TinkerGraph.open();
-        GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
-        Map<String, Object> m = null;
+        GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+        addVertexToGraph(tg, gu, getTableV());
+
+        Vertex v = tg.traversal().V().next();
+        assertNotNull(v);
+        assertTrue(v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).isPresent());
+
+        Map<String, String> list = (Map<String, String>) v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).value();
+        assertEquals(list.size(), 2);
+    }
 
-        m = gu.vertexFromJson(tg, (JsonNode) (getDBV(null)[0][0]));
-        assertNull(m);
 
-        m = gu.vertexFromJson(tg, (JsonNode) (getTableV(null))[0][0]);
-        assertNull(m);
+    @Test
+    public void updatePrimitiveMapProperty() {
+        ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMapPrimitive());
+
+        TinkerGraph tg = TinkerGraph.open();
+        GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+        addVertexToGraph(tg, gu, getTableV());
 
-        m = gu.edgeFromJson(tg, new MappedElementCache(), (JsonNode) getEdge(null)[0][0]);
-        assertNull(m);
+        Vertex v = tg.traversal().V().next();
+        assertNotNull(v);
+        assertTrue(v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).isPresent());
+        assertEquals(((Map) v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).value()).size(), 1);
+        assertEquals(((Map) v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).value()).get(HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY), "1522693834");
+        assertFalse(v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP_VALUE_KEY).isPresent());
+    }
+
+    @Test
+    public void edgeReadAndAddedToGraph() {
+        TinkerGraph tg = TinkerGraph.open();
+        GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
+        Map<String, Object> m = null;
+
+        addVertexToGraph(tg, gu, getDBV(), getTableV());
+        addEdgeToGraph(tg, gu, new MappedElementCache(), getEdge());
 
         Assert.assertEquals((long) tg.traversal().V().count().next(), 2L);
         Assert.assertEquals((long) tg.traversal().E().count().next(), 1L);
@@ -101,4 +152,179 @@ public class GraphSONUtilityTest extends BaseUtils {
         Edge e = tg.edges().next();
         assertTrue(e.property(EDGE_ID_IN_IMPORT_KEY).isPresent());
     }
+
+    @Test
+    public void edgeReadAndArrayIndexAdded() throws IOException {
+        ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+        TinkerGraph tg = TinkerGraph.open();
+        GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+        Map<String, Object> m = null;
+
+        addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+        addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeCol(), getEdgeCol2());
+
+        Iterator<Edge> edges = tg.edges();
+        while(edges.hasNext()) {
+            Edge e = edges.next();
+            String arraySpecificLabel = StringUtils.remove(e.label(), Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+            if(arraySpecificLabel.equals(HIVE_TABLE_COLUMNS_RELATIONSHIP)) {
+                assertTrue(e.property(ATTRIBUTE_INDEX_PROPERTY_KEY).isPresent());
+            }
+            assertTrue(e.property(EDGE_ID_IN_IMPORT_KEY).isPresent());
+        }
+
+        Iterator<Vertex> vertices = tg.vertices();
+        while(vertices.hasNext()) {
+            Vertex v = vertices.next();
+            if(v.property(HIVE_TABLE_NAME_PROPERTY).isPresent()) {
+                if(v.property(HIVE_TABLE_NAME_PROPERTY).value().toString().equals(HIVE_TABLE_NAME)) {
+                    assertTrue(v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).isPresent());
+                }
+            }
+        }
+    }
+
+    @Test
+    public void nonPrimitiveMap_Removed() throws IOException {
+        Set<String> actualKeys = new HashSet<String>() {{
+            add("col3");
+            add("col4");
+        }};
+
+        ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+        TinkerGraph tg = TinkerGraph.open();
+        GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+        addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+        addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeCol3(), getEdgeCol4());
+
+        Iterator<Vertex> vertices = tg.vertices();
+        while(vertices.hasNext()) {
+            Vertex v = vertices.next();
+            if(!v.property(HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP).isPresent()) continue;
+
+            fail("Non-primitive map should be removed during vertex creation.");
+        }
+
+        Iterator<Edge> edges = tg.edges();
+        while(edges.hasNext()) {
+            Edge e = edges.next();
+            String mapSpecificLabel = StringUtils.remove(e.label(), Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+            assertEquals(mapSpecificLabel, HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP);
+            assertTrue(e.property(ATTRIBUTE_KEY_PROPERTY_KEY).isPresent());
+
+            assertTrue(actualKeys.contains((String) e.property(ATTRIBUTE_KEY_PROPERTY_KEY).value()));
+        }
+    }
+
+    @Test
+    public void tagAssociated_NewAttributesAdded() throws IOException {
+
+        ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+        TinkerGraph tg = TinkerGraph.open();
+        GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+        addVertexToGraph(tg, gu, getTagV(), getDBV(), getTableV(), getCol3());
+        addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
+
+        Iterator<Vertex> vertices = tg.vertices();
+        while(vertices.hasNext()) {
+            Vertex v = vertices.next();
+            if(v.id().toString() != "16752") continue;
+
+            assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
+            assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
+            assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
+        }
+
+        Iterator<Edge> edges = tg.edges();
+        while(edges.hasNext()) {
+            Edge e = edges.next();
+            assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+            assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
+            assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+        }
+    }
+
+    @Test
+    public void processEdge_PropagateSetTo_NONE() throws IOException {
+        ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+        TinkerGraph tg = TinkerGraph.open();
+        GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+        addVertexToGraph(tg, gu, getTagV(), getDBV(), getTableV(), getCol3());
+        addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
+
+        Iterator<Vertex> vertices = tg.vertices();
+        while(vertices.hasNext()) {
+            Vertex v = vertices.next();
+            if(v.id().toString() != "16752") continue;
+
+            assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
+            assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
+            assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
+        }
+
+        Iterator<Edge> edges = tg.edges();
+        while(edges.hasNext()) {
+            Edge e = edges.next();
+            assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+            assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
+            assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+        }
+    }
+
+    @Test
+    public void processEdge_PropagateSetTo_ONE_TO_TWO() throws IOException {
+        Map<String, RelationshipCacheGenerator.TypeInfo> typeCache = new HashMap<String, RelationshipCacheGenerator.TypeInfo>() {{
+            put("__Process.inputs", new RelationshipCacheGenerator.TypeInfo("dataset_process_inputs", AtlasRelationshipDef.PropagateTags.TWO_TO_ONE));
+        }};
+
+        ElementProcessors elementProcessors = new ElementProcessors(typeCache, getPostProcessMap());
+        TinkerGraph tg = TinkerGraph.open();
+        GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+        addVertexToGraph(tg, gu, getDBV(), getTableV(), getProcessV());
+        addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeProcess());
+
+        Iterator<Edge> edges = tg.edges();
+        while(edges.hasNext()) {
+            Edge e = edges.next();
+            assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+            assertEquals(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).value().toString(), "TWO_TO_ONE");
+            assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
+            assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+        }
+    }
+
+    private Map<String, Map<String, List<String>>> getPostProcessMap() {
+        Map<String, Map<String, List<String>>> map = new HashMap<>();
+        map.put(HIVE_TABLE_TYPE, new HashMap<>());
+
+        map.get(HIVE_TABLE_TYPE).put("ARRAY", new ArrayList<>());
+        map.get(HIVE_TABLE_TYPE).put("MAP", new ArrayList<>());
+
+        map.get(HIVE_TABLE_TYPE).get("ARRAY").add(HIVE_TABLE_COLUMNS_RELATIONSHIP);
+        map.get(HIVE_TABLE_TYPE).get("MAP").add(HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP);
+
+        return map;
+    }
+
+    private Map<String, Map<String, List<String>>> getPostProcessMapPrimitive() {
+        Map<String, Map<String, List<String>>> map = new HashMap<>();
+        map.put(HIVE_TABLE_TYPE, new HashMap<>());
+        map.get(HIVE_TABLE_TYPE).put("MAP_PRIMITIVE", new ArrayList<>());
+        map.get(HIVE_TABLE_TYPE).get("MAP_PRIMITIVE").add(HIVE_TABLE_COLUMNS_PARAMETERS_MAP);
+
+        return map;
+    }
+
+    private Map<String, Map<String, List<String>>> getNonPrimitiveArray() {
+        Map<String, Map<String, List<String>>> map = new HashMap<>();
+        map.put(HIVE_TABLE_TYPE, new HashMap<>());
+        map.get(HIVE_TABLE_TYPE).put("ARRAY", new ArrayList<>());
+        map.get(HIVE_TABLE_TYPE).get("ARRAY").add(HIVE_TABLE_COLUMNS_RELATIONSHIP);
+
+        return map;
+    }
 }

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java

@@ -36,8 +36,9 @@ import static org.testng.AssertJUnit.assertNotNull;
 
 public class JsonNodeParsersTest extends BaseUtils {
 
-    @Test(dataProvider = "col1")
-    public void parseVertex(JsonNode nd) {
+    @Test
+    public void parseVertex() {
+        JsonNode nd = getCol1();
         final int COL1_ORIGINAL_ID = 98336;
 
         Object nodeId = getId(nd);
@@ -68,8 +69,9 @@ public class JsonNodeParsersTest extends BaseUtils {
         assertProperties(vUsingPe);
     }
 
-    @Test(dataProvider = "edge")
-    public void parseEdge(JsonNode nd) throws IOException {
+    @Test
+    public void parseEdge() {
+        JsonNode nd = getEdge();
         final String EDGE_ORIGINAL_ID = "8k5i-35tc-acyd-1eko";
         Object nodeId = getId(nd);
 
@@ -78,12 +80,12 @@ public class JsonNodeParsersTest extends BaseUtils {
         JsonNodeParsers.ParseElement peVertex = new JsonNodeParsers.ParseVertex();
         peVertex.setContext(graphSONUtility);
 
-        peVertex.parse(tg, cache, (JsonNode) (getDBV(null)[0][0]));
-        peVertex.parse(tg, cache, (JsonNode) (getTableV(null)[0][0]));
+        peVertex.parse(tg, cache, getDBV());
+        peVertex.parse(tg, cache, getTableV());
 
         JsonNodeParsers.ParseElement pe = new JsonNodeParsers.ParseEdge();
         pe.setContext(graphSONUtility);
-        pe.parse(tg, cache, (JsonNode) getEdge(null)[0][0]);
+        pe.parse(tg, cache, getEdge());
 
         updateParseElement(tg, pe, nodeId);
 

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java

@@ -31,8 +31,9 @@ import static org.testng.Assert.*;
 
 public class MappedElementCacheTest extends BaseUtils {
 
-    @Test(dataProvider = "col1")
-    public void vertexFetch(JsonNode node) {
+    @Test
+    public void vertexFetch()  {
+        JsonNode node = getCol1();
         MappedElementCache cache = new MappedElementCache();
         TinkerGraph tg = TinkerGraph.open();
 
@@ -41,32 +42,15 @@ public class MappedElementCacheTest extends BaseUtils {
         Vertex vx = cache.getMappedVertex(tg, 98336);
         assertNotNull(vx);
         assertEquals(cache.lruVertexCache.size(), 1);
-        assertEquals(cache.lruEdgeCache.size(), 0);
     }
 
     @Test
-    public void edgeFetch() throws IOException {
+    public void edgeFetch() {
         MappedElementCache cache = new MappedElementCache();
         TinkerGraph tg = TinkerGraph.open();
 
         addEdge(tg, cache);
 
-        assertEquals(cache.lruVertexCache.size(), 2);
-        assertEquals(cache.lruEdgeCache.size(), 0);
-    }
-
-
-    @Test
-    public void nonExistentVertexReturnsNull() {
-        TinkerGraph tg = TinkerGraph.open();
-        MappedElementCache cache = new MappedElementCache();
-
-        assertNull(cache.fetchVertex(tg, 1111));
-        assertNull(cache.fetchEdge(tg, "abcd"));
-    }
-
-    @DataProvider(name = "col1")
-    public Object[][] getCol1(ITestContext context) throws IOException {
-        return getJsonNodeFromFile("col-legacy.json");
+        assertEquals(cache.lruVertexCache.size(), 4);
     }
 }

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration;
+
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.janus.migration.postProcess.PostProcessListProperty;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.eclipse.jetty.util.BlockingArrayQueue;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+public class PostProcessListPropertyTest extends BaseUtils {
+    final String HIVE_TABLE_TYPE = "hive_table";
+    final String HIVE_COLUMNS_PROPERTY = "hive_table.columns";
+
+    final String col1EdgeId = "816u-35tc-ao0l-47so";
+    final String col2EdgeId = "82rq-35tc-ao0l-2glc";
+
+    @Test
+    public void noRefNoUpdate() throws IOException {
+        TestSetup ts = new TestSetup();
+
+        ts.getPostProcessListProperty().process(ts.getTable(), HIVE_TABLE_TYPE, HIVE_COLUMNS_PROPERTY);
+        ts.assertIncomplete();
+    }
+
+    @Test
+    public void refFoundVertexUpdated() throws IOException {
+        TestSetup ts = new TestSetup();
+        assertNotNull(ts.getTable());
+
+        ts.getPostProcessListProperty().process(ts.getTable(), HIVE_TABLE_TYPE, HIVE_COLUMNS_PROPERTY);
+        ts.assertComplete();
+    }
+
+    @Test
+    public void updateUsingPostProcessConsumer() throws IOException {
+        TestSetup ts = new TestSetup();
+
+        BlockingQueue<Object> bc = new BlockingArrayQueue<>();
+        PostProcessManager.Consumer consumer = new PostProcessManager.Consumer(bc, ts.getGraph(),
+                getTypePropertyMap("hive_table", HIVE_COLUMNS_PROPERTY, "ARRAY"), 5);
+
+        Vertex tableV = fetchTableVertex(ts.getGraph());
+        consumer.processItem(tableV.id());
+        ts.assertComplete();
+    }
+
+    private class TestSetup {
+        private PostProcessListProperty postProcessListProperty;
+        private TinkerGraph tg;
+        private MappedElementCache cache;
+        private Vertex tableV;
+
+        public PostProcessListProperty getPostProcessListProperty() {
+            return postProcessListProperty;
+        }
+
+        public TinkerGraph getGraph() {
+            return tg;
+        }
+
+        public MappedElementCache getCache() {
+            return cache;
+        }
+
+        public Vertex getTable() {
+            return tableV;
+        }
+
+        public TestSetup() throws IOException {
+            postProcessListProperty = new PostProcessListProperty();
+            tg = TinkerGraph.open();
+            cache = new MappedElementCache();
+            addEdge(tg, cache);
+            tableV = fetchTableVertex(tg);
+
+            assertSetup();
+        }
+
+        public void assertSetup() {
+            assertTrue(tableV.property(HIVE_COLUMNS_PROPERTY).isPresent());
+            List list = (List) tableV.property(HIVE_COLUMNS_PROPERTY).value();
+
+            assertEquals(list.size(), 2);
+            assertEquals(list.get(0), col1EdgeId);
+            assertEquals(list.get(1), col2EdgeId);
+        }
+
+        private void assertIncomplete() {
+            assertPropertyRemoved(HIVE_COLUMNS_PROPERTY, tableV);
+
+            Iterator<Edge> edges = tableV.edges(Direction.OUT, getEdgeLabel(HIVE_COLUMNS_PROPERTY));
+            while (edges.hasNext()) {
+                Edge e = edges.next();
+
+                assertFalse(e.property(ATTRIBUTE_INDEX_PROPERTY_KEY).isPresent());
+            }
+        }
+
+        private void assertComplete() {
+            assertPropertyRemoved(HIVE_COLUMNS_PROPERTY, tableV);
+        }
+
+        private void assertPropertyRemoved(String property, Vertex tableV) {
+            assertFalse(tableV.property(property).isPresent());
+        }
+
+        public String getEdgeLabel(String property ) {
+            return Constants.INTERNAL_PROPERTY_KEY_PREFIX + property;
+        }
+    }
+}

graphdb/janus/src/test/resources/col-2-legacy.json

+{
+  "Asset.name": {
+    "type": "string",
+    "value": "col2"
+  },
+  "hive_column.type": {
+    "type": "string",
+    "value": "string"
+  },
+  "__modifiedBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "entityText": {
+    "type": "string",
+    "value": "hive_column owner anonymous qualifiedName stocks.test_table_view.col2@cl1 name col2 position 1 type string table "
+  },
+  "Referenceable.qualifiedName": {
+    "type": "string",
+    "value": "stocks.test_table_view.col2@cl1"
+  },
+  "__guid": {
+    "type": "string",
+    "value": "9cef2494-766c-4671-96a8-828dce677e7e"
+  },
+  "__version": {
+    "type": "integer",
+    "value": 0
+  },
+  "__superTypeNames": {
+    "type": "list",
+    "value": [{
+      "type": "string",
+      "value": "Asset"
+    }, {
+      "type": "string",
+      "value": "DataSet"
+    }, {
+      "type": "string",
+      "value": "Referenceable"
+    }]
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__typeName": {
+    "type": "string",
+    "value": "hive_column"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522693838471
+  },
+  "Asset.owner": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "hive_column.position": {
+    "type": "integer",
+    "value": 1
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522693835017
+  },
+  "_id": 114816,
+  "_type": "vertex"
+}

graphdb/janus/src/test/resources/col-3-legacy.json

+{
+  "Asset.name": {
+    "type": "string",
+    "value": "open"
+  },
+  "hive_column.type": {
+    "type": "string",
+    "value": "string"
+  },
+  "__modifiedBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "entityText": {
+    "type": "string",
+    "value": "hive_column owner anonymous qualifiedName stocks.stocks_daily.open@cl1 name open position 1 type string table  Tag1 "
+  },
+  "Referenceable.qualifiedName": {
+    "type": "string",
+    "value": "stocks.stocks_daily.open@cl1"
+  },
+  "__traitNames": {
+    "type": "list",
+    "value": [
+      {
+        "type": "string",
+        "value": "Tag1"
+      }
+    ]
+  },
+  "__guid": {
+    "type": "string",
+    "value": "8231a95f-d062-4685-81aa-0b62401bc796"
+  },
+  "__version": {
+    "type": "integer",
+    "value": 0
+  },
+  "__superTypeNames": {
+    "type": "list",
+    "value": [
+      {
+        "type": "string",
+        "value": "Asset"
+      },
+      {
+        "type": "string",
+        "value": "DataSet"
+      },
+      {
+        "type": "string",
+        "value": "Referenceable"
+      }
+    ]
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__typeName": {
+    "type": "string",
+    "value": "hive_column"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522693815850
+  },
+  "Asset.owner": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "hive_column.position": {
+    "type": "integer",
+    "value": 1
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522693815850
+  },
+  "_id": 16752,
+  "_type": "vertex"
+}

graphdb/janus/src/test/resources/edge-legacy-col.json

+{
+  "__modifiedBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522693835017
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522693835017
+  },
+  "_id": "816u-35tc-ao0l-47so",
+  "_type": "edge",
+  "_outV": 147504,
+  "_inV": 98336,
+  "_label": "__hive_table.columns"
+}
\ No newline at end of file

graphdb/janus/src/test/resources/edge-legacy-col2.json

+{
+  "__modifiedBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522693835017
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522693835017
+  },
+  "_id": "82rq-35tc-ao0l-2glc",
+  "_type": "edge",
+  "_outV": 147504,
+  "_inV": 114816,
+  "_label": "__hive_table.columns"
+}

graphdb/janus/src/test/resources/edge-legacy-col3.json

+{
+  "__modifiedBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522693835017
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522693835017
+  },
+  "_id": "816u-35tc-ao0l-47aa",
+  "_type": "edge",
+  "_outV": 147504,
+  "_inV": 98336,
+  "_label": "__hive_table.columnsMap.col3"
+}

graphdb/janus/src/test/resources/edge-legacy-col4.json

+{
+  "__modifiedBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522693835017
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522693835017
+  },
+  "_id": "82rq-35tc-ao0l-2gaa",
+  "_type": "edge",
+  "_outV": 147504,
+  "_inV": 114816,
+  "_label": "__hive_table.columnsMap.col4"
+}

graphdb/janus/src/test/resources/edge-legacy-process.json

+{
+  "__modifiedBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522693837285
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522693837285
+  },
+  "_id": "6jgh-23uw-2uqd-1elc",
+  "_type": "edge",
+  "_outV": 98312,
+  "_inV": 147504,
+  "_label": "__Process.inputs"
+}

graphdb/janus/src/test/resources/edge-legacy-tag.json

+{
+  "__modifiedBy": {
+    "type": "string",
+    "value": "admin"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "admin"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522694252176
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522694252176
+  },
+  "_id": "2pk3la-cxc-m61h-2pk0og",
+  "_type": "edge",
+  "_outV": 16752,
+  "_inV": 163856752,
+  "_label": "Tag1"
+}
\ No newline at end of file

graphdb/janus/src/test/resources/lineage-v-98312.json

+{
+  "hive_column_lineage.depenendencyType": {
+    "type": "string",
+    "value": "SIMPLE"
+  },
+  "Asset.name": {
+    "type": "string",
+    "value": "create view test_table_view partitioned on (col4) as select col1, col2, col4 from test_table:col1"
+  },
+  "Process.inputs": {
+    "type": "list",
+    "value": [{
+      "type": "string",
+      "value": "6jgh-23uw-2uqd-1elc"
+    }]
+  },
+  "Process.outputs": {
+    "type": "list",
+    "value": [{
+      "type": "string",
+      "value": "6ept-23uw-2wb9-47so"
+    }]
+  },
+  "__modifiedBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "entityText": {
+    "type": "string",
+    "value": "hive_column_lineage outputs qualifiedName stocks.test_table_view@cl1:1522693834000:col1 inputs query name create view test_table_view partitioned on (col4) as select col1, col2, col4 from test_table:col1 depenendencyType SIMPLE "
+  },
+  "Referenceable.qualifiedName": {
+    "type": "string",
+    "value": "stocks.test_table_view@cl1:1522693834000:col1"
+  },
+  "__guid": {
+    "type": "string",
+    "value": "9336b7a4-9cc0-4ef0-8dc1-01fce2def6a5"
+  },
+  "__version": {
+    "type": "integer",
+    "value": 0
+  },
+  "__superTypeNames": {
+    "type": "list",
+    "value": [{
+      "type": "string",
+      "value": "Asset"
+    }, {
+      "type": "string",
+      "value": "Process"
+    }, {
+      "type": "string",
+      "value": "Referenceable"
+    }]
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "anonymous"
+  },
+  "__typeName": {
+    "type": "string",
+    "value": "hive_column_lineage"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522693837285
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522693837285
+  },
+  "_id": 98312,
+  "_type": "vertex"
+}

graphdb/janus/src/test/resources/table-v-147504.json

@@ -33,10 +33,12 @@
   },
   "hive_table.parameters": {
     "type": "list",
-    "value": [{
+    "value": [
+      {
         "type": "string",
         "value": "transient_lastDdlTime"
-    }]
+      }
+    ]
   },
   "hive_table.retention": {
     "type": "integer",
@@ -44,10 +46,12 @@
   },
   "hive_table.partitionKeys": {
     "type": "list",
-    "value": [{
+    "value": [
+      {
         "type": "string",
         "value": "8dty-35tc-amfp-23xs"
-    }]
+      }
+    ]
   },
   "__guid": {
     "type": "string",
@@ -63,16 +67,20 @@
   },
   "__superTypeNames": {
     "type": "list",
-    "value": [{
+    "value": [
+      {
         "type": "string",
         "value": "Asset"
-    }, {
+      },
+      {
         "type": "string",
         "value": "DataSet"
-    }, {
+      },
+      {
         "type": "string",
         "value": "Referenceable"
-    }]
+      }
+    ]
   },
   "hive_table.viewExpandedText": {
     "type": "string",
@@ -104,13 +112,37 @@
   },
   "hive_table.columns": {
     "type": "list",
-    "value": [{
+    "value": [
+      {
         "type": "string",
         "value": "816u-35tc-ao0l-47so"
-    }, {
+      },
+      {
         "type": "string",
         "value": "82rq-35tc-ao0l-2glc"
-    }]
+      }
+    ]
+  },
+  "hive_table.columnsMap.col3": {
+    "type": "string",
+    "value": "816u-35tc-ao0l-47aa"
+  },
+  "hive_table.columnsMap.col4": {
+    "type": "string",
+    "value": "82rq-35tc-ao0l-2gaa"
+  },
+  "hive_table.columnsMap": {
+    "type": "list",
+    "value": [
+      {
+        "type": "string",
+        "value": "col3"
+      },
+      {
+        "type": "string",
+        "value": "col4"
+      }
+    ]
   },
   "__timestamp": {
     "type": "long",
@@ -119,3 +151,4 @@
   "_id": 147504,
   "_type": "vertex"
 }
+

graphdb/janus/src/test/resources/tag-163856752.json

+{
+  "__superTypeNames": {
+    "type": "list",
+    "value": [{
+      "type": "string",
+      "value": "[]"
+    }]
+  },
+  "__modifiedBy": {
+    "type": "string",
+    "value": "admin"
+  },
+  "__state": {
+    "type": "string",
+    "value": "ACTIVE"
+  },
+  "__createdBy": {
+    "type": "string",
+    "value": "admin"
+  },
+  "__typeName": {
+    "type": "string",
+    "value": "Tag1"
+  },
+  "__modificationTimestamp": {
+    "type": "long",
+    "value": 1522694252176
+  },
+  "__timestamp": {
+    "type": "long",
+    "value": 1522694252176
+  },
+  "_id": 163856752,
+  "_type": "vertex"
+}

intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java

@@ -21,16 +21,12 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.SearchFilter;
 import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
 import org.apache.atlas.model.typedef.AtlasClassificationDef;
-import org.apache.atlas.model.typedef.AtlasClassificationDef.AtlasClassificationDefs;
 import org.apache.atlas.model.typedef.AtlasEntityDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef;
 import org.apache.atlas.model.typedef.AtlasRelationshipDef;
 import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
 
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
 
 /**
  * Interface to persistence store of TypeDef
@@ -107,6 +103,4 @@ public interface AtlasTypeDefStore {
     AtlasBaseTypeDef getByName(String name) throws AtlasBaseException;
 
     AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException;
-
-    void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException;
 }

intg/src/test/java/org/apache/atlas/TestUtilsV2.java

@@ -18,7 +18,6 @@
 
 package org.apache.atlas;
 
-import com.sun.org.apache.xpath.internal.operations.Bool;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;

repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java

@@ -18,11 +18,11 @@
 
 package org.apache.atlas.repository.migration;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.impexp.AtlasImportResult;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
 import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.impexp.ImportTypeDefProcessor;
 import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
 import org.apache.atlas.store.AtlasTypeDefStore;
@@ -56,11 +56,11 @@ public class DataMigrationService implements Service {
     private final Thread        thread;
 
     @Inject
-    public DataMigrationService(AtlasTypeDefStore typeDefStore, Configuration configuration,
+    public DataMigrationService(AtlasGraph graph, AtlasTypeDefStore typeDefStore, Configuration configuration,
                                 GraphBackedSearchIndexer indexer, AtlasTypeDefStoreInitializer storeInitializer,
                                 AtlasTypeRegistry typeRegistry) {
         this.configuration = configuration;
-        this.thread        = new Thread(new FileImporter(typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer));
+        this.thread        = new Thread(new FileImporter(graph, typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer));
     }
 
     @Override
@@ -83,15 +83,17 @@ public class DataMigrationService implements Service {
     }
 
     public static class FileImporter implements Runnable {
+        private final AtlasGraph                   graph;
         private final AtlasTypeDefStore            typeDefStore;
         private final String                       importDirectory;
         private final GraphBackedSearchIndexer     indexer;
         private final AtlasTypeRegistry            typeRegistry;
         private final AtlasTypeDefStoreInitializer storeInitializer;
 
-        public FileImporter(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
+        public FileImporter(AtlasGraph graph, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
                             AtlasTypeDefStoreInitializer storeInitializer,
                             String directoryName, GraphBackedSearchIndexer indexer) {
+            this.graph            = graph;
             this.typeDefStore     = typeDefStore;
             this.typeRegistry     = typeRegistry;
             this.storeInitializer = storeInitializer;
@@ -99,7 +101,16 @@ public class DataMigrationService implements Service {
             this.indexer          = indexer;
         }
 
-        public void performImport() throws AtlasBaseException {
+        @Override
+        public void run() {
+            try {
+                performImport();
+            } catch (AtlasBaseException e) {
+                LOG.error("Data Migration:", e);
+            }
+        }
+
+        private void performImport() throws AtlasBaseException {
             try {
                 if(!performAccessChecks(importDirectory)) {
                     return;
@@ -109,7 +120,7 @@ public class DataMigrationService implements Service {
 
                 FileInputStream fs = new FileInputStream(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_DATA_NAME));
 
-                typeDefStore.loadLegacyData(RelationshipCacheGenerator.get(typeRegistry), fs);
+                graph.importLegacyGraphSON(typeRegistry, fs);
             } catch (Exception ex) {
                 LOG.error("Import failed!", ex);
                 throw new AtlasBaseException(ex);
@@ -117,11 +128,13 @@ public class DataMigrationService implements Service {
         }
 
         private boolean performAccessChecks(String path) {
-            boolean ret = false;
+            final boolean ret;
+
             if(StringUtils.isEmpty(path)) {
                 ret = false;
             } else {
                 File f = new File(path);
+
                 ret = f.exists() && f.isDirectory() && f.canRead();
             }
 
@@ -137,17 +150,19 @@ public class DataMigrationService implements Service {
         private void performInit() throws AtlasBaseException, AtlasException {
             indexer.instanceIsActive();
             storeInitializer.instanceIsActive();
+
             processIncomingTypesDef(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_TYPESDEF_NAME));
         }
 
-        @VisibleForTesting
-        void processIncomingTypesDef(File typesDefFile) throws AtlasBaseException {
+        private void processIncomingTypesDef(File typesDefFile) throws AtlasBaseException {
             try {
                 AtlasImportResult      result    = new AtlasImportResult();
                 String                 jsonStr   = FileUtils.readFileToString(typesDefFile);
                 AtlasTypesDef          typesDef  = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
                 ImportTypeDefProcessor processor = new ImportTypeDefProcessor(typeDefStore, typeRegistry);
+
                 processor.processTypes(typesDef, result);
+
                 LOG.info("  types migrated: {}", result.getMetrics());
             } catch (IOException e) {
                 LOG.error("processIncomingTypesDef: Could not process file: {}! Imported data may not be usable.", typesDefFile.getName());
@@ -157,14 +172,5 @@ public class DataMigrationService implements Service {
         private File getFileFromImportDirectory(String importDirectory, String fileName) {
             return Paths.get(importDirectory, fileName).toFile();
         }
-
-        @Override
-        public void run() {
-            try {
-                performImport();
-            } catch (AtlasBaseException e) {
-                LOG.error("Data Migration:", e);
-            }
-        }
     }
 }

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java

@@ -24,7 +24,6 @@ import static org.apache.atlas.repository.Constants.TYPE_CATEGORY_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY;
 import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.VERTEX_TYPE;
 
-import java.io.InputStream;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -49,7 +48,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
-import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
@@ -110,11 +108,6 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
         LOG.info("<== AtlasTypeDefGraphStoreV1.init()");
     }
 
-    @Override
-    public void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
-        getAtlasGraph().loadLegacyGraphSON(relationshipCache, fs);
-    }
-
     AtlasGraph getAtlasGraph() { return atlasGraph; }
 
     @VisibleForTesting

repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class ComplexAttributesTest extends MigrationBaseAsserts {
+
+    @Inject
+    public ComplexAttributesTest(AtlasGraph graph) {
+        super(graph);
+    }
+
+    @Test
+    public void verify() throws IOException, AtlasBaseException {
+        String STRUCT_TYPE = "struct_type";
+        String ENTITY_TYPE = "entity_type";
+        String ENTITY_WITH_COMPLEX_COLL_TYPE = "entity_with_complex_collection_attr";
+
+        final int EXPECTED_TOTAL_COUNT  = 214;
+        final int EXPECTED_ENTITY_TYPE_COUNT = 16;
+        final int EXPECTED_STRUCT_TYPE_COUNT = 3;
+        final int EXPECTED_ENTITY_WITH_COMPLEX_COLL_TYPE_COUNT = 1;
+
+        runFileImporter("complex-attr_db");
+
+        assertTypeCountNameGuid(STRUCT_TYPE, EXPECTED_STRUCT_TYPE_COUNT,"", "");
+        assertTypeCountNameGuid(ENTITY_TYPE, EXPECTED_ENTITY_TYPE_COUNT, "", "");
+        assertTypeCountNameGuid(ENTITY_WITH_COMPLEX_COLL_TYPE, EXPECTED_ENTITY_WITH_COMPLEX_COLL_TYPE_COUNT, "", "");
+
+        assertEdgesWithLabel(getVertex(ENTITY_WITH_COMPLEX_COLL_TYPE, "").getEdges(AtlasEdgeDirection.OUT).iterator(),1, "__entity_with_complex_collection_attr.listOfEntities");
+        assertEdgesWithLabel(getVertex(ENTITY_WITH_COMPLEX_COLL_TYPE, "").getEdges(AtlasEdgeDirection.OUT).iterator(),9, "__entity_with_complex_collection_attr.mapOfStructs");
+
+        assertMigrationStatus(EXPECTED_TOTAL_COUNT);
+    }
+}

repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java

@@ -21,17 +21,13 @@ package org.apache.atlas.repository.migration;
 import com.google.inject.Inject;
 import org.apache.atlas.TestModules;
 import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.repository.graph.GraphHelper;
 import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
 import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
 import java.io.IOException;
-import java.util.List;
 
-import static org.testng.Assert.assertEquals;
 
 @Guice(modules = TestModules.TestOnlyModule.class)
 public class HiveParititionTest extends  MigrationBaseAsserts {
@@ -41,7 +37,7 @@ public class HiveParititionTest extends  MigrationBaseAsserts {
         super(graph);
     }
 
-    @Test(enabled = false)
+    @Test
     public void fileImporterTest() throws IOException, AtlasBaseException {
         final int EXPECTED_TOTAL_COUNT = 141;
         final int EXPECTED_DB_COUNT = 1;
@@ -50,8 +46,6 @@ public class HiveParititionTest extends  MigrationBaseAsserts {
 
         runFileImporter("parts_db");
 
-        assertPartitionKeyProperty(getVertex("hive_table", "t1"), 1);
-        assertPartitionKeyProperty(getVertex("hive_table", "tv1"), 1);
         assertHiveVertices(EXPECTED_DB_COUNT, EXPECTED_TABLE_COUNT, EXPECTED_COLUMN_COUNT);
 
         assertTypeCountNameGuid("hive_db", 1, "parts_db", "ae30d78b-51b4-42ab-9436-8d60c8f68b95");
@@ -62,9 +56,4 @@ public class HiveParititionTest extends  MigrationBaseAsserts {
 
         assertMigrationStatus(EXPECTED_TOTAL_COUNT);
     }
-
-    private void assertPartitionKeyProperty(AtlasVertex vertex, int expectedCount) {
-        List<String> keys = GraphHelper.getListProperty(vertex, "hive_table.partitionKeys");
-        assertEquals(keys.size(), expectedCount);
-    }
 }

repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java

@@ -35,7 +35,7 @@ public class HiveStocksTest extends MigrationBaseAsserts {
         super(graph);
     }
 
-    @Test(enabled = false)
+    @Test
     public void migrateStocks() throws AtlasBaseException, IOException {
         final int EXPECTED_TOTAL_COUNT  = 188;
         final int EXPECTED_DB_COUNT     = 1;
@@ -47,7 +47,6 @@ public class HiveStocksTest extends MigrationBaseAsserts {
         assertHiveVertices(EXPECTED_DB_COUNT, EXPECTED_TABLE_COUNT, EXPECTED_COLUMN_COUNT);
         assertTypeCountNameGuid("hive_db", 1, "stocks", "4e13b36b-9c54-4616-9001-1058221165d0");
         assertTypeCountNameGuid("hive_table", 1, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774");
-        assertTypeAttribute("hive_table", 7, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774", "hive_table.columns");
         assertTypeCountNameGuid("hive_column", 1, "high", "d72ce4fb-6f17-4e68-aa85-967366c9e891");
         assertTypeCountNameGuid("hive_column", 1, "open", "788ba8fe-b7d8-41ba-84ef-c929732924ec");
         assertTypeCountNameGuid("hive_column", 1, "dt", "643a0a71-0d97-477d-a43b-7ca433f85160");

repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java

@@ -34,7 +34,6 @@ import org.testng.annotations.AfterClass;
 
 import java.io.IOException;
 import java.util.Iterator;
-import java.util.List;
 
 import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
 import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
@@ -48,10 +47,10 @@ public class MigrationBaseAsserts {
     private final String R_GUID_PROPERTY_NAME = "_r__guid";
 
     @Inject
-    private AtlasTypeDefStore typeDefStore;
+    protected AtlasTypeDefStore typeDefStore;
 
     @Inject
-    private AtlasTypeRegistry typeRegistry;
+    protected AtlasTypeRegistry typeRegistry;
 
     @Inject
     private AtlasTypeDefStoreInitializer storeInitializer;
@@ -74,7 +73,7 @@ public class MigrationBaseAsserts {
         }
     }
 
-    private void loadTypesFromJson() throws IOException, AtlasBaseException {
+    protected void loadTypesFromJson() throws IOException, AtlasBaseException {
         loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
         loadModelFromJson("1000-Hadoop/1020-fs_model.json", typeDefStore, typeRegistry);
         loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
@@ -83,7 +82,7 @@ public class MigrationBaseAsserts {
     protected void runFileImporter(String directoryToImport) throws IOException, AtlasBaseException {
         loadTypesFromJson();
         String directoryName = TestResourceFileUtils.getDirectory(directoryToImport);
-        DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(typeDefStore, typeRegistry,
+        DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(graph, typeDefStore, typeRegistry,
                 storeInitializer, directoryName, indexer);
 
         fi.run();
@@ -152,14 +151,19 @@ public class MigrationBaseAsserts {
         assertEquals(count, expectedItems, String.format("%s", edgeTypeName));
     }
 
-    protected void assertTypeAttribute(String typeName, int expectedSize, String name, String guid, String propertyName) {
-        AtlasVertex v         = getVertex(typeName, name);
-        String     guidActual = GraphHelper.getGuid(v);
-        List list       = (List) GraphHelper.getProperty(v, propertyName);
+    protected void assertEdgesWithLabel(Iterator<AtlasEdge> results, int startIdx, String edgeTypeName) {
+        int count = 0;
+        AtlasEdge e = null;
+        for (Iterator<AtlasEdge> it = results; it.hasNext() && count < startIdx; count++) {
+            e = it.next();
+        }
 
-        assertEquals(guidActual, guid);
-        assertNotNull(list);
-        assertEquals(list.size(), expectedSize);
+        assertNotNull(GraphHelper.getProperty(e, R_GUID_PROPERTY_NAME));
+        assertNotNull(GraphHelper.getProperty(e, "tagPropagation"));
+
+        if(StringUtils.isNotEmpty(edgeTypeName)) {
+            assertEquals(e.getLabel(), edgeTypeName, edgeTypeName);
+        }
     }
 
     protected void assertTypeCountNameGuid(String typeName, int expectedItems, String name, String guid) {

repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java → repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java

@@ -23,29 +23,27 @@ import org.apache.atlas.TestModules;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.typedef.AtlasRelationshipDef;
 import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils;
-import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.graphdb.janus.migration.RelationshipCacheGenerator;
 import org.apache.atlas.store.AtlasTypeDefStore;
 import org.apache.atlas.type.AtlasRelationshipType;
 import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.v1.typesystem.types.utils.TypesUtil;
 import org.apache.commons.lang.StringUtils;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import org.jcodings.util.Hash;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Map;
 
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
 import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
 import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertTrue;
 
 @Guice(modules = TestModules.TestOnlyModule.class)
-public class RelationshipMappingTest {
+public class RelationshipCacheGeneratorTest {
 
     @Inject
     private AtlasTypeDefStore typeDefStore;
@@ -61,12 +59,23 @@ public class RelationshipMappingTest {
 
     @Test
     public void createLookup() {
-        Map<String, String> cache = RelationshipCacheGenerator.get(typeRegistry);
+        final String PROCESS_INPUT_KEY = "__Process.inputs";
+        final String PROCESS_OUTPUT_KEY = "__Process.outputs";
+        String ONE_TO_TWO_STR = ONE_TO_TWO.toString();
+        String TWO_TO_ONE_STR = TWO_TO_ONE.toString();
+
+        Map<String, RelationshipCacheGenerator.TypeInfo> cache = RelationshipCacheGenerator.get(typeRegistry);
         assertEquals(cache.size(), getLegacyAttributeCount() - 1);
-        for (Map.Entry<String, String> entry : cache.entrySet()) {
+        for (Map.Entry<String, RelationshipCacheGenerator.TypeInfo> entry : cache.entrySet()) {
             assertTrue(StringUtils.isNotEmpty(entry.getKey()));
             assertTrue(entry.getKey().startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX), entry.getKey());
         }
+
+        assertEquals(cache.get(PROCESS_INPUT_KEY).getTypeName(), "dataset_process_inputs");
+        assertEquals(cache.get(PROCESS_INPUT_KEY).getPropagateTags(), ONE_TO_TWO_STR);
+
+        assertEquals(cache.get(PROCESS_OUTPUT_KEY).getTypeName(), "process_dataset_outputs");
+        assertEquals(cache.get(PROCESS_OUTPUT_KEY).getPropagateTags(), TWO_TO_ONE_STR);
     }
 
     private int getLegacyAttributeCount() {

repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.janus.migration.TypesWithCollectionsFinder;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts {
+    @Inject
+    protected TypesWithCollectionsFinderTest(AtlasGraph graph) {
+        super(graph);
+    }
+
+    @BeforeClass
+    public void setup() throws IOException, AtlasBaseException {
+        loadTypesFromJson();
+    }
+
+    @Test
+    public void fetchAll() {
+        Map<String, Map<String, List<String>>> typeAttrMap = TypesWithCollectionsFinder.getVertexPropertiesForCollectionAttributes(typeRegistry);
+
+        assertEquals(typeAttrMap.size(), 9);
+
+        assertProperties(typeAttrMap, "__AtlasUserProfile", "ARRAY", "__AtlasUserProfile.savedSearches");
+
+        assertProperties(typeAttrMap, "Process", "ARRAY", "Process.inputs");
+        assertProperties(typeAttrMap, "Process", "ARRAY", "Process.outputs");
+
+        assertProperties(typeAttrMap, "hdfs_path", "MAP_PRIMITIVE", "hdfs_path.extendedAttributes");
+
+        assertProperties(typeAttrMap, "hive_column_lineage", "ARRAY", "Process.inputs");
+        assertProperties(typeAttrMap, "hive_column_lineage", "ARRAY", "Process.outputs");
+
+        assertProperties(typeAttrMap, "hive_db", "MAP_PRIMITIVE", "hive_db.parameters");
+
+        assertProperties(typeAttrMap, "hive_process", "ARRAY", "Process.inputs");
+        assertProperties(typeAttrMap, "hive_process", "ARRAY", "Process.outputs");
+
+        assertProperties(typeAttrMap, "hive_storagedesc", "ARRAY", "hive_storagedesc.sortCols");
+
+        assertProperties(typeAttrMap, "hive_serde", "MAP_PRIMITIVE", "hive_serde.parameters");
+
+        assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.partitionKeys");
+        assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.columns");
+        assertProperties(typeAttrMap, "hive_table", "MAP_PRIMITIVE", "hive_table.parameters");
+    }
+
+    private void assertProperties(Map<String, Map<String, List<String>>> typeAttrMap, String typeName, String typeCategory, String propertyName) {
+        List<String> actualProperties = typeAttrMap.get(typeName).get(typeCategory);
+
+        assertTrue(actualProperties.contains(propertyName));
+    }
+}

repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java

@@ -35,7 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
 import org.apache.atlas.repository.graph.AtlasGraphProvider;
 import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
-import org.apache.atlas.repository.impexp.ExportService;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
 import org.apache.atlas.repository.store.graph.AtlasEntityStore;
 import org.apache.atlas.runner.LocalSolrRunner;
@@ -82,6 +82,9 @@ public class AtlasEntityTestBase {
     @Inject
     private EntityGraphMapper graphMapper;
 
+    @Inject
+    protected AtlasGraph graph;
+
     AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
 
     @BeforeClass

repository/src/test/resources/complex-attr_db/atlas-migration-typesdef.json

+{
+  "enumDefs": [
+    {
+      "category": "ENUM",
+      "guid": "460e6d99-b06c-48d3-8bb2-9c40ca1235d0",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656721280,
+      "updateTime": 1523656721280,
+      "version": 1,
+      "name": "hive_principal_type",
+      "description": "hive_principal_type",
+      "typeVersion": "1.0",
+      "elementDefs": [
+        {
+          "value": "USER",
+          "ordinal": 1
+        },
+        {
+          "value": "ROLE",
+          "ordinal": 2
+        },
+        {
+          "value": "GROUP",
+          "ordinal": 3
+        }
+      ]
+    },
+    {
+      "category": "ENUM",
+      "guid": "df00ab81-0791-4dbb-bd56-e4183e039111",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656718218,
+      "updateTime": 1523656718218,
+      "version": 1,
+      "name": "file_action",
+      "description": "file_action",
+      "typeVersion": "1.0",
+      "elementDefs": [
+        {
+          "value": "NONE",
+          "ordinal": 0
+        },
+        {
+          "value": "EXECUTE",
+          "ordinal": 1
+        },
+        {
+          "value": "WRITE",
+          "ordinal": 2
+        },
+        {
+          "value": "WRITE_EXECUTE",
+          "ordinal": 3
+        },
+        {
+          "value": "READ",
+          "ordinal": 4
+        },
+        {
+          "value": "READ_EXECUTE",
+          "ordinal": 5
+        },
+        {
+          "value": "READ_WRITE",
+          "ordinal": 6
+        },
+        {
+          "value": "ALL",
+          "ordinal": 7
+        }
+      ]
+    }
+  ],
+  "structDefs": [
+    {
+      "category": "STRUCT",
+      "guid": "bd0ba5b3-525e-4d35-9527-7ffa10b4e5bc",
+      "createdBy": "admin",
+      "updatedBy": "admin",
+      "createTime": 1526494657148,
+      "updateTime": 1526494657148,
+      "version": 1,
+      "name": "struct_type",
+      "description": "struct_type_description",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        }
+      ]
+    },
+    {
+      "category": "STRUCT",
+      "guid": "30993e57-4d69-46ab-b531-02371eace3d9",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656721304,
+      "updateTime": 1523656721304,
+      "version": 1,
+      "name": "hive_order",
+      "description": "hive_order",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "order",
+          "typeName": "int",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "col",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ]
+    },
+    {
+      "category": "STRUCT",
+      "guid": "db96593c-2453-4795-966f-edaa05e69cde",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656722372,
+      "updateTime": 1523656722372,
+      "version": 1,
+      "name": "hive_serde",
+      "description": "hive_serde",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "serializationLib",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "parameters",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ]
+    },
+    {
+      "category": "STRUCT",
+      "guid": "9d847c57-5940-4ff9-a9d1-ee36365b9cb2",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656718322,
+      "updateTime": 1523656718322,
+      "version": 1,
+      "name": "fs_permissions",
+      "description": "fs_permissions",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "group",
+          "typeName": "file_action",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "user",
+          "typeName": "file_action",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "sticky",
+          "typeName": "boolean",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "others",
+          "typeName": "file_action",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        }
+      ]
+    }
+  ],
+  "classificationDefs": [
+    {
+      "category": "CLASSIFICATION",
+      "guid": "5af3e7e8-68a6-4cf6-b88e-4c619731f3e2",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656711303,
+      "updateTime": 1523656711303,
+      "version": 1,
+      "name": "TaxonomyTerm",
+      "description": "TaxonomyTerm",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "atlas.taxonomy",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [],
+      "subTypes": []
+    }
+  ],
+  "entityDefs": [
+    {
+      "category": "ENTITY",
+      "guid": "861f6201-6bbd-4a94-ac04-6b4982c4f94e",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656725295,
+      "updateTime": 1523656725295,
+      "version": 1,
+      "name": "falcon_feed_replication",
+      "description": "falcon_feed_replication",
+      "typeVersion": "1.0",
+      "attributeDefs": [],
+      "superTypes": [
+        "Process"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "0a5f7d23-c5bd-4ff0-84df-0c0df226e47e",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656725387,
+      "updateTime": 1523656725387,
+      "version": 1,
+      "name": "falcon_process",
+      "description": "falcon_process",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "frequency",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "runs-on",
+          "typeName": "falcon_cluster",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "tags",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "pipelines",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "workflow-properties",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "Process"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "38c97017-cb6c-4f64-9610-6930d04c7645",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656712591,
+      "updateTime": 1523656712591,
+      "version": 1,
+      "name": "DataSet",
+      "description": "DataSet",
+      "typeVersion": "1.0",
+      "attributeDefs": [],
+      "superTypes": [
+        "Referenceable",
+        "Asset"
+      ],
+      "subTypes": [
+        "hbase_column",
+        "jms_topic",
+        "falcon_feed",
+        "hbase_table",
+        "hbase_column_family",
+        "kafka_topic",
+        "hive_table",
+        "hive_column",
+        "sqoop_dbdatastore",
+        "hbase_namespace",
+        "fs_path"
+      ]
+    },
+    {
+      "category": "ENTITY",
+      "guid": "c289abd1-5065-41f3-92dd-265f4297d933",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656725408,
+      "updateTime": 1523656725408,
+      "version": 1,
+      "name": "falcon_feed_creation",
+      "description": "falcon_feed_creation",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "stored-in",
+          "typeName": "falcon_cluster",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        }
+      ],
+      "superTypes": [
+        "Process"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "3da46128-6e98-4bce-b3d1-cbc4bf9b9055",
+      "createdBy": "admin",
+      "updatedBy": "admin",
+      "createTime": 1526494658830,
+      "updateTime": 1526494658830,
+      "version": 1,
+      "name": "entity_with_complex_collection_attr",
+      "description": "entity_with_complex_collection_attr_description",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": true,
+          "isIndexable": true
+        },
+        {
+          "name": "listOfStructs",
+          "typeName": "array<struct_type>",
+          "isOptional": true,
+          "cardinality": "LIST",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 2147483647,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "listOfEntities",
+          "typeName": "array<entity_type>",
+          "isOptional": true,
+          "cardinality": "LIST",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 2147483647,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ]
+        },
+        {
+          "name": "mapOfStructs",
+          "typeName": "map<string,struct_type>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "mapOfEntities",
+          "typeName": "map<string,entity_type>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ]
+        }
+      ],
+      "superTypes": [],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "e273b345-4c6b-4272-8b2c-f2936eb1941b",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656713377,
+      "updateTime": 1523656713377,
+      "version": 1,
+      "name": "Process",
+      "description": "Process",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "inputs",
+          "typeName": "array<DataSet>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "outputs",
+          "typeName": "array<DataSet>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "Referenceable",
+        "Asset"
+      ],
+      "subTypes": [
+        "falcon_feed_replication",
+        "falcon_process",
+        "falcon_feed_creation",
+        "sqoop_process",
+        "storm_topology",
+        "hive_column_lineage",
+        "hive_process"
+      ]
+    },
+    {
+      "category": "ENTITY",
+      "guid": "dfab14ca-1978-4157-8a9d-3c4e1d608830",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656722469,
+      "updateTime": 1523656731237,
+      "version": 2,
+      "name": "hive_table",
+      "description": "hive_table",
+      "typeVersion": "1.1",
+      "options": {
+        "schemaElementsAttribute": "columns"
+      },
+      "attributeDefs": [
+        {
+          "name": "db",
+          "typeName": "hive_db",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "createTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "lastAccessTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "comment",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "retention",
+          "typeName": "int",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "sd",
+          "typeName": "hive_storagedesc",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ]
+        },
+        {
+          "name": "partitionKeys",
+          "typeName": "array<hive_column>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ]
+        },
+        {
+          "name": "aliases",
+          "typeName": "array<string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "columns",
+          "typeName": "array<hive_column>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ]
+        },
+        {
+          "name": "parameters",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "viewOriginalText",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "viewExpandedText",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "tableType",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "temporary",
+          "typeName": "boolean",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "69331a04-420b-4610-9892-398e18e1fa5f",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656714170,
+      "updateTime": 1523656714170,
+      "version": 1,
+      "name": "__AtlasUserProfile",
+      "description": "__AtlasUserProfile",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": true,
+          "isIndexable": true
+        },
+        {
+          "name": "fullName",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "savedSearches",
+          "typeName": "array<__AtlasUserSavedSearch>",
+          "isOptional": true,
+          "cardinality": "LIST",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 2147483647,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ]
+        }
+      ],
+      "superTypes": [
+        "__internal"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "5e1898f6-4a62-40b2-a9c8-3401d5936b61",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656722615,
+      "updateTime": 1523656722615,
+      "version": 1,
+      "name": "hive_db",
+      "description": "hive_db",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "clusterName",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "location",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "parameters",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "ownerType",
+          "typeName": "hive_principal_type",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "Referenceable",
+        "Asset"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "1a219261-1258-492b-a145-a1ed96ac6024",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656724387,
+      "updateTime": 1523656724387,
+      "version": 1,
+      "name": "sqoop_dbdatastore",
+      "description": "sqoop_dbdatastore",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "dbStoreType",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "storeUse",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "storeUri",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "source",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "ff0890ec-c80d-4a35-bd7c-b0145cb714a9",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656726191,
+      "updateTime": 1523656726191,
+      "version": 1,
+      "name": "hbase_namespace",
+      "description": "hbase_namespace",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "clusterName",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "parameters",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "createTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "modifiedTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "a382861a-9dd2-4310-897e-20f8666198b4",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656722390,
+      "updateTime": 1523656722390,
+      "version": 1,
+      "name": "hive_process",
+      "description": "hive_process",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "startTime",
+          "typeName": "date",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "endTime",
+          "typeName": "date",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "userName",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "operationType",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "queryText",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "queryPlan",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "queryId",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "recentQueries",
+          "typeName": "array<string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "clusterName",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "queryGraph",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "Process"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "a2456728-4a88-43f6-8d64-d875802db437",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656727979,
+      "updateTime": 1523656727979,
+      "version": 1,
+      "name": "storm_node",
+      "description": "storm_node",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "description",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "driverClass",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "conf",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [],
+      "subTypes": [
+        "storm_bolt",
+        "storm_spout"
+      ]
+    },
+    {
+      "category": "ENTITY",
+      "guid": "b44eaa9e-3727-4b0c-aad1-c0e59eea4d73",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656726233,
+      "updateTime": 1523656726233,
+      "version": 1,
+      "name": "hbase_column",
+      "description": "hbase_column",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "column_family",
+          "typeName": "hbase_column_family",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "inverseRef",
+              "params": {
+                "attribute": "columns"
+              }
+            }
+          ]
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "f4b80d9b-e72c-419f-ab40-9a3256a541b9",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656711776,
+      "updateTime": 1523656711776,
+      "version": 1,
+      "name": "Referenceable",
+      "description": "Referenceable",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "qualifiedName",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": true,
+          "isIndexable": true
+        }
+      ],
+      "superTypes": [],
+      "subTypes": [
+        "hive_storagedesc",
+        "DataSet",
+        "Infrastructure",
+        "Process",
+        "hive_db"
+      ]
+    },
+    {
+      "category": "ENTITY",
+      "guid": "6303fe7d-466d-4750-aef5-fb9cacbdef55",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656727081,
+      "updateTime": 1523656727081,
+      "version": 1,
+      "name": "jms_topic",
+      "description": "jms_topic",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "topic",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "uri",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "e82bc222-1bd2-429b-a73d-6a4b45d1e104",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656726210,
+      "updateTime": 1523656728900,
+      "version": 2,
+      "name": "hbase_table",
+      "description": "hbase_table",
+      "typeVersion": "1.2",
+      "attributeDefs": [
+        {
+          "name": "uri",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "column_families",
+          "typeName": "array<hbase_column_family>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ]
+        },
+        {
+          "name": "namespace",
+          "typeName": "hbase_namespace",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "parameters",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "createTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "modifiedTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "fce68d4b-42d3-4579-bc0d-0465f18788fd",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656725316,
+      "updateTime": 1523656725316,
+      "version": 1,
+      "name": "falcon_feed",
+      "description": "falcon_feed",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "frequency",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "stored-in",
+          "typeName": "falcon_cluster",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "groups",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "tags",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "86313734-80ae-469f-b4c2-f505e1d68846",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656727932,
+      "updateTime": 1523656727932,
+      "version": 1,
+      "name": "storm_topology",
+      "description": "storm_topology",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "id",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": true,
+          "isIndexable": true
+        },
+        {
+          "name": "startTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "endTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "conf",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "clusterName",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "nodes",
+          "typeName": "array<storm_node>",
+          "isOptional": false,
+          "cardinality": "LIST",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 2147483647,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "Process"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "887f7f1f-c16a-44af-a399-b81a30502f52",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656713347,
+      "updateTime": 1523656713347,
+      "version": 1,
+      "name": "Infrastructure",
+      "description": "Infrastructure",
+      "typeVersion": "1.0",
+      "attributeDefs": [],
+      "superTypes": [
+        "Referenceable",
+        "Asset"
+      ],
+      "subTypes": [
+        "falcon_cluster"
+      ]
+    },
+    {
+      "category": "ENTITY",
+      "guid": "1ac62577-4ec0-4932-8913-67a4f51cfaa7",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656727994,
+      "updateTime": 1523656727994,
+      "version": 1,
+      "name": "storm_spout",
+      "description": "storm_spout",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "outputs",
+          "typeName": "array<string>",
+          "isOptional": false,
+          "cardinality": "LIST",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 2147483647,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "storm_node"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "55673891-cb91-4169-8e77-65d402943ce2",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656726223,
+      "updateTime": 1523656729584,
+      "version": 2,
+      "name": "hbase_column_family",
+      "description": "hbase_column_family",
+      "typeVersion": "1.1",
+      "attributeDefs": [
+        {
+          "name": "table",
+          "typeName": "hbase_table",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "inverseRef",
+              "params": {
+                "attribute": "column_families"
+              }
+            }
+          ]
+        },
+        {
+          "name": "columns",
+          "typeName": "array<hbase_column>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "ownedRef"
+            }
+          ]
+        },
+        {
+          "name": "createTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "modifiedTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "4dffd142-fd58-4103-80ef-6a773b25fcb2",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656712492,
+      "updateTime": 1523656712492,
+      "version": 1,
+      "name": "Asset",
+      "description": "Asset",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "description",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "owner",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        }
+      ],
+      "superTypes": [],
+      "subTypes": [
+        "DataSet",
+        "Infrastructure",
+        "Process",
+        "hive_db"
+      ]
+    },
+    {
+      "category": "ENTITY",
+      "guid": "210330a4-4777-4fa4-a3a0-a455e8d2a959",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656722634,
+      "updateTime": 1523656731493,
+      "version": 4,
+      "name": "hive_column",
+      "description": "hive_column",
+      "typeVersion": "1.3",
+      "options": {
+        "schemaAttributes": "[\"name\", \"description\", \"owner\", \"type\", \"comment\", \"position\"]"
+      },
+      "attributeDefs": [
+        {
+          "name": "type",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "comment",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "position",
+          "typeName": "int",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "table",
+          "typeName": "hive_table",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "679ffbf8-db6d-4fad-95fb-f0e86561e1a6",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656714878,
+      "updateTime": 1523656714878,
+      "version": 1,
+      "name": "__AtlasUserSavedSearch",
+      "description": "__AtlasUserSavedSearch",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "ownerName",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "uniqueName",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": true,
+          "isIndexable": true
+        },
+        {
+          "name": "searchType",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "searchParameters",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "__internal"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "35619cb1-5e95-43a0-b51d-b8f228290f1c",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656727066,
+      "updateTime": 1523656727066,
+      "version": 1,
+      "name": "kafka_topic",
+      "description": "kafka_topic",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "topic",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "uri",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "e974af16-6278-4fee-83f5-90603a58ae83",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656722572,
+      "updateTime": 1523656722572,
+      "version": 1,
+      "name": "hive_storagedesc",
+      "description": "hive_storagedesc",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "table",
+          "typeName": "hive_table",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false,
+          "constraints": [
+            {
+              "type": "inverseRef",
+              "params": {
+                "attribute": "sd"
+              }
+            }
+          ]
+        },
+        {
+          "name": "location",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "inputFormat",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "outputFormat",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "compressed",
+          "typeName": "boolean",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "numBuckets",
+          "typeName": "int",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "serdeInfo",
+          "typeName": "hive_serde",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "bucketCols",
+          "typeName": "array<string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "sortCols",
+          "typeName": "array<hive_order>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "parameters",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "storedAsSubDirectories",
+          "typeName": "boolean",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "Referenceable"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "cc19bd6e-fe24-4499-ac74-ae40ca934eef",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656718524,
+      "updateTime": 1523656718524,
+      "version": 1,
+      "name": "hdfs_path",
+      "description": "hdfs_path",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "clusterName",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "numberOfReplicas",
+          "typeName": "int",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "extendedAttributes",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "fs_path"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "e37409c4-6e2d-4ee7-b87a-f1dc32bb8aee",
+      "createdBy": "admin",
+      "updatedBy": "admin",
+      "createTime": 1526494658119,
+      "updateTime": 1526494658119,
+      "version": 1,
+      "name": "entity_type",
+      "description": "entity_type_description",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "name",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": true,
+          "isIndexable": true
+        },
+        {
+          "name": "isReplicated",
+          "typeName": "boolean",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "3eb69f70-1569-41a9-b827-c16a77946b41",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656724359,
+      "updateTime": 1523656724359,
+      "version": 1,
+      "name": "sqoop_process",
+      "description": "sqoop_process",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "operation",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "commandlineOpts",
+          "typeName": "map<string,string>",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "startTime",
+          "typeName": "date",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "endTime",
+          "typeName": "date",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "userName",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        }
+      ],
+      "superTypes": [
+        "Process"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "8d0bf90b-0a3c-4e66-ba32-5349732c0967",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656722705,
+      "updateTime": 1523656722705,
+      "version": 1,
+      "name": "hive_column_lineage",
+      "description": "hive_column_lineage",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "query",
+          "typeName": "hive_process",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "depenendencyType",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "expression",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "Process"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "25e22509-b6fd-42ed-b5bc-8e1726f37103",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656711806,
+      "updateTime": 1523656711806,
+      "version": 1,
+      "name": "__internal",
+      "description": "__internal",
+      "typeVersion": "1.0",
+      "attributeDefs": [],
+      "superTypes": [],
+      "subTypes": [
+        "__AtlasUserSavedSearch",
+        "__AtlasUserProfile"
+      ]
+    },
+    {
+      "category": "ENTITY",
+      "guid": "d1e4da7c-83ea-48f1-b098-1f3201c4d58e",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656728002,
+      "updateTime": 1523656728002,
+      "version": 1,
+      "name": "storm_bolt",
+      "description": "storm_bolt",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "inputs",
+          "typeName": "array<string>",
+          "isOptional": false,
+          "cardinality": "LIST",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 2147483647,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "outputs",
+          "typeName": "array<string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "storm_node"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "f252485f-992a-4588-9b54-f96cf98ef3d2",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656725304,
+      "updateTime": 1523656725304,
+      "version": 1,
+      "name": "falcon_cluster",
+      "description": "falcon_cluster",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "colo",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "tags",
+          "typeName": "map<string,string>",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        }
+      ],
+      "superTypes": [
+        "Infrastructure"
+      ],
+      "subTypes": []
+    },
+    {
+      "category": "ENTITY",
+      "guid": "c24bf9da-6c38-4f69-bf72-9017a7a32707",
+      "createdBy": "root",
+      "updatedBy": "root",
+      "createTime": 1523656718424,
+      "updateTime": 1523656718424,
+      "version": 1,
+      "name": "fs_path",
+      "description": "fs_path",
+      "typeVersion": "1.0",
+      "attributeDefs": [
+        {
+          "name": "path",
+          "typeName": "string",
+          "isOptional": false,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 1,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "createTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "modifiedTime",
+          "typeName": "date",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "isFile",
+          "typeName": "boolean",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "isSymlink",
+          "typeName": "boolean",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "fileSize",
+          "typeName": "long",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": false
+        },
+        {
+          "name": "group",
+          "typeName": "string",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        },
+        {
+          "name": "posixPermissions",
+          "typeName": "fs_permissions",
+          "isOptional": true,
+          "cardinality": "SINGLE",
+          "valuesMinCount": 0,
+          "valuesMaxCount": 1,
+          "isUnique": false,
+          "isIndexable": true
+        }
+      ],
+      "superTypes": [
+        "DataSet"
+      ],
+      "subTypes": [
+        "hdfs_path"
+      ]
+    }
+  ]
+}
\ No newline at end of file