ISSUE-38 Map type to graph with type prefixes to enable search. Contributed by Venkatesh Seetharam

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphUtils.java

@@ -23,10 +23,14 @@ import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.Graph;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
+import org.apache.hadoop.metadata.ITypedInstance;
+import org.apache.hadoop.metadata.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.storage.Id;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.Iterator;
+import java.util.UUID;
 
 /**
  * Utility class for graph operations.
@@ -38,40 +42,26 @@ public final class GraphUtils {
     private GraphUtils() {
     }
 
-    public static Edge addEdge(Vertex fromVertex, Vertex toVertex,
-                               String vertexPropertyKey, String edgeLabel) {
-        return addEdge(fromVertex, toVertex, vertexPropertyKey, edgeLabel, null);
+    public static Vertex createVertex(Graph graph,
+                                      ITypedReferenceableInstance typedInstance) {
+        return createVertex(graph, typedInstance, typedInstance.getId());
     }
 
-    public static Edge addEdge(Vertex fromVertex, Vertex toVertex,
-                               String vertexPropertyKey, String edgeLabel, String timestamp) {
-        Edge edge = findEdge(fromVertex, toVertex, vertexPropertyKey, edgeLabel);
+    public static Vertex createVertex(Graph graph,
+                                      ITypedInstance typedInstance,
+                                      Id typedInstanceId) {
+        final Vertex instanceVertex = graph.addVertex(null);
+        // type
+        instanceVertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typedInstance.getTypeName());
 
-        Edge edgeToVertex = edge != null ? edge : fromVertex.addEdge(edgeLabel, toVertex);
-        if (timestamp != null) {
-            edgeToVertex.setProperty(Constants.TIMESTAMP_PROPERTY_KEY, timestamp);
-        }
-
-        return edgeToVertex;
-    }
+        // id
+        final String guid = UUID.randomUUID().toString();
+        instanceVertex.setProperty(Constants.GUID_PROPERTY_KEY, guid);
 
-    public static Edge findEdge(Vertex fromVertex, Vertex toVertex,
-                                String vertexPropertyKey, String edgeLabel) {
-        return findEdge(fromVertex, toVertex.getProperty(vertexPropertyKey),
-                vertexPropertyKey, edgeLabel);
-    }
-
-    public static Edge findEdge(Vertex fromVertex, Object toVertexName,
-                                String vertexPropertyKey, String edgeLabel) {
-        Edge edgeToFind = null;
-        for (Edge edge : fromVertex.getEdges(Direction.OUT, edgeLabel)) {
-            if (edge.getVertex(Direction.IN).getProperty(vertexPropertyKey).equals(toVertexName)) {
-                edgeToFind = edge;
-                break;
-            }
-        }
+        // version
+        instanceVertex.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
 
-        return edgeToFind;
+        return instanceVertex;
     }
 
     public static Vertex findVertex(Graph blueprintsGraph,
@@ -97,23 +87,23 @@ public final class GraphUtils {
 
     public static String edgeString(final Edge edge) {
         return "e[" + edge.getLabel() + "], ["
-                + edge.getVertex(Direction.OUT).getProperty("name")
+                + edge.getVertex(Direction.OUT)
                 + " -> " + edge.getLabel() + " -> "
-                + edge.getVertex(Direction.IN).getProperty("name")
+                + edge.getVertex(Direction.IN)
                 + "]";
     }
 
     public static void dumpToLog(final Graph graph) {
+        LOG.debug("*******************Graph Dump****************************");
         LOG.debug("Vertices of {}", graph);
         for (Vertex vertex : graph.getVertices()) {
             LOG.debug(vertexString(vertex));
-            System.out.println(vertexString(vertex));
         }
 
         LOG.debug("Edges of {}", graph);
         for (Edge edge : graph.getEdges()) {
             LOG.debug(edgeString(edge));
-            System.out.println(edgeString(edge));
         }
+        LOG.debug("*******************Graph Dump****************************");
     }
 }
\ No newline at end of file

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java

@@ -34,15 +34,16 @@ import org.apache.hadoop.metadata.types.Multiplicity;
 import org.apache.hadoop.metadata.types.StructTypeDefinition;
 import org.apache.hadoop.metadata.types.TraitType;
 import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
+import scala.actors.threadpool.Arrays;
 
 import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.UUID;
 
 /**
@@ -50,28 +51,29 @@ import java.util.UUID;
  */
 public class EntityJerseyResourceIT extends BaseResourceIT {
 
+    private static final Logger LOG = LoggerFactory.getLogger(EntityJerseyResourceIT.class);
+
     private static final String DATABASE_TYPE = "hive_database";
     private static final String DATABASE_NAME = "foo";
     private static final String TABLE_TYPE = "hive_table";
     private static final String TABLE_NAME = "bar";
-    private static final String TRAIT_TYPE = "hive_fetl";
 
-    private String tableInstanceAsJSON;
+    private ITypedReferenceableInstance tableInstance;
     private String guid;
 
     @BeforeClass
     public void setUp() throws Exception {
         super.setUp();
 
-        List<HierarchicalTypeDefinition> typeDefinitions = createHiveTypes();
-        submitTypes(typeDefinitions);
+        createHiveTypes();
+        submitTypes();
     }
 
     @Test
     public void testSubmitEntity() throws Exception {
-        ITypedReferenceableInstance tableInstance = createHiveTableInstance();
-
-        tableInstanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
+        tableInstance = createHiveTableInstance();
+        String tableInstanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
+        LOG.debug("tableInstance = " + tableInstanceAsJSON);
 
         WebResource resource = service
                 .path("api/metadata/entities/submit")
@@ -119,12 +121,24 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
 
         final String definition = response.getString("definition");
         Assert.assertNotNull(definition);
+        LOG.debug("tableInstanceAfterGet = " + definition);
+
+        // todo - this fails with type error, strange
+        // ITypedReferenceableInstance tableInstanceAfterGet = Serialization$.MODULE$.fromJson(definition);
+        // Assert.assertTrue(areEqual(tableInstance, tableInstanceAfterGet));
+    }
 
-        System.out.println("definition = " + definition);
-        System.out.println("tableInstanceAsJSON = " + tableInstanceAsJSON);
-        // Assert.assertEquals(definition, tableInstanceAsJSON);
+/*
+    private boolean areEqual(ITypedReferenceableInstance actual,
+                             ITypedReferenceableInstance expected) throws Exception {
+        for (AttributeInfo attributeInfo : actual.fieldMapping().fields.values()) {
+            Assert.assertEquals(actual.get(attributeInfo.name), expected.get(attributeInfo.name));
         }
 
+        return true;
+    }
+*/
+
     @Test
     public void testGetInvalidEntityDefinition() {
         WebResource resource = service
@@ -165,15 +179,19 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         System.out.println("response = " + response);
     }
 
-    private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception {
-        ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
-
+    private void createHiveTypes() throws Exception {
         HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
                 createClassTypeDef(DATABASE_TYPE,
                         ImmutableList.<String>of(),
                         createRequiredAttrDef("name", DataTypes.STRING_TYPE),
                         createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-        typeDefinitions.add(databaseTypeDefinition);
+
+        StructTypeDefinition structTypeDefinition =
+                new StructTypeDefinition("serdeType",
+                        new AttributeDefinition[] {
+                        createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
+                        });
 
         HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
                 createClassTypeDef(TABLE_TYPE,
@@ -181,32 +199,31 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
                         createRequiredAttrDef("name", DataTypes.STRING_TYPE),
                         createRequiredAttrDef("description", DataTypes.STRING_TYPE),
                         createRequiredAttrDef("type", DataTypes.STRING_TYPE),
-                        new AttributeDefinition(DATABASE_TYPE,
-                                DATABASE_TYPE, Multiplicity.REQUIRED, true, DATABASE_TYPE));
-        typeDefinitions.add(tableTypeDefinition);
-
-        HierarchicalTypeDefinition<TraitType> fetlTypeDefinition =
-                createTraitTypeDef(TRAIT_TYPE,
+                        new AttributeDefinition("serde1",
+                                "serdeType", Multiplicity.REQUIRED, false, null),
+                        new AttributeDefinition("serde2",
+                                "serdeType", Multiplicity.REQUIRED, false, null),
+                        new AttributeDefinition("database",
+                                DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
+
+        HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
+                createTraitTypeDef("classification",
                         ImmutableList.<String>of(),
-                        createRequiredAttrDef("level", DataTypes.INT_TYPE));
-        typeDefinitions.add(fetlTypeDefinition);
+                        createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
 
         typeSystem.defineTypes(
-                ImmutableList.<StructTypeDefinition>of(),
-                ImmutableList.of(fetlTypeDefinition),
+                ImmutableList.of(structTypeDefinition),
+                ImmutableList.of(classificationTypeDefinition),
                 ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
-
-        return typeDefinitions;
     }
 
-    private void submitTypes(List<HierarchicalTypeDefinition> typeDefinitions) throws Exception {
-        for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
-            String typesAsJSON = TypesSerialization.toJson(
-                    typeSystem, typeDefinition.typeName);
+    private void submitTypes() throws Exception {
+        String typesAsJSON = TypesSerialization.toJson(typeSystem,
+                Arrays.asList(new String[]{DATABASE_TYPE, TABLE_TYPE, "serdeType", "classification"}));
 
         WebResource resource = service
                 .path("api/metadata/types/submit")
-                    .path(typeDefinition.typeName);
+                .path(TABLE_TYPE);
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
@@ -218,25 +235,34 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         Assert.assertNotNull(responseAsString);
 
         JSONObject response = new JSONObject(responseAsString);
-            Assert.assertEquals(response.get("typeName"), typeDefinition.typeName);
+        Assert.assertEquals(response.get("typeName"), TABLE_TYPE);
         Assert.assertNotNull(response.get("types"));
         Assert.assertNotNull(response.get("requestId"));
     }
-    }
 
     protected ITypedReferenceableInstance createHiveTableInstance() throws Exception {
         Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
         databaseInstance.set("name", DATABASE_NAME);
         databaseInstance.set("description", "foo database");
 
-        Referenceable tableInstance = new Referenceable(TABLE_TYPE, TRAIT_TYPE);
+        Referenceable tableInstance = new Referenceable(TABLE_TYPE, "classification");
         tableInstance.set("name", TABLE_NAME);
         tableInstance.set("description", "bar table");
         tableInstance.set("type", "managed");
-        tableInstance.set(DATABASE_TYPE, databaseInstance);
+        tableInstance.set("database", databaseInstance);
+
+        Struct traitInstance = (Struct) tableInstance.getTrait("classification");
+        traitInstance.set("tag", "foundation_etl");
+
+        Struct serde1Instance = new Struct("serdeType");
+        serde1Instance.set("name", "serde1");
+        serde1Instance.set("serde", "serde1");
+        tableInstance.set("serde1", serde1Instance);
 
-        Struct traitInstance = (Struct) tableInstance.getTrait(TRAIT_TYPE);
-        traitInstance.set("level", 1);
+        Struct serde2Instance = new Struct("serdeType");
+        serde2Instance.set("name", "serde2");
+        serde2Instance.set("serde", "serde2");
+        tableInstance.set("serde2", serde2Instance);
 
         ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
         return tableType.convert(tableInstance, Multiplicity.REQUIRED);