ISSUE-35 Add typed instance put/get end to end. Contributed by Venkatesh Seetharam

repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java

@@ -83,8 +83,10 @@ public class DefaultMetadataService implements MetadataService {
 
             return response;
         } catch (ParseException e) {
+            LOG.error("Unable to parse JSON for type {}", typeName, e);
             throw new MetadataException("validation failed for: " + typeName);
         } catch (JSONException e) {
+            LOG.error("Unable to persist type {}", typeName, e);
             throw new MetadataException("Unable to create response for: " + typeName);
         }
     }
@@ -147,6 +149,7 @@ public class DefaultMetadataService implements MetadataService {
                     Serialization$.MODULE$.fromJson(entityDefinition);
             return repository.createEntity(entityInstance, entityType);
         } catch (ParseException e) {
+            LOG.error("Unable to parse JSON {} for type {}", entityDefinition, entityType, e);
             throw new MetadataException("validation failed for: " + entityType);
         }
     }
@@ -154,7 +157,9 @@ public class DefaultMetadataService implements MetadataService {
     private void validateEntity(String entity, String entityType) throws ParseException {
         Preconditions.checkNotNull(entity, "entity cannot be null");
         Preconditions.checkNotNull(entityType, "entity type cannot be null");
-        JSONValue.parseWithException(entity);
+
+        // todo: this is failing for instances but not types
+        // JSONValue.parseWithException(entity);
     }
 
     /**
@@ -167,21 +172,9 @@ public class DefaultMetadataService implements MetadataService {
     public String getEntityDefinition(String guid) throws MetadataException {
         final ITypedReferenceableInstance instance =
                 repository.getEntityDefinition(guid);
-        return Serialization$.MODULE$.toJson(instance);
-    }
-
-    /**
-     * Return the definition for the given entity name and type.
-     *
-     * @param entityName name
-     * @param entityType type
-     * @return entity definition as JSON
-     */
-    @Override
-    public String getEntityDefinition(String entityName,
-                                      String entityType) throws MetadataException {
-
-        throw new UnsupportedOperationException();
+        return instance == null
+                ? null
+                : Serialization$.MODULE$.toJson(instance);
     }
 
     /**

repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java

@@ -73,15 +73,6 @@ public interface MetadataService extends Service {
     String getEntityDefinition(String guid) throws MetadataException;
 
     /**
-     * Return the definition for the given entity name and type.
-     *
-     * @param entityName name
-     * @param entityType type
-     * @return entity definition as JSON
-     */
-    String getEntityDefinition(String entityName, String entityType) throws MetadataException;
-
-    /**
      * Return the list of entity names for the given type in the repository.
      *
      * @param entityType type

repository/src/main/java/org/apache/hadoop/metadata/util/GraphUtils.java

@@ -23,9 +23,45 @@ public final class GraphUtils {
 
     private static final Logger LOG = LoggerFactory.getLogger(GraphUtils.class);
 
+    private static final String GUID_PROPERTY_KEY = "guid";
+    private static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
+
     private GraphUtils() {
     }
 
+    public static Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
+        return addEdge(fromVertex, toVertex, edgeLabel, null);
+    }
+
+    public static Edge addEdge(Vertex fromVertex, Vertex toVertex,
+                               String edgeLabel, String timestamp) {
+        Edge edge = findEdge(fromVertex, toVertex, edgeLabel);
+
+        Edge edgeToVertex = edge != null ? edge : fromVertex.addEdge(edgeLabel, toVertex);
+        if (timestamp != null) {
+            edgeToVertex.setProperty(TIMESTAMP_PROPERTY_KEY, timestamp);
+        }
+
+        return edgeToVertex;
+    }
+
+    public static Edge findEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
+        return findEdge(fromVertex, toVertex.getProperty(GUID_PROPERTY_KEY), edgeLabel);
+    }
+
+    public static Edge findEdge(Vertex fromVertex, Object toVertexName, String edgeLabel) {
+        Edge edgeToFind = null;
+        for (Edge edge : fromVertex.getEdges(Direction.OUT, edgeLabel)) {
+            if (edge.getVertex(Direction.IN).getProperty(
+                    GUID_PROPERTY_KEY).equals(toVertexName)) {
+                edgeToFind = edge;
+                break;
+            }
+        }
+
+        return edgeToFind;
+    }
+
     public static Vertex findVertex(Graph blueprintsGraph,
                                     String guid) {
         LOG.debug("Finding vertex for: guid={}", guid);

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java

@@ -67,22 +67,23 @@ public class EntityResource {
     }
 
     @POST
-    @Path("submit/{entityType}")
+    @Path("submit/{typeName}")
     @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
     public Response submit(@Context HttpServletRequest request,
-                           @PathParam("entityType") final String entityType) {
+                           @PathParam("typeName") final String typeName) {
         try {
             final String entity = Servlets.getRequestPayload(request);
-            System.out.println("entity = " + entity);
+            LOG.debug("submitting entity {} ", entity);
 
-            final String guid = metadataService.createEntity(entity, entityType);
+            final String guid = metadataService.createEntity(typeName, entity);
             JSONObject response = new JSONObject();
             response.put("GUID", guid);
             response.put("requestId", Thread.currentThread().getName());
 
             return Response.ok(response).build();
         } catch (Exception e) {
+            LOG.error("Unable to persist instance for type {}", typeName, e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
         }
@@ -109,8 +110,7 @@ public class EntityResource {
             return Response.status(status).entity(response).build();
 
         } catch (Exception e) {
-            LOG.error("Action failed: {}\nError: {}",
-                    Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
+            LOG.error("Unable to get instance definition for GUID {}", guid, e);
             throw new WebApplicationException(e, Response
                     .status(Response.Status.INTERNAL_SERVER_ERROR)
                     .entity(e.getMessage())

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java

@@ -10,16 +10,21 @@ import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
 import org.apache.hadoop.metadata.types.IDataType;
 import org.apache.hadoop.metadata.types.Multiplicity;
 import org.apache.hadoop.metadata.types.TraitType;
+import org.apache.hadoop.metadata.types.TypeSystem;
 import org.testng.annotations.BeforeClass;
 
 import javax.ws.rs.core.UriBuilder;
 
-public class BaseResourceIT {
+public abstract class BaseResourceIT {
 
+    protected TypeSystem typeSystem;
     protected WebResource service;
 
     @BeforeClass
     public void setUp() throws Exception {
+        typeSystem = TypeSystem.getInstance();
+        typeSystem.reset();
+
         String baseUrl = "http://localhost:21000/";
 
         DefaultClientConfig config = new DefaultClientConfig();

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java

@@ -18,54 +18,78 @@
 
 package org.apache.hadoop.metadata.web.resources;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
+import com.google.common.collect.ImmutableList;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.metadata.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.Referenceable;
+import org.apache.hadoop.metadata.Struct;
+import org.apache.hadoop.metadata.json.Serialization$;
+import org.apache.hadoop.metadata.json.TypesSerialization;
+import org.apache.hadoop.metadata.types.AttributeDefinition;
+import org.apache.hadoop.metadata.types.ClassType;
+import org.apache.hadoop.metadata.types.DataTypes;
+import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
+import org.apache.hadoop.metadata.types.Multiplicity;
+import org.apache.hadoop.metadata.types.StructTypeDefinition;
+import org.apache.hadoop.metadata.types.TraitType;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
 
 import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-
-import org.json.simple.JSONValue;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import com.google.gson.JsonElement;
-import com.google.gson.JsonParser;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
 
 /**
  * Integration tests for Entity Jersey Resource.
  */
-@Test (enabled = false)
 public class EntityJerseyResourceIT extends BaseResourceIT {
 
-    private static final String ENTITY_NAME = "clicks-table";
-    private static final String ENTITY_TYPE = "hive-table";
-    private static final String DATABASE_NAME = "ads";
-    private static final String TABLE_NAME = "clicks-table";
+    private static final String DATABASE_TYPE = "hive_database";
+    private static final String DATABASE_NAME = "foo";
+    private static final String TABLE_TYPE = "hive_table";
+    private static final String TABLE_NAME = "bar";
+    private static final String TRAIT_TYPE = "hive_fetl";
+
+    private String guid;
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        super.setUp();
+
+        List<HierarchicalTypeDefinition> typeDefinitions = createHiveTypes();
+        submitTypes(typeDefinitions);
+    }
+
+    @Test
+    public void testSubmitEntity() throws Exception {
+        ITypedReferenceableInstance tableInstance = createHiveTableInstance();
 
-    @Test (enabled = false)
-    public void testSubmitEntity() {
-        String entityStream = getTestEntityJSON();
-        JsonParser parser = new JsonParser();
+        String instanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
 
         WebResource resource = service
                 .path("api/metadata/entities/submit")
-                .path(ENTITY_TYPE);
+                .path(TABLE_TYPE);
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
-                .method(HttpMethod.POST, ClientResponse.class, entityStream);
+                .method(HttpMethod.POST, ClientResponse.class, instanceAsJSON);
         Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
 
-        String response = clientResponse.getEntity(String.class);
-        Assert.assertNotNull(response);
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
+
+        JSONObject response = new JSONObject(responseAsString);
+        Assert.assertNotNull(response.get("requestId"));
 
-        JsonElement elem = parser.parse(response);
-        String guid = elem.getAsJsonObject().get("GUID").getAsString();
+        guid = response.get("GUID").toString();
+        Assert.assertNotNull(guid);
 
         try {
             Assert.assertNotNull(UUID.fromString(guid));
@@ -74,12 +98,11 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         }
     }
 
-    @Test (dependsOnMethods = "testSubmitEntity", enabled = false)
+    @Test (dependsOnMethods = "testSubmitEntity")
     public void testGetEntityDefinition() {
         WebResource resource = service
                 .path("api/metadata/entities/definition")
-                .path(ENTITY_TYPE)
-                .path(ENTITY_NAME);
+                .path(guid);
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
@@ -90,20 +113,10 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         System.out.println("response = " + response);
     }
 
-    private static String getTestEntityJSON() {
-        Map<String, String> props = new HashMap<>();
-        props.put("entityName", ENTITY_NAME);
-        props.put("entityType", ENTITY_TYPE);
-        props.put("database", DATABASE_NAME);
-        props.put("table", TABLE_NAME);
-        return JSONValue.toJSONString(props);
-    }
-
-    @Test (enabled = false)
+    @Test
     public void testGetInvalidEntityDefinition() {
         WebResource resource = service
                 .path("api/metadata/entities/definition")
-                .path(ENTITY_TYPE)
                 .path("blah");
 
         ClientResponse clientResponse = resource
@@ -119,7 +132,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
     public void testGetEntityList() {
         ClientResponse clientResponse = service
                 .path("api/metadata/entities/list/")
-                .path(ENTITY_TYPE)
+                .path(TABLE_TYPE)
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
@@ -139,4 +152,81 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         String response = clientResponse.getEntity(String.class);
         System.out.println("response = " + response);
     }
+
+    private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception {
+        ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
+
+        HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
+                createClassTypeDef(DATABASE_TYPE,
+                        ImmutableList.<String>of(),
+                        createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+        typeDefinitions.add(databaseTypeDefinition);
+
+        HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
+                createClassTypeDef(TABLE_TYPE,
+                        ImmutableList.<String>of(),
+                        createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        createRequiredAttrDef("description", DataTypes.STRING_TYPE),
+                        createRequiredAttrDef("type", DataTypes.STRING_TYPE),
+                        new AttributeDefinition(DATABASE_TYPE,
+                                DATABASE_TYPE, Multiplicity.REQUIRED, true, DATABASE_TYPE));
+        typeDefinitions.add(tableTypeDefinition);
+
+        HierarchicalTypeDefinition<TraitType> fetlTypeDefinition =
+                createTraitTypeDef(TRAIT_TYPE,
+                        ImmutableList.<String>of(),
+                        createRequiredAttrDef("level", DataTypes.INT_TYPE));
+        typeDefinitions.add(fetlTypeDefinition);
+
+        typeSystem.defineTypes(
+                ImmutableList.<StructTypeDefinition>of(),
+                ImmutableList.of(fetlTypeDefinition),
+                ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
+
+        return typeDefinitions;
+    }
+
+    private void submitTypes(List<HierarchicalTypeDefinition> typeDefinitions) throws Exception {
+        for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
+            String typesAsJSON = TypesSerialization.toJson(
+                    typeSystem, typeDefinition.typeName);
+
+            WebResource resource = service
+                    .path("api/metadata/types/submit")
+                    .path(typeDefinition.typeName);
+
+            ClientResponse clientResponse = resource
+                    .accept(MediaType.APPLICATION_JSON)
+                    .type(MediaType.APPLICATION_JSON)
+                    .method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
+            Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+            String responseAsString = clientResponse.getEntity(String.class);
+            Assert.assertNotNull(responseAsString);
+
+            JSONObject response = new JSONObject(responseAsString);
+            Assert.assertEquals(response.get("typeName"), typeDefinition.typeName);
+            Assert.assertNotNull(response.get("types"));
+            Assert.assertNotNull(response.get("requestId"));
+        }
+    }
+
+    protected ITypedReferenceableInstance createHiveTableInstance() throws Exception {
+        Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+        databaseInstance.set("name", DATABASE_NAME);
+        databaseInstance.set("description", "foo database");
+
+        Referenceable tableInstance = new Referenceable(TABLE_TYPE, TRAIT_TYPE);
+        tableInstance.set("name", TABLE_NAME);
+        tableInstance.set("description", "bar table");
+        tableInstance.set("type", "managed");
+        tableInstance.set(DATABASE_TYPE, databaseInstance);
+
+        Struct traitInstance = (Struct) tableInstance.getTrait(TRAIT_TYPE);
+        traitInstance.set("level", 1);
+
+        ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
+        return tableType.convert(tableInstance, Multiplicity.REQUIRED);
+    }
 }

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/TypesJerseyResourceIT.java

@@ -21,7 +21,6 @@ package org.apache.hadoop.metadata.web.resources;
 import com.google.common.collect.ImmutableList;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.json.TypesSerialization;
 import org.apache.hadoop.metadata.types.AttributeDefinition;
 import org.apache.hadoop.metadata.types.ClassType;
@@ -30,7 +29,6 @@ import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
 import org.apache.hadoop.metadata.types.Multiplicity;
 import org.apache.hadoop.metadata.types.StructTypeDefinition;
 import org.apache.hadoop.metadata.types.TraitType;
-import org.apache.hadoop.metadata.types.TypeSystem;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
 import org.testng.Assert;
@@ -49,14 +47,12 @@ import java.util.List;
  */
 public class TypesJerseyResourceIT extends BaseResourceIT {
 
-    private TypeSystem typeSystem;
     private List<HierarchicalTypeDefinition> typeDefinitions;
 
     @BeforeClass
     public void setUp() throws Exception {
         super.setUp();
 
-        typeSystem = TypeSystem.getInstance();
         typeDefinitions = createHiveTypes();
     }
 
@@ -151,11 +147,12 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
         Assert.assertNotNull(list);
     }
 
-    private List<HierarchicalTypeDefinition> createHiveTypes() throws MetadataException {
+    private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception {
         ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
 
         HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
-                createClassTypeDef("database", ImmutableList.<String>of(),
+                createClassTypeDef("database",
+                        ImmutableList.<String>of(),
                         createRequiredAttrDef("name", DataTypes.STRING_TYPE),
                         createRequiredAttrDef("description", DataTypes.STRING_TYPE));
         typeDefinitions.add(databaseTypeDefinition);