ISSUE-39 Add typed instance list end to end. Contributed by Venkatesh Seetharam

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java

@@ -23,6 +23,7 @@ import com.thinkaurelius.titan.core.TitanVertex;
 import com.tinkerpop.blueprints.Direction;
 import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.Graph;
+import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.TransactionalGraph;
 import com.tinkerpop.blueprints.Vertex;
 import org.apache.commons.collections.map.HashedMap;
@@ -58,6 +59,7 @@ import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -160,6 +162,27 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
         }
     }
 
+    @Override
+    public List<String> getEntityList(String entityType) throws RepositoryException {
+        LOG.info("Retrieving entity list for type={}", entityType);
+
+        GraphQuery query = graphService.getBlueprintsGraph().query()
+                .has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
+        Iterator<Vertex> results = query.vertices().iterator();
+        if (!results.hasNext()) {
+            return Collections.emptyList();
+        }
+
+        ArrayList<String> entityList = new ArrayList<>();
+        while (results.hasNext()) {
+            Vertex vertex = results.next();
+            entityList.add(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY));
+        }
+
+        return entityList;
+    }
+
+    @Override
     public List<Map<String,String>> rawSearch(String gremlinQuery) throws RepositoryException {
         ScriptEngineManager manager = new ScriptEngineManager();
         ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
@@ -209,12 +232,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
         }
     }
 
-    @Override
-    public List<String> getEntityList(String entityType) throws RepositoryException {
-        LOG.info("Retrieving entity list for type={}", entityType);
-        return Collections.emptyList();
-    }
-
     private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
 
         public final Map<Id, Id> idToNewIdMap;
@@ -328,7 +345,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
             throws MetadataException {
             String typedInstanceGUID = null;
             for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse over newInstances
-                LOG.debug("Adding typed instance {}", typedInstance);
+                LOG.debug("Adding typed instance {}", typedInstance.getTypeName());
 
                 Id id = typedInstance.getId();
                 if (id == null) { // oops
@@ -364,7 +381,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
                                          Map<String, AttributeInfo> fields,
                                          Map<Id, Vertex> idToVertexMap) throws MetadataException {
             LOG.debug("Mapping instance {} to vertex {} for fields {}",
-                    typedInstance, instanceVertex, fields);
+                    typedInstance.getTypeName(), instanceVertex, fields);
             for (AttributeInfo attributeInfo : fields.values()) {
                 LOG.debug("mapping attributeInfo {}", attributeInfo);
                 final IDataType dataType = attributeInfo.dataType();
@@ -611,7 +628,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
         private void mapVertexToStructInstance(Vertex instanceVertex,
                                                ITypedInstance typedInstance,
                                                AttributeInfo attributeInfo) throws MetadataException {
-            LOG.debug("mapping vertex {} to struct {}", typedInstance, attributeInfo.name);
+            LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
             StructType structType = typeSystem.getDataType(
                     StructType.class, attributeInfo.dataType().getName());
             ITypedStruct structInstance = structType.createInstance();

repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java

@@ -27,11 +27,11 @@ import org.apache.hadoop.metadata.json.TypesSerialization;
 import org.apache.hadoop.metadata.listener.TypedInstanceChangeListener;
 import org.apache.hadoop.metadata.listener.TypesChangeListener;
 import org.apache.hadoop.metadata.repository.MetadataRepository;
+import org.apache.hadoop.metadata.storage.RepositoryException;
 import org.apache.hadoop.metadata.types.IDataType;
 import org.apache.hadoop.metadata.types.TypeSystem;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
-import org.json.simple.JSONValue;
 import org.json.simple.parser.ParseException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -73,7 +73,7 @@ public class DefaultMetadataService implements MetadataService {
     public JSONObject createType(String typeName,
                                  String typeDefinition) throws MetadataException {
         try {
-            validate(typeName, typeDefinition);
+            validateTypeDoesNotExist(typeName, typeDefinition);
 
             TypesDef typesDef = TypesSerialization.fromJson(typeDefinition);
             Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef);
@@ -86,20 +86,16 @@ public class DefaultMetadataService implements MetadataService {
             }
 
             return response;
-        } catch (ParseException e) {
-            LOG.error("Unable to parse JSON for type {}", typeName, e);
-            throw new MetadataException("validation failed for: " + typeName);
         } catch (JSONException e) {
             LOG.error("Unable to persist type {}", typeName, e);
             throw new MetadataException("Unable to create response for: " + typeName);
         }
     }
 
-    private void validate(String typeName,
-                          String typeDefinition) throws ParseException, MetadataException {
+    private void validateTypeDoesNotExist(String typeName,
+                                          String typeDefinition) throws MetadataException {
         Preconditions.checkNotNull(typeName, "type name cannot be null");
         Preconditions.checkNotNull(typeDefinition, "type definition cannot be null");
-        JSONValue.parseWithException(typeDefinition);
 
         // verify if the type already exists
         String existingTypeDefinition = null;
@@ -110,7 +106,7 @@ public class DefaultMetadataService implements MetadataService {
         }
 
         if (existingTypeDefinition != null) {
-            throw new MetadataException("type is already defined for : " + typeName);
+            throw new RepositoryException("type is already defined for : " + typeName);
         }
     }
 
@@ -178,8 +174,9 @@ public class DefaultMetadataService implements MetadataService {
      */
     @Override
     public String getEntityDefinition(String guid) throws MetadataException {
-        final ITypedReferenceableInstance instance =
-                repository.getEntityDefinition(guid);
+        Preconditions.checkNotNull(guid, "guid cannot be null");
+
+        final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
         return instance == null
                 ? null
                 : Serialization$.MODULE$.toJson(instance);
@@ -192,8 +189,26 @@ public class DefaultMetadataService implements MetadataService {
      * @return list of entity names for the given type in the repository
      */
     @Override
-    public List<String> getEntityNamesList(String entityType) throws MetadataException {
-        throw new UnsupportedOperationException();
+    public List<String> getEntityList(String entityType) throws MetadataException {
+        validateTypeExists(entityType);
+
+        return repository.getEntityList(entityType);
+    }
+
+    private void validateTypeExists(String entityType) throws MetadataException {
+        Preconditions.checkNotNull(entityType, "entity type cannot be null");
+
+        // verify if the type exists
+        String existingTypeDefinition = null;
+        try {
+            existingTypeDefinition = getTypeDefinition(entityType);
+        } catch (MetadataException ignore) {
+            // do nothing
+        }
+
+        if (existingTypeDefinition == null) {
+            throw new RepositoryException("type is not defined for : " + entityType);
+        }
     }
 
     private void onAdd(Map<String, IDataType> typesAdded) throws MetadataException {

repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java

@@ -78,5 +78,5 @@ public interface MetadataService extends Service {
      * @param entityType type
      * @return list of entity names for the given type in the repository
      */
-    List<String> getEntityNamesList(String entityType) throws MetadataException;
+    List<String> getEntityList(String entityType) throws MetadataException;
 }

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java

@@ -19,8 +19,11 @@
 package org.apache.hadoop.metadata.web.resources;
 
 import com.google.common.base.Preconditions;
+import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.services.MetadataService;
 import org.apache.hadoop.metadata.web.util.Servlets;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,6 +43,7 @@ import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.util.List;
 
 /**
  * Entity management operations as REST API.
@@ -120,19 +124,27 @@ public class EntityResource {
     }
 
     @GET
-    @Path("definition/{entityType}/{entityName}")
-    @Produces(MediaType.APPLICATION_JSON)
-    public Response getEntityDefinition(@PathParam("entityType") String entityType,
-                                        @PathParam("entityName") String entityName) {
-        return Response.status(Response.Status.SERVICE_UNAVAILABLE).build();
-    }
-
-    @GET
     @Path("list/{entityType}")
     @Produces(MediaType.APPLICATION_JSON)
     public Response getEntityList(@PathParam("entityType") String entityType,
                                   @DefaultValue("0") @QueryParam("offset") Integer offset,
                                   @QueryParam("numResults") Integer resultsPerPage) {
-        return Response.ok().build();
+        try {
+            final List<String> entityList = metadataService.getEntityList(entityType);
+
+            JSONObject response = new JSONObject();
+            response.put("requestId", Thread.currentThread().getName());
+            response.put("list", new JSONArray(entityList));
+
+            return Response.ok(response).build();
+        } catch (MetadataException e) {
+            LOG.error("Unable to get entity list for type {}", entityType, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+        } catch (JSONException e) {
+            LOG.error("Unable to get entity list for type {}", entityType, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+        }
     }
 }

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java

@@ -124,7 +124,7 @@ public class TypesResource {
 
             return Response.ok(response).build();
         } catch (Exception e) {
-            LOG.error("Unable to persist entity object", e);
+            LOG.error("Unable to get types list", e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
         }

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java

@@ -33,6 +33,8 @@ import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
 import org.apache.hadoop.metadata.types.Multiplicity;
 import org.apache.hadoop.metadata.types.StructTypeDefinition;
 import org.apache.hadoop.metadata.types.TraitType;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -140,7 +142,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
 */
 
     @Test
-    public void testGetInvalidEntityDefinition() {
+    public void testGetInvalidEntityDefinition() throws Exception {
         WebResource resource = service
                 .path("api/metadata/entities/definition")
                 .path("blah");
@@ -150,12 +152,13 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
         Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
-        String response = clientResponse.getEntity(String.class);
-        System.out.println("response = " + response);
+
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
     }
 
-    @Test (dependsOnMethods = "testSubmitEntity", enabled = false)
-    public void testGetEntityList() {
+    @Test (dependsOnMethods = "testSubmitEntity")
+    public void testGetEntityList() throws Exception {
         ClientResponse clientResponse = service
                 .path("api/metadata/entities/list/")
                 .path(TABLE_TYPE)
@@ -163,20 +166,63 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
         Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-        String response = clientResponse.getEntity(String.class);
-        System.out.println("response = " + response);
+
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
+
+        JSONObject response = new JSONObject(responseAsString);
+        Assert.assertNotNull(response.get("requestId"));
+
+        final JSONArray list = response.getJSONArray("list");
+        Assert.assertNotNull(list);
+        Assert.assertEquals(list.length(), 1);
     }
 
-    @Test (enabled = false) // todo: enable this later
-    public void testGetEntityListForBadEntityType() {
+    @Test
+    public void testGetEntityListForBadEntityType() throws Exception {
         ClientResponse clientResponse = service
                 .path("api/metadata/entities/list/blah")
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
-        Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
-        String response = clientResponse.getEntity(String.class);
-        System.out.println("response = " + response);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
+    }
+
+    @Test
+    public void testGetEntityListForNoInstances() throws Exception {
+        addNewType();
+
+        ClientResponse clientResponse = service
+                .path("api/metadata/entities/list/test")
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
+
+        JSONObject response = new JSONObject(responseAsString);
+        Assert.assertNotNull(response.get("requestId"));
+
+        final JSONArray list = response.getJSONArray("list");
+        Assert.assertEquals(list.length(), 0);
+    }
+
+    private void addNewType() throws Exception {
+        HierarchicalTypeDefinition<ClassType> testTypeDefinition =
+                createClassTypeDef("test",
+                        ImmutableList.<String>of(),
+                        createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+        typeSystem.defineClassType(testTypeDefinition);
+
+        String typesAsJSON = TypesSerialization.toJson(typeSystem,
+                Arrays.asList(new String[]{"test"}));
+        sumbitType(typesAsJSON, "test");
     }
 
     private void createHiveTypes() throws Exception {
@@ -220,10 +266,13 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
     private void submitTypes() throws Exception {
         String typesAsJSON = TypesSerialization.toJson(typeSystem,
                 Arrays.asList(new String[]{DATABASE_TYPE, TABLE_TYPE, "serdeType", "classification"}));
+        sumbitType(typesAsJSON, TABLE_TYPE);
+    }
 
+    private void sumbitType(String typesAsJSON, String type) throws JSONException {
         WebResource resource = service
                 .path("api/metadata/types/submit")
-                .path(TABLE_TYPE);
+                .path(type);
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
@@ -235,12 +284,12 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         Assert.assertNotNull(responseAsString);
 
         JSONObject response = new JSONObject(responseAsString);
-        Assert.assertEquals(response.get("typeName"), TABLE_TYPE);
+        Assert.assertEquals(response.get("typeName"), type);
         Assert.assertNotNull(response.get("types"));
         Assert.assertNotNull(response.get("requestId"));
     }
 
-    protected ITypedReferenceableInstance createHiveTableInstance() throws Exception {
+    private ITypedReferenceableInstance createHiveTableInstance() throws Exception {
         Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
         databaseInstance.set("name", DATABASE_NAME);
         databaseInstance.set("description", "foo database");