BUG-32592 Add REST API for Search DSL. Contributed by Venkatesh Seetharam

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java

@@ -19,10 +19,8 @@
 package org.apache.hadoop.metadata.web.resources;
 
 import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.metadata.MetadataException;
+import org.apache.hadoop.metadata.discovery.DiscoveryException;
 import org.apache.hadoop.metadata.discovery.DiscoveryService;
-import org.apache.hadoop.metadata.types.TypeSystem;
 import org.apache.hadoop.metadata.web.util.Servlets;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
@@ -41,7 +39,6 @@ import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -64,9 +61,7 @@ public class MetadataDiscoveryResource {
 	private final DiscoveryService discoveryService;
     
     public static final String RESULTS = "results";
-    public static final String TOTAL_SIZE = "totalSize";
-    
-    public static final List<String> typesList = TypeSystem.getInstance().getTypeNames();
+    // public static final String TOTAL_SIZE = "totalSize";
     
      /**
      * Created by the Guice ServletModule and injected with the
@@ -78,11 +73,80 @@ public class MetadataDiscoveryResource {
     public MetadataDiscoveryResource(DiscoveryService discoveryService) {
     	this.discoveryService = discoveryService;
     }
-   
+
+    /**
+     * Search using query DSL.
+     *
+     * @param query search query in raw gremlin or DSL format falling back to full text.
+     * @return JSON representing the type and results.
+     */
     @GET
-    @Path("search/gremlin/{gremlinQuery}")
+    @Path("search")
     @Produces(MediaType.APPLICATION_JSON)
-    public Response searchUsingGremlinQuery(@PathParam("gremlinQuery") String gremlinQuery) {
+    public Response search(@QueryParam("query") String query) {
+        Preconditions.checkNotNull(query, "query cannot be null");
+
+        if (query.startsWith("g.")) { // raw gremlin query
+            return searchUsingGremlinQuery(query);
+        }
+
+        try {
+            JSONObject response = new JSONObject();
+            response.put("requestId", Thread.currentThread().getName());
+            response.put("query", query);
+
+            try {   // fall back to dsl
+                final String jsonResult = discoveryService.searchByDSL(query);
+                response.put("queryType", "dsl");
+                response.put(RESULTS, new JSONObject(jsonResult));
+
+            } catch (Throwable throwable) {
+                LOG.error("Unable to get entity list for query {} using dsl", query, throwable);
+
+                // todo: fall back to full text search
+                response.put("queryType", "full-text");
+                response.put(RESULTS, new JSONObject());
+            }
+
+            return Response.ok(response).build();
+        } catch (JSONException e) {
+            LOG.error("Unable to get entity list for query {}", query, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+        }
+    }
+
+    @GET
+    @Path("search/dsl")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response searchUsingQueryDSL(@QueryParam("query") String dslQuery) {
+        Preconditions.checkNotNull(dslQuery, "dslQuery cannot be null");
+
+        try {
+            final String jsonResult = discoveryService.searchByDSL(dslQuery);
+
+            JSONObject response = new JSONObject();
+            response.put("requestId", Thread.currentThread().getName());
+            response.put("query", dslQuery);
+            response.put("queryType", "dsl");
+            response.put(RESULTS, new JSONObject(jsonResult));
+
+            return Response.ok(response).build();
+        } catch (DiscoveryException e) {
+            LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+        } catch (JSONException e) {
+            LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
+            throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+        }
+    }
+
+    @GET
+    @Path("search/gremlin")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response searchUsingGremlinQuery(@QueryParam("query") String gremlinQuery) {
         Preconditions.checkNotNull(gremlinQuery, "gremlinQuery cannot be null");
 
         try {
@@ -90,15 +154,17 @@ public class MetadataDiscoveryResource {
 
             JSONObject response = new JSONObject();
             response.put("requestId", Thread.currentThread().getName());
+            response.put("query", gremlinQuery);
+            response.put("queryType", "gremlin");
 
             JSONArray list = new JSONArray();
             for (Map<String, String> result : results) {
                 list.put(new JSONObject(result));
             }
-            response.put("results", list);
+            response.put(RESULTS, list);
 
             return Response.ok(response).build();
-        } catch (MetadataException e) {
+        } catch (DiscoveryException e) {
             LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
             throw new WebApplicationException(
                     Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
@@ -110,7 +176,7 @@ public class MetadataDiscoveryResource {
     }
     
     /**
-     * Return a list of Vertices and Edges that eminate from the provided GUID to the depth specified.
+     * Return a list of Vertices and Edges that emanate from the provided GUID to the depth specified.
      * 
      * GET http://host/api/metadata/discovery/search/relationships/{guid}
      * 
@@ -122,7 +188,8 @@ public class MetadataDiscoveryResource {
     @Path("/search/relationships/{guid}")
     @Produces({MediaType.APPLICATION_JSON})
     public Response getLineageResults(@PathParam("guid") final String guid,
-            @DefaultValue("1") @QueryParam("depth") final int depth, @QueryParam("edgesToFollow") final String edgesToFollow) {
+                                      @DefaultValue("1") @QueryParam("depth") final int depth,
+                                      @QueryParam("edgesToFollow") final String edgesToFollow) {
     	
         LOG.info("Performing GUID lineage search for guid= {}", guid);
         Preconditions.checkNotNull(guid, "Invalid argument: \"guid\" cannot be null.");
@@ -161,7 +228,8 @@ public class MetadataDiscoveryResource {
     @Path("/search/fulltext")
     @Produces({MediaType.APPLICATION_JSON})
     public Response getFullTextResults(@QueryParam("text") final String searchText,
-            @DefaultValue("1") @QueryParam("depth") final int depth,@DefaultValue("guid") @QueryParam("property") final String prop) {
+                                       @DefaultValue("1") @QueryParam("depth") final int depth,
+                                       @DefaultValue("guid") @QueryParam("property") final String prop) {
     	
         LOG.info("Performing full text search for vertices with property {} matching= {}", prop, searchText);
         Preconditions.checkNotNull(searchText, "Invalid argument: \"text\" cannot be null.");
@@ -170,7 +238,8 @@ public class MetadataDiscoveryResource {
         // Parent JSON Object
         JSONObject response = new JSONObject();
                 
-        Map<String, HashMap<String, JSONObject>> resultMap = discoveryService.textSearch(searchText, depth, prop);
+        Map<String, HashMap<String, JSONObject>> resultMap = discoveryService.textSearch(
+                searchText, depth, prop);
             	
    		try {
             response.put("requestId", Thread.currentThread().getName());
@@ -182,7 +251,8 @@ public class MetadataDiscoveryResource {
    			}
    		} catch (JSONException e) {
    			throw new WebApplicationException(
-                    Servlets.getErrorResponse("Search: Error building JSON result set.", Response.Status.INTERNAL_SERVER_ERROR));
+                    Servlets.getErrorResponse("Search: Error building JSON result set.",
+                            Response.Status.INTERNAL_SERVER_ERROR));
    		}
         
    		LOG.debug("JSON result:" + response.toString());
@@ -202,19 +272,17 @@ public class MetadataDiscoveryResource {
     @Path("/getIndexedFields")
     @Produces({MediaType.APPLICATION_JSON})
     public Response getLineageResults() {
-    	
     	JSONObject response = new JSONObject();
     	
     	try {
     		response.put("indexed_fields:",discoveryService.getGraphIndexedFields());
     	} catch (JSONException e) {
    			throw new WebApplicationException(
-                    Servlets.getErrorResponse("Search: Error building JSON result set.", Response.Status.INTERNAL_SERVER_ERROR));
+                    Servlets.getErrorResponse("Search: Error building JSON result set.",
+                            Response.Status.INTERNAL_SERVER_ERROR));
    		}
     	
     	LOG.debug("JSON result:" + response.toString());
        	return Response.ok(response).build();
-    	
     }
-
 }

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.metadata.web.resources;
 
 import com.google.common.collect.ImmutableList;
 import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
 import org.apache.hadoop.metadata.types.AttributeDefinition;
@@ -29,8 +30,13 @@ import org.apache.hadoop.metadata.types.IDataType;
 import org.apache.hadoop.metadata.types.Multiplicity;
 import org.apache.hadoop.metadata.types.TraitType;
 import org.apache.hadoop.metadata.types.TypeSystem;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
 
 public abstract class BaseResourceIT {
@@ -52,6 +58,26 @@ public abstract class BaseResourceIT {
         service = client.resource(UriBuilder.fromUri(baseUrl).build());
     }
 
+    protected void sumbitType(String typesAsJSON, String type) throws Exception {
+        WebResource resource = service
+                .path("api/metadata/types/submit")
+                .path(type);
+
+        ClientResponse clientResponse = resource
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
+
+        JSONObject response = new JSONObject(responseAsString);
+        Assert.assertEquals(response.get("typeName"), type);
+        Assert.assertNotNull(response.get("types"));
+        Assert.assertNotNull(response.get("requestId"));
+    }
+
     protected AttributeDefinition createUniqueRequiredAttrDef(String name,
                                                               IDataType dataType) {
         return new AttributeDefinition(name, dataType.getName(),

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResourceIT.java

@@ -18,33 +18,149 @@
 
 package org.apache.hadoop.metadata.web.resources;
 
+import com.google.common.collect.ImmutableList;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.metadata.ITypedReferenceableInstance;
+import org.apache.hadoop.metadata.Referenceable;
+import org.apache.hadoop.metadata.json.Serialization$;
+import org.apache.hadoop.metadata.json.TypesSerialization;
+import org.apache.hadoop.metadata.types.ClassType;
+import org.apache.hadoop.metadata.types.DataTypes;
+import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
+import org.apache.hadoop.metadata.types.Multiplicity;
+import org.apache.hadoop.metadata.types.StructTypeDefinition;
+import org.apache.hadoop.metadata.types.TraitType;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+import scala.actors.threadpool.Arrays;
+
 import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 
-import org.testng.Assert;
-import org.testng.annotations.Test;
 
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
+public class MetadataDiscoveryResourceIT extends BaseResourceIT {
 
+    @BeforeClass
+    public void setUp() throws Exception {
+        super.setUp();
+
+        createTypes();
+        submitTypes();
+        ITypedReferenceableInstance instance = createInstance();
+        String instanceAsJSON = Serialization$.MODULE$.toJson(instance);
+        submitEntity(instanceAsJSON);
+    }
+
+    @Test
+    public void testSearchByDSL() throws Exception {
+        String dslQuery = "from dsl_test_type";
+        WebResource resource = service
+                .path("api/metadata/discovery/search/dsl")
+                .queryParam("query", dslQuery);
+
+        ClientResponse clientResponse = resource
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
+
+        JSONObject response = new JSONObject(responseAsString);
+        Assert.assertNotNull(response.get("requestId"));
+
+        Assert.assertEquals(response.getString("query"), dslQuery);
+        Assert.assertEquals(response.getString("queryType"), "dsl");
+
+        JSONObject results = response.getJSONObject("results");
+        Assert.assertNotNull(results);
+
+        JSONArray rows = results.getJSONArray("rows");
+        Assert.assertEquals(rows.length(), 1);
+    }
+
+    @Test
+    public void testSearchByDSLForUnknownType() throws Exception {
+        String dslQuery = "from blah";
+        WebResource resource = service
+                .path("api/metadata/discovery/search/dsl")
+                .queryParam("query", dslQuery);
+
+        ClientResponse clientResponse = resource
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(),
+                Response.Status.INTERNAL_SERVER_ERROR.getStatusCode());
+    }
+
+    @Test
+    public void testSearchUsingGremlin() throws Exception {
+         String query = "g.V.has('type', 'dsl_test_type').toList()";
+        WebResource resource = service
+                .path("api/metadata/discovery/search")
+                .queryParam("query", query);
+
+        ClientResponse clientResponse = resource
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
+
+        JSONObject response = new JSONObject(responseAsString);
+        Assert.assertNotNull(response.get("requestId"));
+
+        Assert.assertEquals(response.getString("query"), query);
+        Assert.assertEquals(response.getString("queryType"), "gremlin");
+    }
+
+    @Test
+    public void testSearchUsingDSL() throws Exception {
+        String query = "from dsl_test_type";
+        WebResource resource = service
+                .path("api/metadata/discovery/search")
+                .queryParam("query", query);
+
+        ClientResponse clientResponse = resource
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.GET, ClientResponse.class);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
+
+        JSONObject response = new JSONObject(responseAsString);
+        Assert.assertNotNull(response.get("requestId"));
+
+        Assert.assertEquals(response.getString("query"), query);
+        Assert.assertEquals(response.getString("queryType"), "dsl");
+    }
 
-public class MetadataDiscoveryResourceIT extends BaseResourceIT {
-	
     @Test
     public void testFullTextUriExists() throws Exception {
         WebResource resource = service
                 .path("api/metadata/discovery/search/fulltext")
-                .queryParam("depth", "0").queryParam("text","foo").queryParam("property","Name");
+                .queryParam("depth", "0").queryParam("text", "foo").queryParam("property", "Name");
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
-        Assert.assertNotEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
-        
+        Assert.assertNotEquals(clientResponse.getStatus(),
+                Response.Status.NOT_FOUND.getStatusCode());
     }
-    
+
     @Test
     public void testGetIndexedProperties() throws Exception {
         WebResource resource = service
@@ -54,70 +170,119 @@ public class MetadataDiscoveryResourceIT extends BaseResourceIT {
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
-        Assert.assertNotEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
-        
+        Assert.assertNotEquals(clientResponse.getStatus(),
+                Response.Status.NOT_FOUND.getStatusCode());
     }
-    
+
     @Test
     public void testLineageUriExists() throws Exception {
         WebResource resource = service
                 .path("api/metadata/discovery/search/relationships/1")
-                .queryParam("depth", "1").queryParam("edgesToFollow","bar");
+                .queryParam("depth", "1").queryParam("edgesToFollow", "bar");
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
-        Assert.assertNotEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
-        
+        Assert.assertNotEquals(clientResponse.getStatus(),
+                Response.Status.NOT_FOUND.getStatusCode());
     }
-    
-    @Test  (dependsOnMethods = "testFullTextUriExists")
+
+    @Test(dependsOnMethods = "testFullTextUriExists")
     public void testSearchForText() throws Exception {
         WebResource resource = service
                 .path("api/metadata/discovery/search/fulltext")
-                .queryParam("depth", "3").queryParam("text","bar").queryParam("property","hive_table.name");
+                .queryParam("depth", "3").queryParam("text", "bar")
+                .queryParam("property", "hive_table.name");
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
-        
+
         //TODO - Assure zero vertices and edges.        
-        Assert.assertNotEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
-        
+        Assert.assertNotEquals(clientResponse.getStatus(),
+                Response.Status.NOT_FOUND.getStatusCode());
     }
-    
-    @Test  (dependsOnMethods = "testUriExists", enabled = false)
+
+    @Test(dependsOnMethods = "testUriExists", enabled = false)
     public void testSearchForTextNoDepth() throws Exception {
         WebResource resource = service
                 .path("api/metadata/discovery/search/fulltext")
-                .queryParam("depth", "0").queryParam("text","kid").queryParam("property","Name");
+                .queryParam("depth", "0").queryParam("text", "kid").queryParam("property", "Name");
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
-        //TODO - Get count of expected vertices and Edges
-        Assert.assertEquals(true, true);
-        
+        // TODO - Get count of expected vertices and Edges
+        // Assert.assertEquals(true, true);
     }
-    
-    @Test  (dependsOnMethods = "testUriExists", enabled = false)
+
+    @Test(dependsOnMethods = "testUriExists", enabled = false)
     public void testSearchTextWithDepth() throws Exception {
         WebResource resource = service
                 .path("api/metadata/discovery/search/fulltext")
-                .queryParam("depth", "4").queryParam("text","Grandad").queryParam("property","Name");
+                .queryParam("depth", "4").queryParam("text", "Grandad")
+                .queryParam("property", "Name");
 
         ClientResponse clientResponse = resource
                 .accept(MediaType.APPLICATION_JSON)
                 .type(MediaType.APPLICATION_JSON)
                 .method(HttpMethod.GET, ClientResponse.class);
-        //TODO - Get count of expected vertices and Edges
-        Assert.assertEquals(true, true);
-        
+        // TODO - Get count of expected vertices and Edges
+        // Assert.assertEquals(true, true);
+    }
+
+    private void submitEntity(String instanceAsJSON) throws JSONException {
+        WebResource resource = service
+                .path("api/metadata/entities/submit")
+                .path("dsl_test_type");
+
+        ClientResponse clientResponse = resource
+                .accept(MediaType.APPLICATION_JSON)
+                .type(MediaType.APPLICATION_JSON)
+                .method(HttpMethod.POST, ClientResponse.class, instanceAsJSON);
+        Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+        String responseAsString = clientResponse.getEntity(String.class);
+        Assert.assertNotNull(responseAsString);
+
+        JSONObject response = new JSONObject(responseAsString);
+        Assert.assertNotNull(response.get("requestId"));
+
+        String guid = response.get("GUID").toString();
+        Assert.assertNotNull(guid);
+    }
+
+    private void createTypes() throws Exception {
+        HierarchicalTypeDefinition<ClassType> dslTestTypeDefinition =
+                createClassTypeDef("dsl_test_type",
+                        ImmutableList.<String>of(),
+                        createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+                        createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+
+        typeSystem.defineTypes(
+                ImmutableList.<StructTypeDefinition>of(),
+                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+                ImmutableList.of(dslTestTypeDefinition));
+    }
+
+    private void submitTypes() throws Exception {
+        @SuppressWarnings("unchecked")
+        String typesAsJSON = TypesSerialization.toJson(typeSystem,
+                Arrays.asList(new String[]{
+                        "dsl_test_type",
+                }));
+        sumbitType(typesAsJSON, "dsl_test_type");
     }
-    
-    
 
+    private ITypedReferenceableInstance createInstance() throws Exception {
+        Referenceable entityInstance = new Referenceable("dsl_test_type");
+        entityInstance.set("name", "foo name");
+        entityInstance.set("description", "bar description");
+
+        ClassType tableType = typeSystem.getDataType(ClassType.class, "dsl_test_type");
+        return tableType.convert(entityInstance, Multiplicity.REQUIRED);
+    }
 }
\ No newline at end of file