BUG-32830 Fixed - Result count for all search API calls

client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java

@@ -51,13 +51,16 @@ public class MetadataServiceClient {
     private static final Logger LOG = LoggerFactory.getLogger(MetadataServiceClient.class);
     public static final String REQUEST_ID = "requestId";
     public static final String RESULTS = "results";
-    public static final String TOTAL_SIZE = "totalSize";
+    public static final String COUNT = "count";
+    public static final String ROWS = "rows";
+
     private static final String BASE_URI = "api/metadata/";
     private static final String URI_TYPES = "types";
     private static final String URI_ENTITIES = "entities";
     private static final String URI_TRAITS = "traits";
     private static final String URI_SEARCH = "discovery/search";
 
+
     private WebResource service;
 
     public MetadataServiceClient(String baseUrl) {

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java

@@ -143,14 +143,10 @@ public class EntityResource {
      * Gets the list of entities for a given entity type.
      *
      * @param entityType     name of a type which is unique
-     * @param offset         starting offset for pagination
-     * @param resultsPerPage number of results for pagination
      */
     @GET
     @Produces(MediaType.APPLICATION_JSON)
-    public Response getEntityListByType(@QueryParam("type") String entityType,
-                                  @DefaultValue("0") @QueryParam("offset") Integer offset,
-                                  @QueryParam("numResults") Integer resultsPerPage) {
+    public Response getEntityListByType(@QueryParam("type") String entityType) {
         Preconditions.checkNotNull(entityType, "Entity type cannot be null");
         try {
             LOG.debug("Fetching entity list for type={} ", entityType);
@@ -160,7 +156,7 @@ public class EntityResource {
             response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
             response.put("type", entityType);
             response.put(MetadataServiceClient.RESULTS, new JSONArray(entityList));
-            response.put(MetadataServiceClient.TOTAL_SIZE, entityList.size());
+            response.put(MetadataServiceClient.COUNT, entityList.size());
 
             return Response.ok(response).build();
         } catch (MetadataException | IllegalArgumentException e) {
@@ -225,7 +221,7 @@ public class EntityResource {
             response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
             response.put(GUID, guid);
             response.put(MetadataServiceClient.RESULTS, new JSONArray(traitNames));
-            response.put(MetadataServiceClient.TOTAL_SIZE, traitNames.size());
+            response.put(MetadataServiceClient.COUNT, traitNames.size());
 
             return Response.ok(response).build();
         } catch (MetadataException | IllegalArgumentException e) {

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java

@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -49,6 +49,11 @@ import java.util.Map;
 public class MetadataDiscoveryResource {
 
     private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
+    private static final String QUERY = "query";
+    private static final String QUERY_TYPE = "queryType";
+    private static final String QUERY_TYPE_DSL = "dsl";
+    private static final String QUERY_TYPE_GREMLIN = "gremlin";
+    private static final String QUERY_TYPE_FULLTEXT = "full-text";
 
     private final DiscoveryService discoveryService;
 
@@ -79,38 +84,36 @@ public class MetadataDiscoveryResource {
             return searchUsingGremlinQuery(query);
         }
 
-        try {
-            JSONObject response = new JSONObject();
-            response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
-            response.put("query", query);
-
-            try {   // fall back to dsl
-                final String jsonResult = discoveryService.searchByDSL(query);
-                response.put("queryType", "dsl");
-                response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
-
-            } catch (Throwable throwable) {
-                LOG.error("Unable to get entity list for query {} using dsl", query, throwable);
-
-                try {   //fall back to full-text
-                    final String jsonResult = discoveryService.searchByFullText(query);
-                    response.put("queryType", "full-text");
-                    response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
-                } catch (DiscoveryException e) {
-                    LOG.error("Unable to get entity list for query {}", query, e);
-                    throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
-                } catch (JSONException e) {
-                    LOG.error("Unable to get entity list for query {}", query, e);
-                    throw new WebApplicationException(
-                            Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
-                }
-            }
+        JSONObject response = null;
 
-            return Response.ok(response).build();
-        } catch (JSONException e) {
-            LOG.error("Unable to get entity list for query {}", query, e);
-            throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+        try {   // fall back to dsl
+            final String jsonResultStr = discoveryService.searchByDSL(query);
+            response = new DSLJSONResponseBuilder().results(jsonResultStr)
+                .query(query)
+                .build();
+
+        } catch (Throwable throwable) {
+            LOG.error("Unable to get entity list for query {} using dsl", query, throwable);
+
+            try {   //fall back to full-text
+                final String jsonResultStr = discoveryService.searchByFullText(query);
+                response = new FullTextJSonResponseBuilder().results(jsonResultStr)
+                    .query(query)
+                    .build();
+
+            } catch (DiscoveryException e) {
+                LOG.error("Unable to get entity list for query {}", query, e);
+                throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+            } catch (JSONException e) {
+                LOG.error("Unable to get entity list for query {}", query, e);
+                throw new WebApplicationException(
+                    Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+            }
         }
+
+        return Response.ok(response)
+            .build();
+
     }
 
     /**
@@ -126,23 +129,22 @@ public class MetadataDiscoveryResource {
         Preconditions.checkNotNull(dslQuery, "dslQuery cannot be null");
 
         try {
-            final String jsonResult = discoveryService.searchByDSL(dslQuery);
+            final String jsonResultStr = discoveryService.searchByDSL(dslQuery);
 
-            JSONObject response = new JSONObject();
-            response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
-            response.put("query", dslQuery);
-            response.put("queryType", "dsl");
-            response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
+            JSONObject response = new DSLJSONResponseBuilder().results(jsonResultStr)
+                .query(dslQuery)
+                .build();
 
-            return Response.ok(response).build();
+            return Response.ok(response)
+                .build();
         } catch (DiscoveryException e) {
             LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
             throw new WebApplicationException(
-                    Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+                Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
         } catch (JSONException e) {
             LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
             throw new WebApplicationException(
-                    Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+                Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
         }
     }
 
@@ -160,29 +162,30 @@ public class MetadataDiscoveryResource {
 
         try {
             final List<Map<String, String>> results = discoveryService
-                    .searchByGremlin(gremlinQuery);
+                .searchByGremlin(gremlinQuery);
 
             JSONObject response = new JSONObject();
             response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
-            response.put("query", gremlinQuery);
-            response.put("queryType", "gremlin");
+            response.put(QUERY, gremlinQuery);
+            response.put(QUERY_TYPE, QUERY_TYPE_GREMLIN);
 
             JSONArray list = new JSONArray();
             for (Map<String, String> result : results) {
                 list.put(new JSONObject(result));
             }
             response.put(MetadataServiceClient.RESULTS, list);
-            response.put(MetadataServiceClient.TOTAL_SIZE, list.length());
+            response.put(MetadataServiceClient.COUNT, list.length());
 
-            return Response.ok(response).build();
+            return Response.ok(response)
+                .build();
         } catch (DiscoveryException e) {
             LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
             throw new WebApplicationException(
-                    Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+                Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
         } catch (JSONException e) {
             LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
             throw new WebApplicationException(
-                    Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+                Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
         }
     }
 
@@ -199,15 +202,14 @@ public class MetadataDiscoveryResource {
         Preconditions.checkNotNull(query, "query cannot be null");
 
         try {
-            final String jsonResult = discoveryService.searchByFullText(query);
-
-            JSONObject response = new JSONObject();
-            response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
-            response.put("query", query);
-            response.put("queryType", "full-text");
-            response.put(MetadataServiceClient.RESULTS, new JSONArray(jsonResult));
+            final String jsonResultStr = discoveryService.searchByFullText(query);
+            JSONArray rowsJsonArr = new JSONArray(jsonResultStr);
 
-            return Response.ok(response).build();
+            JSONObject response = new FullTextJSonResponseBuilder().results(rowsJsonArr)
+                .query(query)
+                .build();
+            return Response.ok(response)
+                .build();
         } catch (DiscoveryException e) {
             LOG.error("Unable to get entity list for query {}", query, e);
             throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
@@ -215,6 +217,104 @@ public class MetadataDiscoveryResource {
             LOG.error("Unable to get entity list for query {}", query, e);
             throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
         }
+    }
+
+    private class JsonResponseBuilder {
+
+        protected int count = 0;
+        protected String query;
+        protected String queryType;
+        protected JSONObject response;
+
+        JsonResponseBuilder() {
+            this.response = new JSONObject();
+        }
+
+        protected JsonResponseBuilder count(int count) {
+            this.count = count;
+            return this;
+        }
+
+        public JsonResponseBuilder query(String query) {
+            this.query = query;
+            return this;
+        }
 
+        public JsonResponseBuilder queryType(String queryType) {
+            this.queryType = queryType;
+            return this;
+        }
+
+        protected JSONObject build() throws JSONException {
+
+            Preconditions.checkNotNull(query, "Query cannot be null");
+            Preconditions.checkNotNull(queryType, "Query Type must be specified");
+            Preconditions.checkArgument(count >= 0, "Search Result count should be > 0");
+
+            response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+            response.put(QUERY, query);
+            response.put(QUERY_TYPE, queryType);
+            response.put(MetadataServiceClient.COUNT, count);
+            return response;
+        }
+    }
+
+    private class DSLJSONResponseBuilder extends JsonResponseBuilder {
+
+        DSLJSONResponseBuilder() {
+            super();
+        }
+
+        private JSONObject dslResults;
+
+        public DSLJSONResponseBuilder results(JSONObject dslResults) {
+            this.dslResults = dslResults;
+            return this;
+        }
+
+        public DSLJSONResponseBuilder results(String dslResults) throws JSONException {
+            return results(new JSONObject(dslResults));
+        }
+
+        @Override
+        public JSONObject build() throws JSONException {
+            Preconditions.checkNotNull(dslResults);
+            JSONArray rowsJsonArr = dslResults.getJSONArray(MetadataServiceClient.ROWS);
+            count(rowsJsonArr.length());
+            queryType(QUERY_TYPE_DSL);
+            JSONObject response = super.build();
+            response.put(MetadataServiceClient.RESULTS, dslResults);
+            return response;
+        }
+
+    }
+
+    private class FullTextJSonResponseBuilder extends JsonResponseBuilder {
+
+        private JSONArray fullTextResults;
+
+        public FullTextJSonResponseBuilder results(JSONArray fullTextResults) {
+            this.fullTextResults = fullTextResults;
+            return this;
+        }
+
+        public FullTextJSonResponseBuilder results(String dslResults) throws JSONException {
+            return results(new JSONArray(dslResults));
+        }
+
+        public FullTextJSonResponseBuilder() {
+            super();
+        }
+
+        @Override
+        public JSONObject build() throws JSONException {
+            Preconditions.checkNotNull(fullTextResults);
+            count(fullTextResults.length());
+            queryType(QUERY_TYPE_FULLTEXT);
+
+            JSONObject response = super.build();
+            response.put(MetadataServiceClient.RESULTS, fullTextResults);
+            return response;
+        }
     }
 }
\ No newline at end of file

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java

@@ -165,7 +165,7 @@ public class RexsterGraphResource {
 
             JSONObject response = new JSONObject();
             response.put(MetadataServiceClient.RESULTS, new JSONObject(vertexProperties));
-            response.put(MetadataServiceClient.TOTAL_SIZE, vertexProperties.size());
+            response.put(MetadataServiceClient.COUNT, vertexProperties.size());
             return Response.ok(response).build();
         } catch (JSONException e) {
             throw new WebApplicationException(
@@ -276,7 +276,7 @@ public class RexsterGraphResource {
         if (!countOnly) {
             response.put(MetadataServiceClient.RESULTS, elementArray);
         }
-        response.put(MetadataServiceClient.TOTAL_SIZE, counter);
+        response.put(MetadataServiceClient.COUNT, counter);
         return Response.ok(response).build();
     }
 
@@ -323,7 +323,7 @@ public class RexsterGraphResource {
 
         JSONObject response = new JSONObject();
         response.put(MetadataServiceClient.RESULTS, vertexArray);
-        response.put(MetadataServiceClient.TOTAL_SIZE, counter);
+        response.put(MetadataServiceClient.COUNT, counter);
 
         return response;
     }

webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java

@@ -136,7 +136,7 @@ public class TypesResource {
 
             JSONObject response = new JSONObject();
             response.put(MetadataServiceClient.RESULTS, new JSONArray(result));
-            response.put(MetadataServiceClient.TOTAL_SIZE, result.size());
+            response.put(MetadataServiceClient.COUNT, result.size());
             response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
 
             return Response.ok(response).build();

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryJerseyResourceIT.java

@@ -83,8 +83,12 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
         JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
         Assert.assertNotNull(results);
 
-        JSONArray rows = results.getJSONArray("rows");
+        JSONArray rows = results.getJSONArray(MetadataServiceClient.ROWS);
         Assert.assertEquals(rows.length(), 1);
+
+        int numRows = response.getInt(MetadataServiceClient.COUNT);
+        Assert.assertEquals(numRows, 1);
+
     }
 
     @Test
@@ -164,6 +168,9 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
         Assert.assertNotNull(row.get("guid"));
         Assert.assertEquals(row.getString("typeName"), "dsl_test_type");
         Assert.assertNotNull(row.get("score"));
+
+        int numRows = response.getInt(MetadataServiceClient.COUNT);
+        Assert.assertEquals(numRows, 1);
     }
 
     private void createTypes() throws Exception {