Skip to content

A
atlas
Commit 9d50a245 by Suma Shivaprasad
Options

BUG-32830 Fixed - Result count for all search API calls

parent 54c3c7f0
Showing with 177 additions and 70 deletions
client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
......@@ -51,13 +51,16 @@ public class MetadataServiceClient {
private static final Logger LOG = LoggerFactory.getLogger(MetadataServiceClient.class);
public static final String REQUEST_ID = "requestId";
public static final String RESULTS = "results";
public static final String TOTAL_SIZE = "totalSize";
public static final String COUNT = "count";
public static final String ROWS = "rows";
private static final String BASE_URI = "api/metadata/";
private static final String URI_TYPES = "types";
private static final String URI_ENTITIES = "entities";
private static final String URI_TRAITS = "traits";
private static final String URI_SEARCH = "discovery/search";
private WebResource service;
public MetadataServiceClient(String baseUrl) {
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java
......@@ -143,14 +143,10 @@ public class EntityResource {
* Gets the list of entities for a given entity type.
*
* @param entityType name of a type which is unique
* @param offset starting offset for pagination
* @param resultsPerPage number of results for pagination
*/
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getEntityListByType(@QueryParam("type") String entityType,
@DefaultValue("0") @QueryParam("offset") Integer offset,
@QueryParam("numResults") Integer resultsPerPage) {
public Response getEntityListByType(@QueryParam("type") String entityType) {
Preconditions.checkNotNull(entityType, "Entity type cannot be null");
try {
LOG.debug("Fetching entity list for type={} ", entityType);
......@@ -160,7 +156,7 @@ public class EntityResource {
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
response.put("type", entityType);
response.put(MetadataServiceClient.RESULTS, new JSONArray(entityList));
response.put(MetadataServiceClient.TOTAL_SIZE, entityList.size());
response.put(MetadataServiceClient.COUNT, entityList.size());
return Response.ok(response).build();
} catch (MetadataException | IllegalArgumentException e) {
......@@ -225,7 +221,7 @@ public class EntityResource {
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
response.put(GUID, guid);
response.put(MetadataServiceClient.RESULTS, new JSONArray(traitNames));
response.put(MetadataServiceClient.TOTAL_SIZE, traitNames.size());
response.put(MetadataServiceClient.COUNT, traitNames.size());
return Response.ok(response).build();
} catch (MetadataException | IllegalArgumentException e) {
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java
......@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......@@ -49,6 +49,11 @@ import java.util.Map;
public class MetadataDiscoveryResource {
private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
private static final String QUERY = "query";
private static final String QUERY_TYPE = "queryType";
private static final String QUERY_TYPE_DSL = "dsl";
private static final String QUERY_TYPE_GREMLIN = "gremlin";
private static final String QUERY_TYPE_FULLTEXT = "full-text";
private final DiscoveryService discoveryService;
......@@ -79,38 +84,36 @@ public class MetadataDiscoveryResource {
return searchUsingGremlinQuery(query);
}
try {
JSONObject response = new JSONObject();
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
response.put("query", query);
try { // fall back to dsl
final String jsonResult = discoveryService.searchByDSL(query);
response.put("queryType", "dsl");
response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
} catch (Throwable throwable) {
LOG.error("Unable to get entity list for query {} using dsl", query, throwable);
try { //fall back to full-text
final String jsonResult = discoveryService.searchByFullText(query);
response.put("queryType", "full-text");
response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
} catch (DiscoveryException e) {
LOG.error("Unable to get entity list for query {}", query, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (JSONException e) {
LOG.error("Unable to get entity list for query {}", query, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
JSONObject response = null;
return Response.ok(response).build();
} catch (JSONException e) {
LOG.error("Unable to get entity list for query {}", query, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
try { // fall back to dsl
final String jsonResultStr = discoveryService.searchByDSL(query);
response = new DSLJSONResponseBuilder().results(jsonResultStr)
.query(query)
.build();
} catch (Throwable throwable) {
LOG.error("Unable to get entity list for query {} using dsl", query, throwable);
try { //fall back to full-text
final String jsonResultStr = discoveryService.searchByFullText(query);
response = new FullTextJSonResponseBuilder().results(jsonResultStr)
.query(query)
.build();
} catch (DiscoveryException e) {
LOG.error("Unable to get entity list for query {}", query, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (JSONException e) {
LOG.error("Unable to get entity list for query {}", query, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
return Response.ok(response)
.build();
}
/**
......@@ -126,23 +129,22 @@ public class MetadataDiscoveryResource {
Preconditions.checkNotNull(dslQuery, "dslQuery cannot be null");
try {
final String jsonResult = discoveryService.searchByDSL(dslQuery);
final String jsonResultStr = discoveryService.searchByDSL(dslQuery);
JSONObject response = new JSONObject();
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
response.put("query", dslQuery);
response.put("queryType", "dsl");
response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
JSONObject response = new DSLJSONResponseBuilder().results(jsonResultStr)
.query(dslQuery)
.build();
return Response.ok(response).build();
return Response.ok(response)
.build();
} catch (DiscoveryException e) {
LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (JSONException e) {
LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
......@@ -160,29 +162,30 @@ public class MetadataDiscoveryResource {
try {
final List<Map<String, String>> results = discoveryService
.searchByGremlin(gremlinQuery);
.searchByGremlin(gremlinQuery);
JSONObject response = new JSONObject();
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
response.put("query", gremlinQuery);
response.put("queryType", "gremlin");
response.put(QUERY, gremlinQuery);
response.put(QUERY_TYPE, QUERY_TYPE_GREMLIN);
JSONArray list = new JSONArray();
for (Map<String, String> result : results) {
list.put(new JSONObject(result));
}
response.put(MetadataServiceClient.RESULTS, list);
response.put(MetadataServiceClient.TOTAL_SIZE, list.length());
response.put(MetadataServiceClient.COUNT, list.length());
return Response.ok(response).build();
return Response.ok(response)
.build();
} catch (DiscoveryException e) {
LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (JSONException e) {
LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
......@@ -199,15 +202,14 @@ public class MetadataDiscoveryResource {
Preconditions.checkNotNull(query, "query cannot be null");
try {
final String jsonResult = discoveryService.searchByFullText(query);
JSONObject response = new JSONObject();
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
response.put("query", query);
response.put("queryType", "full-text");
response.put(MetadataServiceClient.RESULTS, new JSONArray(jsonResult));
final String jsonResultStr = discoveryService.searchByFullText(query);
JSONArray rowsJsonArr = new JSONArray(jsonResultStr);
return Response.ok(response).build();
JSONObject response = new FullTextJSonResponseBuilder().results(rowsJsonArr)
.query(query)
.build();
return Response.ok(response)
.build();
} catch (DiscoveryException e) {
LOG.error("Unable to get entity list for query {}", query, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
......@@ -215,6 +217,104 @@ public class MetadataDiscoveryResource {
LOG.error("Unable to get entity list for query {}", query, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
private class JsonResponseBuilder {
protected int count = 0;
protected String query;
protected String queryType;
protected JSONObject response;
JsonResponseBuilder() {
this.response = new JSONObject();
}
protected JsonResponseBuilder count(int count) {
this.count = count;
return this;
}
public JsonResponseBuilder query(String query) {
this.query = query;
return this;
}
public JsonResponseBuilder queryType(String queryType) {
this.queryType = queryType;
return this;
}
protected JSONObject build() throws JSONException {
Preconditions.checkNotNull(query, "Query cannot be null");
Preconditions.checkNotNull(queryType, "Query Type must be specified");
Preconditions.checkArgument(count >= 0, "Search Result count should be > 0");
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
response.put(QUERY, query);
response.put(QUERY_TYPE, queryType);
response.put(MetadataServiceClient.COUNT, count);
return response;
}
}
private class DSLJSONResponseBuilder extends JsonResponseBuilder {
DSLJSONResponseBuilder() {
super();
}
private JSONObject dslResults;
public DSLJSONResponseBuilder results(JSONObject dslResults) {
this.dslResults = dslResults;
return this;
}
public DSLJSONResponseBuilder results(String dslResults) throws JSONException {
return results(new JSONObject(dslResults));
}
@Override
public JSONObject build() throws JSONException {
Preconditions.checkNotNull(dslResults);
JSONArray rowsJsonArr = dslResults.getJSONArray(MetadataServiceClient.ROWS);
count(rowsJsonArr.length());
queryType(QUERY_TYPE_DSL);
JSONObject response = super.build();
response.put(MetadataServiceClient.RESULTS, dslResults);
return response;
}
}
private class FullTextJSonResponseBuilder extends JsonResponseBuilder {
private JSONArray fullTextResults;
public FullTextJSonResponseBuilder results(JSONArray fullTextResults) {
this.fullTextResults = fullTextResults;
return this;
}
public FullTextJSonResponseBuilder results(String dslResults) throws JSONException {
return results(new JSONArray(dslResults));
}
public FullTextJSonResponseBuilder() {
super();
}
@Override
public JSONObject build() throws JSONException {
Preconditions.checkNotNull(fullTextResults);
count(fullTextResults.length());
queryType(QUERY_TYPE_FULLTEXT);
JSONObject response = super.build();
response.put(MetadataServiceClient.RESULTS, fullTextResults);
return response;
}
}
}
\ No newline at end of file
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java
......@@ -165,7 +165,7 @@ public class RexsterGraphResource {
JSONObject response = new JSONObject();
response.put(MetadataServiceClient.RESULTS, new JSONObject(vertexProperties));
response.put(MetadataServiceClient.TOTAL_SIZE, vertexProperties.size());
response.put(MetadataServiceClient.COUNT, vertexProperties.size());
return Response.ok(response).build();
} catch (JSONException e) {
throw new WebApplicationException(
......@@ -276,7 +276,7 @@ public class RexsterGraphResource {
if (!countOnly) {
response.put(MetadataServiceClient.RESULTS, elementArray);
}
response.put(MetadataServiceClient.TOTAL_SIZE, counter);
response.put(MetadataServiceClient.COUNT, counter);
return Response.ok(response).build();
}
......@@ -323,7 +323,7 @@ public class RexsterGraphResource {
JSONObject response = new JSONObject();
response.put(MetadataServiceClient.RESULTS, vertexArray);
response.put(MetadataServiceClient.TOTAL_SIZE, counter);
response.put(MetadataServiceClient.COUNT, counter);
return response;
}
......
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java
......@@ -136,7 +136,7 @@ public class TypesResource {
JSONObject response = new JSONObject();
response.put(MetadataServiceClient.RESULTS, new JSONArray(result));
response.put(MetadataServiceClient.TOTAL_SIZE, result.size());
response.put(MetadataServiceClient.COUNT, result.size());
response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
return Response.ok(response).build();
......
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryJerseyResourceIT.java
......@@ -83,8 +83,12 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
Assert.assertNotNull(results);
JSONArray rows = results.getJSONArray("rows");
JSONArray rows = results.getJSONArray(MetadataServiceClient.ROWS);
Assert.assertEquals(rows.length(), 1);
int numRows = response.getInt(MetadataServiceClient.COUNT);
Assert.assertEquals(numRows, 1);
}
@Test
......@@ -164,6 +168,9 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
Assert.assertNotNull(row.get("guid"));
Assert.assertEquals(row.getString("typeName"), "dsl_test_type");
Assert.assertNotNull(row.get("score"));
int numRows = response.getInt(MetadataServiceClient.COUNT);
Assert.assertEquals(numRows, 1);
}
private void createTypes() throws Exception {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment