Commit 309af260 by Venkatesh Seetharam

Add more Search DSL tests and fix Example

parent 0d38165d
......@@ -58,12 +58,10 @@ import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.MetadataServiceException;
import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge;
import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.io.BufferedWriter;
......@@ -258,7 +256,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
String typeName = HiveDataTypes.HIVE_DB.getName();
MetadataServiceClient dgiClient = dgiBridge.getMetadataServiceClient();
JSONObject result = dgiClient.search(typeName, "name", dbName);
JSONObject result = dgiClient.rawSearch(typeName, "name", dbName);
JSONArray results = (JSONArray) result.get("results");
if (results.length() == 0) {
......@@ -283,7 +281,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
String typeName = HiveDataTypes.HIVE_TABLE.getName();
MetadataServiceClient dgiClient = dgiBridge.getMetadataServiceClient();
JSONObject result = dgiClient.search(typeName, "tableName", tableName);
JSONObject result = dgiClient.rawSearch(typeName, "tableName", tableName);
JSONArray results = (JSONArray) result.get("results");
if (results.length() == 0) {
......
......@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.hive.hook;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.metadata.MetadataServiceClient;
......@@ -29,7 +28,6 @@ import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator;
import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.apache.hadoop.metadata.typesystem.TypesDef;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
......@@ -130,7 +128,7 @@ public class HiveHookIT {
}
private void assertInstanceIsRegistered(String typeName, String colName, String colValue) throws Exception{
JSONObject result = dgiCLient.search(typeName, colName, colValue);
JSONObject result = dgiCLient.rawSearch(typeName, colName, colValue);
JSONArray results = (JSONArray) result.get("results");
Assert.assertEquals(results.length(), 1);
JSONObject resultRow = (JSONObject) results.get(0);
......
......@@ -137,6 +137,12 @@ public class MetadataServiceClient {
return callAPI(API.GET_ENTITY, null, guid);
}
public JSONObject searchEntity(String searchQuery) throws MetadataServiceException {
WebResource resource = getResource(API.SEARCH);
resource = resource.queryParam("query", searchQuery);
return callAPIWithResource(API.SEARCH, resource);
}
/**
* Search given type name, an attribute and its value. Uses search dsl
* @param typeName name of the entity type
......@@ -145,10 +151,12 @@ public class MetadataServiceClient {
* @return result json object
* @throws MetadataServiceException
*/
public JSONObject search(String typeName, String attributeName, Object attributeValue) throws MetadataServiceException {
String gremlinQuery = String.format("g.V.has(\"typeName\",\"%s\").and(_().has(\"%s.%s\", T.eq, \"%s\")).toList()",
public JSONObject rawSearch(String typeName, String attributeName,
Object attributeValue) throws MetadataServiceException {
String gremlinQuery = String.format(
"g.V.has(\"typeName\",\"%s\").and(_().has(\"%s.%s\", T.eq, \"%s\")).toList()",
typeName, typeName, attributeName, attributeValue);
return search(gremlinQuery);
return searchByGremlin(gremlinQuery);
}
/**
......@@ -169,10 +177,10 @@ public class MetadataServiceClient {
* @return result json object
* @throws MetadataServiceException
*/
public JSONObject search(String gremlinQuery) throws MetadataServiceException {
WebResource resource = getResource(API.SEARCH);
public JSONObject searchByGremlin(String gremlinQuery) throws MetadataServiceException {
WebResource resource = getResource(API.SEARCH_GREMLIN);
resource = resource.queryParam("query", gremlinQuery);
return callAPIWithResource(API.SEARCH, resource);
return callAPIWithResource(API.SEARCH_GREMLIN, resource);
}
public String getRequestId(JSONObject json) throws MetadataServiceException {
......@@ -193,13 +201,16 @@ public class MetadataServiceClient {
return resource;
}
private JSONObject callAPIWithResource(API api, WebResource resource) throws MetadataServiceException {
private JSONObject callAPIWithResource(API api,
WebResource resource) throws MetadataServiceException {
return callAPIWithResource(api, resource, null);
}
private JSONObject callAPIWithResource(API api, WebResource resource, Object requestObject)
throws MetadataServiceException {
ClientResponse clientResponse = resource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON)
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(api.getMethod(), ClientResponse.class, requestObject);
if (clientResponse.getStatus() == Response.Status.OK.getStatusCode()) {
......@@ -213,7 +224,8 @@ public class MetadataServiceClient {
throw new MetadataServiceException(api, clientResponse.getClientResponseStatus());
}
private JSONObject callAPI(API api, Object requestObject, String... pathParams) throws MetadataServiceException {
private JSONObject callAPI(API api, Object requestObject,
String... pathParams) throws MetadataServiceException {
WebResource resource = getResource(api, pathParams);
return callAPIWithResource(api, resource, requestObject);
}
......
......@@ -47,6 +47,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Graph backed implementation of Search.
*/
public class GraphBackedDiscoveryService implements DiscoveryService {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedDiscoveryService.class);
......@@ -69,12 +72,17 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
*/
@Override
public String searchByDSL(String dslQuery) throws DiscoveryException {
QueryParser queryParser = new QueryParser();
Either<Parsers.NoSuccess, Expressions.Expression> either = queryParser.apply(dslQuery);
if (either.isRight()) {
Expressions.Expression expression = either.right().get();
GremlinQueryResult queryResult = evaluate(expression);
return queryResult.toJson();
LOG.info("Executing dsl query={}", dslQuery);
try {
QueryParser queryParser = new QueryParser();
Either<Parsers.NoSuccess, Expressions.Expression> either = queryParser.apply(dslQuery);
if (either.isRight()) {
Expressions.Expression expression = either.right().get();
GremlinQueryResult queryResult = evaluate(expression);
return queryResult.toJson();
}
} catch (Exception e) { // unable to catch ExpressionException
throw new DiscoveryException("Invalid expression : " + dslQuery);
}
throw new DiscoveryException("Invalid expression : " + dslQuery);
......@@ -102,6 +110,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
@Override
public List<Map<String, String>> searchByGremlin(String gremlinQuery)
throws DiscoveryException {
LOG.info("Executing gremlin query={}", gremlinQuery);
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
Bindings bindings = engine.createBindings();
......
......@@ -165,19 +165,57 @@ public class GraphBackedDiscoveryServiceTest {
@DataProvider(name = "dslQueriesProvider")
private Object[][] createDSLQueries() {
return new String[][] {
{"from DB"},
{"DB"},
{"from Table"},
{"Table"},
{"DB, Table"},
/*{"DB as db1 Table where db1.name = \"Reporting\""},*/
{"DB name = \"Reporting\""},
{"Column where Column isa PII"},
{"Table is Dimension"},
{"View is Dimension"},
/*{"Column where Column isa PII select Column.name"},*/
{"Column select Column.name"},
{"from Table select Table.name"},
{"from DB"},
{"DB"},
{"DB where DB.name=\"Reporting\""},
{"DB DB.name = \"Reporting\""},
{"DB where DB.name=\"Reporting\" select name, owner"},
{"DB has name"},
{"DB, Table"},
{"DB is JdbcAccess"},
/*
{"DB, LoadProcess has name"},
{"DB as db1, Table where db1.name = \"Reporting\""},
{"DB where DB.name=\"Reporting\" and DB.createTime < " + System.currentTimeMillis()},
*/
{"from Table"},
{"Table"},
{"Table is Dimension"},
{"Column where Column isa PII"},
{"View is Dimension"},
/*{"Column where Column isa PII select Column.name"},*/
{"Column select Column.name"},
{"Column select name"},
{"Column where Column.name=\"customer_id\""},
{"from Table select Table.name"},
{"DB where (name = \"Reporting\")"},
{"DB where (name = \"Reporting\") select name as _col_0, owner as _col_1"},
{"DB where DB is JdbcAccess"},
{"DB where DB has name"},
{"DB Table"},
{"DB where DB has name"},
{"DB as db1 Table where (db1.name = \"Reporting\")"},
{"DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
/*
todo: does not work
{"DB where (name = \"Reporting\") and ((createTime + 1) > 0)"},
{"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
{"DB as db1 Table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
{"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
{"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
*/
// trait searches
{"Dimension"},
/*{"Fact"}, - todo: does not work*/
{"JdbcAccess"},
{"ETL"},
{"Metric"},
{"PII"},
// Lineage
{"Table LoadProcess outputTable"},
{"Table loop (LoadProcess outputTable)"},
{"Table as _loop0 loop (LoadProcess outputTable) withPath"},
{"Table as src loop (LoadProcess outputTable) as dest select src.name as srcTable, dest.name as destTable withPath"},
};
}
......@@ -201,7 +239,24 @@ public class GraphBackedDiscoveryServiceTest {
JSONArray rows = results.getJSONArray("rows");
Assert.assertNotNull(rows);
Assert.assertTrue(rows.length() > 0);
Assert.assertTrue(rows.length() >= 0); // some queries may not have any results
System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
}
@DataProvider(name = "invalidDslQueriesProvider")
private Object[][] createInvalidDSLQueries() {
return new String[][] {
{"from Unknown"},
{"Unknown"},
{"Unknown is Blah"},
};
}
@Test (dataProvider = "invalidDslQueriesProvider", expectedExceptions = DiscoveryException.class)
public void testSearchByDSLInvalidQueries(String dslQuery) throws Exception {
System.out.println("Executing dslQuery = " + dslQuery);
discoveryService.searchByDSL(dslQuery);
Assert.fail();
}
@Test
......
......@@ -44,7 +44,9 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.List;
/**
* Search Integration Tests.
*/
public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
@BeforeClass
......@@ -96,7 +98,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
.type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class);
Assert.assertEquals(clientResponse.getStatus(),
Response.Status.INTERNAL_SERVER_ERROR.getStatusCode());
Response.Status.BAD_REQUEST.getStatusCode());
}
@Test
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment