Commit 309af260 by Venkatesh Seetharam

Add more Search DSL tests and fix Example

parent 0d38165d
...@@ -58,12 +58,10 @@ import org.apache.hadoop.hive.ql.parse.ParseDriver; ...@@ -58,12 +58,10 @@ import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.metadata.MetadataServiceClient; import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.MetadataServiceException;
import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge; import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge;
import org.apache.hadoop.metadata.hive.model.HiveDataTypes; import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.apache.hadoop.metadata.typesystem.Referenceable; import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import java.io.BufferedWriter; import java.io.BufferedWriter;
...@@ -258,7 +256,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo ...@@ -258,7 +256,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
String typeName = HiveDataTypes.HIVE_DB.getName(); String typeName = HiveDataTypes.HIVE_DB.getName();
MetadataServiceClient dgiClient = dgiBridge.getMetadataServiceClient(); MetadataServiceClient dgiClient = dgiBridge.getMetadataServiceClient();
JSONObject result = dgiClient.search(typeName, "name", dbName); JSONObject result = dgiClient.rawSearch(typeName, "name", dbName);
JSONArray results = (JSONArray) result.get("results"); JSONArray results = (JSONArray) result.get("results");
if (results.length() == 0) { if (results.length() == 0) {
...@@ -283,7 +281,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo ...@@ -283,7 +281,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
String typeName = HiveDataTypes.HIVE_TABLE.getName(); String typeName = HiveDataTypes.HIVE_TABLE.getName();
MetadataServiceClient dgiClient = dgiBridge.getMetadataServiceClient(); MetadataServiceClient dgiClient = dgiBridge.getMetadataServiceClient();
JSONObject result = dgiClient.search(typeName, "tableName", tableName); JSONObject result = dgiClient.rawSearch(typeName, "tableName", tableName);
JSONArray results = (JSONArray) result.get("results"); JSONArray results = (JSONArray) result.get("results");
if (results.length() == 0) { if (results.length() == 0) {
......
...@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.hive.hook; ...@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.hive.hook;
import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.metadata.MetadataServiceClient; import org.apache.hadoop.metadata.MetadataServiceClient;
...@@ -29,7 +28,6 @@ import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator; ...@@ -29,7 +28,6 @@ import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator;
import org.apache.hadoop.metadata.hive.model.HiveDataTypes; import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.apache.hadoop.metadata.typesystem.TypesDef; import org.apache.hadoop.metadata.typesystem.TypesDef;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization; import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert; import org.testng.Assert;
...@@ -130,7 +128,7 @@ public class HiveHookIT { ...@@ -130,7 +128,7 @@ public class HiveHookIT {
} }
private void assertInstanceIsRegistered(String typeName, String colName, String colValue) throws Exception{ private void assertInstanceIsRegistered(String typeName, String colName, String colValue) throws Exception{
JSONObject result = dgiCLient.search(typeName, colName, colValue); JSONObject result = dgiCLient.rawSearch(typeName, colName, colValue);
JSONArray results = (JSONArray) result.get("results"); JSONArray results = (JSONArray) result.get("results");
Assert.assertEquals(results.length(), 1); Assert.assertEquals(results.length(), 1);
JSONObject resultRow = (JSONObject) results.get(0); JSONObject resultRow = (JSONObject) results.get(0);
......
...@@ -137,6 +137,12 @@ public class MetadataServiceClient { ...@@ -137,6 +137,12 @@ public class MetadataServiceClient {
return callAPI(API.GET_ENTITY, null, guid); return callAPI(API.GET_ENTITY, null, guid);
} }
public JSONObject searchEntity(String searchQuery) throws MetadataServiceException {
WebResource resource = getResource(API.SEARCH);
resource = resource.queryParam("query", searchQuery);
return callAPIWithResource(API.SEARCH, resource);
}
/** /**
* Search given type name, an attribute and its value. Uses search dsl * Search given type name, an attribute and its value. Uses search dsl
* @param typeName name of the entity type * @param typeName name of the entity type
...@@ -145,10 +151,12 @@ public class MetadataServiceClient { ...@@ -145,10 +151,12 @@ public class MetadataServiceClient {
* @return result json object * @return result json object
* @throws MetadataServiceException * @throws MetadataServiceException
*/ */
public JSONObject search(String typeName, String attributeName, Object attributeValue) throws MetadataServiceException { public JSONObject rawSearch(String typeName, String attributeName,
String gremlinQuery = String.format("g.V.has(\"typeName\",\"%s\").and(_().has(\"%s.%s\", T.eq, \"%s\")).toList()", Object attributeValue) throws MetadataServiceException {
String gremlinQuery = String.format(
"g.V.has(\"typeName\",\"%s\").and(_().has(\"%s.%s\", T.eq, \"%s\")).toList()",
typeName, typeName, attributeName, attributeValue); typeName, typeName, attributeName, attributeValue);
return search(gremlinQuery); return searchByGremlin(gremlinQuery);
} }
/** /**
...@@ -169,10 +177,10 @@ public class MetadataServiceClient { ...@@ -169,10 +177,10 @@ public class MetadataServiceClient {
* @return result json object * @return result json object
* @throws MetadataServiceException * @throws MetadataServiceException
*/ */
public JSONObject search(String gremlinQuery) throws MetadataServiceException { public JSONObject searchByGremlin(String gremlinQuery) throws MetadataServiceException {
WebResource resource = getResource(API.SEARCH); WebResource resource = getResource(API.SEARCH_GREMLIN);
resource = resource.queryParam("query", gremlinQuery); resource = resource.queryParam("query", gremlinQuery);
return callAPIWithResource(API.SEARCH, resource); return callAPIWithResource(API.SEARCH_GREMLIN, resource);
} }
public String getRequestId(JSONObject json) throws MetadataServiceException { public String getRequestId(JSONObject json) throws MetadataServiceException {
...@@ -193,13 +201,16 @@ public class MetadataServiceClient { ...@@ -193,13 +201,16 @@ public class MetadataServiceClient {
return resource; return resource;
} }
private JSONObject callAPIWithResource(API api, WebResource resource) throws MetadataServiceException { private JSONObject callAPIWithResource(API api,
WebResource resource) throws MetadataServiceException {
return callAPIWithResource(api, resource, null); return callAPIWithResource(api, resource, null);
} }
private JSONObject callAPIWithResource(API api, WebResource resource, Object requestObject) private JSONObject callAPIWithResource(API api, WebResource resource, Object requestObject)
throws MetadataServiceException { throws MetadataServiceException {
ClientResponse clientResponse = resource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON) ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.method(api.getMethod(), ClientResponse.class, requestObject); .method(api.getMethod(), ClientResponse.class, requestObject);
if (clientResponse.getStatus() == Response.Status.OK.getStatusCode()) { if (clientResponse.getStatus() == Response.Status.OK.getStatusCode()) {
...@@ -213,7 +224,8 @@ public class MetadataServiceClient { ...@@ -213,7 +224,8 @@ public class MetadataServiceClient {
throw new MetadataServiceException(api, clientResponse.getClientResponseStatus()); throw new MetadataServiceException(api, clientResponse.getClientResponseStatus());
} }
private JSONObject callAPI(API api, Object requestObject, String... pathParams) throws MetadataServiceException { private JSONObject callAPI(API api, Object requestObject,
String... pathParams) throws MetadataServiceException {
WebResource resource = getResource(api, pathParams); WebResource resource = getResource(api, pathParams);
return callAPIWithResource(api, resource, requestObject); return callAPIWithResource(api, resource, requestObject);
} }
......
...@@ -47,6 +47,9 @@ import java.util.HashMap; ...@@ -47,6 +47,9 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
/**
* Graph backed implementation of Search.
*/
public class GraphBackedDiscoveryService implements DiscoveryService { public class GraphBackedDiscoveryService implements DiscoveryService {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedDiscoveryService.class); private static final Logger LOG = LoggerFactory.getLogger(GraphBackedDiscoveryService.class);
...@@ -69,6 +72,8 @@ public class GraphBackedDiscoveryService implements DiscoveryService { ...@@ -69,6 +72,8 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
*/ */
@Override @Override
public String searchByDSL(String dslQuery) throws DiscoveryException { public String searchByDSL(String dslQuery) throws DiscoveryException {
LOG.info("Executing dsl query={}", dslQuery);
try {
QueryParser queryParser = new QueryParser(); QueryParser queryParser = new QueryParser();
Either<Parsers.NoSuccess, Expressions.Expression> either = queryParser.apply(dslQuery); Either<Parsers.NoSuccess, Expressions.Expression> either = queryParser.apply(dslQuery);
if (either.isRight()) { if (either.isRight()) {
...@@ -76,6 +81,9 @@ public class GraphBackedDiscoveryService implements DiscoveryService { ...@@ -76,6 +81,9 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
GremlinQueryResult queryResult = evaluate(expression); GremlinQueryResult queryResult = evaluate(expression);
return queryResult.toJson(); return queryResult.toJson();
} }
} catch (Exception e) { // unable to catch ExpressionException
throw new DiscoveryException("Invalid expression : " + dslQuery);
}
throw new DiscoveryException("Invalid expression : " + dslQuery); throw new DiscoveryException("Invalid expression : " + dslQuery);
} }
...@@ -102,6 +110,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService { ...@@ -102,6 +110,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
@Override @Override
public List<Map<String, String>> searchByGremlin(String gremlinQuery) public List<Map<String, String>> searchByGremlin(String gremlinQuery)
throws DiscoveryException { throws DiscoveryException {
LOG.info("Executing gremlin query={}", gremlinQuery);
ScriptEngineManager manager = new ScriptEngineManager(); ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("gremlin-groovy"); ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
Bindings bindings = engine.createBindings(); Bindings bindings = engine.createBindings();
......
...@@ -167,17 +167,55 @@ public class GraphBackedDiscoveryServiceTest { ...@@ -167,17 +167,55 @@ public class GraphBackedDiscoveryServiceTest {
return new String[][] { return new String[][] {
{"from DB"}, {"from DB"},
{"DB"}, {"DB"},
{"DB where DB.name=\"Reporting\""},
{"DB DB.name = \"Reporting\""},
{"DB where DB.name=\"Reporting\" select name, owner"},
{"DB has name"},
{"DB, Table"},
{"DB is JdbcAccess"},
/*
{"DB, LoadProcess has name"},
{"DB as db1, Table where db1.name = \"Reporting\""},
{"DB where DB.name=\"Reporting\" and DB.createTime < " + System.currentTimeMillis()},
*/
{"from Table"}, {"from Table"},
{"Table"}, {"Table"},
{"DB, Table"},
/*{"DB as db1 Table where db1.name = \"Reporting\""},*/
{"DB name = \"Reporting\""},
{"Column where Column isa PII"},
{"Table is Dimension"}, {"Table is Dimension"},
{"Column where Column isa PII"},
{"View is Dimension"}, {"View is Dimension"},
/*{"Column where Column isa PII select Column.name"},*/ /*{"Column where Column isa PII select Column.name"},*/
{"Column select Column.name"}, {"Column select Column.name"},
{"Column select name"},
{"Column where Column.name=\"customer_id\""},
{"from Table select Table.name"}, {"from Table select Table.name"},
{"DB where (name = \"Reporting\")"},
{"DB where (name = \"Reporting\") select name as _col_0, owner as _col_1"},
{"DB where DB is JdbcAccess"},
{"DB where DB has name"},
{"DB Table"},
{"DB where DB has name"},
{"DB as db1 Table where (db1.name = \"Reporting\")"},
{"DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
/*
todo: does not work
{"DB where (name = \"Reporting\") and ((createTime + 1) > 0)"},
{"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
{"DB as db1 Table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
{"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
{"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
*/
// trait searches
{"Dimension"},
/*{"Fact"}, - todo: does not work*/
{"JdbcAccess"},
{"ETL"},
{"Metric"},
{"PII"},
// Lineage
{"Table LoadProcess outputTable"},
{"Table loop (LoadProcess outputTable)"},
{"Table as _loop0 loop (LoadProcess outputTable) withPath"},
{"Table as src loop (LoadProcess outputTable) as dest select src.name as srcTable, dest.name as destTable withPath"},
}; };
} }
...@@ -201,7 +239,24 @@ public class GraphBackedDiscoveryServiceTest { ...@@ -201,7 +239,24 @@ public class GraphBackedDiscoveryServiceTest {
JSONArray rows = results.getJSONArray("rows"); JSONArray rows = results.getJSONArray("rows");
Assert.assertNotNull(rows); Assert.assertNotNull(rows);
Assert.assertTrue(rows.length() > 0); Assert.assertTrue(rows.length() >= 0); // some queries may not have any results
System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
}
@DataProvider(name = "invalidDslQueriesProvider")
private Object[][] createInvalidDSLQueries() {
return new String[][] {
{"from Unknown"},
{"Unknown"},
{"Unknown is Blah"},
};
}
@Test (dataProvider = "invalidDslQueriesProvider", expectedExceptions = DiscoveryException.class)
public void testSearchByDSLInvalidQueries(String dslQuery) throws Exception {
System.out.println("Executing dslQuery = " + dslQuery);
discoveryService.searchByDSL(dslQuery);
Assert.fail();
} }
@Test @Test
......
...@@ -44,7 +44,9 @@ import javax.ws.rs.core.MediaType; ...@@ -44,7 +44,9 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import java.util.List; import java.util.List;
/**
* Search Integration Tests.
*/
public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT { public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
@BeforeClass @BeforeClass
...@@ -96,7 +98,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT { ...@@ -96,7 +98,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
.type(MediaType.APPLICATION_JSON) .type(MediaType.APPLICATION_JSON)
.method(HttpMethod.GET, ClientResponse.class); .method(HttpMethod.GET, ClientResponse.class);
Assert.assertEquals(clientResponse.getStatus(), Assert.assertEquals(clientResponse.getStatus(),
Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()); Response.Status.BAD_REQUEST.getStatusCode());
} }
@Test @Test
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment