Commit 3e695bad by Shwetha GS

handling all hive query operations in hive hook

parent 1abeba45
...@@ -40,25 +40,6 @@ ...@@ -40,25 +40,6 @@
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-client</artifactId>
<version>${version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
<scope>runtime</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId> <artifactId>hadoop-minikdc</artifactId>
<version>${hadoop.version}</version> <version>${hadoop.version}</version>
...@@ -99,6 +80,25 @@ ...@@ -99,6 +80,25 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-client</artifactId>
<version>${version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
<scope>runtime</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId> <artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version> <version>${hadoop.version}</version>
......
...@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive; ...@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.metadata.MetadataServiceClient; import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.MetadataServiceException;
import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator; import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator;
import org.apache.hadoop.metadata.hive.model.HiveDataTypes; import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.apache.hadoop.metadata.typesystem.Referenceable; import org.apache.hadoop.metadata.typesystem.Referenceable;
...@@ -91,31 +92,6 @@ public class HiveMetaStoreBridge { ...@@ -91,31 +92,6 @@ public class HiveMetaStoreBridge {
} }
} }
/**
* Gets reference for the database
*
*
* @param databaseName
* @param clusterName cluster name
* @return Reference for database if exists, else null
* @throws Exception
*/
private Referenceable getDatabaseReference(String databaseName, String clusterName) throws Exception {
LOG.debug("Getting reference for database {}", databaseName);
String typeName = HiveDataTypes.HIVE_DB.getName();
MetadataServiceClient dgiClient = getMetadataServiceClient();
String dslQuery = String.format("%s where name = '%s' and clusterName = '%s'",
HiveDataTypes.HIVE_DB.getName(), databaseName, clusterName);
JSONArray results = dgiClient.searchByDSL(dslQuery);
if (results.length() == 0) {
return null;
} else {
String guid = getGuidFromDSLResponse(results.getJSONObject(0));
return new Referenceable(guid, typeName, null);
}
}
public Referenceable registerDatabase(String databaseName) throws Exception { public Referenceable registerDatabase(String databaseName) throws Exception {
Referenceable dbRef = getDatabaseReference(databaseName, clusterName); Referenceable dbRef = getDatabaseReference(databaseName, clusterName);
if (dbRef == null) { if (dbRef == null) {
...@@ -169,6 +145,35 @@ public class HiveMetaStoreBridge { ...@@ -169,6 +145,35 @@ public class HiveMetaStoreBridge {
} }
/** /**
* Gets reference for the database
*
*
* @param databaseName
* @param clusterName cluster name
* @return Reference for database if exists, else null
* @throws Exception
*/
private Referenceable getDatabaseReference(String databaseName, String clusterName) throws Exception {
LOG.debug("Getting reference for database {}", databaseName);
String typeName = HiveDataTypes.HIVE_DB.getName();
String dslQuery = String.format("%s where name = '%s' and clusterName = '%s'", HiveDataTypes.HIVE_DB.getName(),
databaseName, clusterName);
return getEntityReferenceFromDSL(typeName, dslQuery);
}
private Referenceable getEntityReferenceFromDSL(String typeName, String dslQuery) throws Exception {
MetadataServiceClient dgiClient = getMetadataServiceClient();
JSONArray results = dgiClient.searchByDSL(dslQuery);
if (results.length() == 0) {
return null;
} else {
String guid = getGuidFromDSLResponse(results.getJSONObject(0));
return new Referenceable(guid, typeName, null);
}
}
/**
* Gets reference for the table * Gets reference for the table
* *
* @param dbName * @param dbName
...@@ -180,19 +185,47 @@ public class HiveMetaStoreBridge { ...@@ -180,19 +185,47 @@ public class HiveMetaStoreBridge {
LOG.debug("Getting reference for table {}.{}", dbName, tableName); LOG.debug("Getting reference for table {}.{}", dbName, tableName);
String typeName = HiveDataTypes.HIVE_TABLE.getName(); String typeName = HiveDataTypes.HIVE_TABLE.getName();
MetadataServiceClient dgiClient = getMetadataServiceClient();
String query = String.format("%s where name = '%s', dbName where name = '%s' and clusterName = '%s'", // String dslQuery = String.format("%s as t where name = '%s' dbName where name = '%s' and "
HiveDataTypes.HIVE_TABLE.getName(), tableName, dbName, clusterName); // + "clusterName = '%s' select t",
JSONArray results = dgiClient.searchByDSL(query); // HiveDataTypes.HIVE_TABLE.getName(), tableName, dbName, clusterName);
String dbType = HiveDataTypes.HIVE_DB.getName();
String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.name', '%s').as('t').out"
+ "('__%s.dbName').has('%s.name', '%s').has('%s.clusterName', '%s').back('t').toList()",
typeName, typeName, tableName, typeName, dbType, dbName, dbType, clusterName);
return getEntityReferenceFromGremlin(typeName, gremlinQuery);
}
private Referenceable getEntityReferenceFromGremlin(String typeName, String gremlinQuery) throws MetadataServiceException,
JSONException {
MetadataServiceClient client = getMetadataServiceClient();
JSONObject response = client.searchByGremlin(gremlinQuery);
JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
if (results.length() == 0) { if (results.length() == 0) {
return null; return null;
} else { }
//There should be just one instance with the given name String guid = results.getJSONObject(0).getString("__guid");
String guid = getGuidFromDSLResponse(results.getJSONObject(0));
LOG.debug("Got reference for table {}.{} = {}", dbName, tableName, guid);
return new Referenceable(guid, typeName, null); return new Referenceable(guid, typeName, null);
} }
private Referenceable getPartitionReference(String dbName, String tableName, List<String> values) throws Exception {
String valuesStr = "['" + StringUtils.join(values, "', '") + "']";
LOG.debug("Getting reference for partition for {}.{} with values {}", dbName, tableName, valuesStr);
String typeName = HiveDataTypes.HIVE_PARTITION.getName();
// String dslQuery = String.format("%s as p where values = %s, tableName where name = '%s', "
// + "dbName where name = '%s' and clusterName = '%s' select p", typeName, valuesStr, tableName,
// dbName, clusterName);
String dbType = HiveDataTypes.HIVE_DB.getName();
String tableType = HiveDataTypes.HIVE_TABLE.getName();
String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', %s).as('p')."
+ "out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+ ".has('%s.clusterName', '%s').back('p').toList()", typeName, typeName, valuesStr, typeName,
tableType, tableName, tableType, dbType, dbName, dbType, clusterName);
return getEntityReferenceFromGremlin(typeName, gremlinQuery);
} }
private String getGuidFromDSLResponse(JSONObject jsonObject) throws JSONException { private String getGuidFromDSLResponse(JSONObject jsonObject) throws JSONException {
...@@ -292,14 +325,27 @@ public class HiveMetaStoreBridge { ...@@ -292,14 +325,27 @@ public class HiveMetaStoreBridge {
} }
} }
//todo should be idempotent public Referenceable registerPartition(Partition partition) throws Exception {
String dbName = partition.getTable().getDbName();
String tableName = partition.getTable().getTableName();
Referenceable dbRef = registerDatabase(dbName);
Referenceable tableRef = registerTable(dbName, tableName);
Referenceable sdRef = getSDForTable(dbName, tableName);
return importPartition(partition, dbRef, tableRef, sdRef);
}
private Referenceable importPartition(Partition hivePart, private Referenceable importPartition(Partition hivePart,
Referenceable dbReferenceable, Referenceable dbReferenceable,
Referenceable tableReferenceable, Referenceable tableReferenceable,
Referenceable sdReferenceable) throws Exception { Referenceable sdReferenceable) throws Exception {
LOG.info("Importing partition for {}.{} with values {}", dbReferenceable, tableReferenceable, LOG.info("Importing partition for {}.{} with values {}", dbReferenceable, tableReferenceable,
StringUtils.join(hivePart.getValues(), ",")); StringUtils.join(hivePart.getValues(), ","));
Referenceable partRef = new Referenceable(HiveDataTypes.HIVE_PARTITION.getName()); String dbName = hivePart.getTable().getDbName();
String tableName = hivePart.getTable().getTableName();
Referenceable partRef = getPartitionReference(dbName, tableName, hivePart.getValues());
if (partRef == null) {
partRef = new Referenceable(HiveDataTypes.HIVE_PARTITION.getName());
partRef.set("values", hivePart.getValues()); partRef.set("values", hivePart.getValues());
partRef.set("dbName", dbReferenceable); partRef.set("dbName", dbReferenceable);
...@@ -315,8 +361,12 @@ public class HiveMetaStoreBridge { ...@@ -315,8 +361,12 @@ public class HiveMetaStoreBridge {
partRef.set("sd", sdReferenceable); partRef.set("sd", sdReferenceable);
partRef.set("parameters", hivePart.getParameters()); partRef.set("parameters", hivePart.getParameters());
partRef = createInstance(partRef);
return createInstance(partRef); } else {
LOG.info("Partition {}.{} with values {} is already registered with id {}", dbName, tableName,
StringUtils.join(hivePart.getValues(), ","), partRef.getId().id);
}
return partRef;
} }
private void importIndexes(String db, String table, private void importIndexes(String db, String table,
......
...@@ -189,17 +189,28 @@ public class HiveHook implements ExecuteWithHookContext { ...@@ -189,17 +189,28 @@ public class HiveHook implements ExecuteWithHookContext {
switch (event.operation) { switch (event.operation) {
case CREATEDATABASE: case CREATEDATABASE:
Set<WriteEntity> outputs = event.outputs; handleCreateDB(dgiBridge, event);
for (WriteEntity entity : outputs) {
if (entity.getType() == Entity.Type.DATABASE) {
dgiBridge.registerDatabase(entity.getDatabase().getName());
}
}
break; break;
case CREATETABLE: case CREATETABLE:
outputs = event.outputs; handleCreateTable(dgiBridge, event);
for (WriteEntity entity : outputs) { break;
case CREATETABLE_AS_SELECT:
case CREATEVIEW:
case LOAD:
case EXPORT:
case IMPORT:
case QUERY:
registerProcess(dgiBridge, event);
break;
default:
}
}
private void handleCreateTable(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
for (WriteEntity entity : event.outputs) {
if (entity.getType() == Entity.Type.TABLE) { if (entity.getType() == Entity.Type.TABLE) {
Table table = entity.getTable(); Table table = entity.getTable();
...@@ -209,17 +220,17 @@ public class HiveHook implements ExecuteWithHookContext { ...@@ -209,17 +220,17 @@ public class HiveHook implements ExecuteWithHookContext {
dgiBridge.registerTable(dbReferenceable, table.getDbName(), table.getTableName()); dgiBridge.registerTable(dbReferenceable, table.getDbName(), table.getTableName());
} }
} }
break; }
case CREATETABLE_AS_SELECT:
registerCTAS(dgiBridge, event);
break;
default: private void handleCreateDB(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
for (WriteEntity entity : event.outputs) {
if (entity.getType() == Entity.Type.DATABASE) {
dgiBridge.registerDatabase(entity.getDatabase().getName());
}
} }
} }
private void registerCTAS(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception { private void registerProcess(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
Set<ReadEntity> inputs = event.inputs; Set<ReadEntity> inputs = event.inputs;
Set<WriteEntity> outputs = event.outputs; Set<WriteEntity> outputs = event.outputs;
...@@ -243,7 +254,7 @@ public class HiveHook implements ExecuteWithHookContext { ...@@ -243,7 +254,7 @@ public class HiveHook implements ExecuteWithHookContext {
processReferenceable.set("userName", event.user); processReferenceable.set("userName", event.user);
List<Referenceable> source = new ArrayList<>(); List<Referenceable> source = new ArrayList<>();
for (ReadEntity readEntity : inputs) { for (ReadEntity readEntity : inputs) {
if (readEntity.getTyp() == Entity.Type.TABLE) { if (readEntity.getType() == Entity.Type.TABLE) {
Table table = readEntity.getTable(); Table table = readEntity.getTable();
String dbName = table.getDbName().toLowerCase(); String dbName = table.getDbName().toLowerCase();
source.add(dgiBridge.registerTable(dbName, table.getTableName())); source.add(dgiBridge.registerTable(dbName, table.getTableName()));
...@@ -252,11 +263,14 @@ public class HiveHook implements ExecuteWithHookContext { ...@@ -252,11 +263,14 @@ public class HiveHook implements ExecuteWithHookContext {
processReferenceable.set("inputTables", source); processReferenceable.set("inputTables", source);
List<Referenceable> target = new ArrayList<>(); List<Referenceable> target = new ArrayList<>();
for (WriteEntity writeEntity : outputs) { for (WriteEntity writeEntity : outputs) {
if (writeEntity.getTyp() == Entity.Type.TABLE) { if (writeEntity.getType() == Entity.Type.TABLE || writeEntity.getType() == Entity.Type.PARTITION) {
Table table = writeEntity.getTable(); Table table = writeEntity.getTable();
String dbName = table.getDbName().toLowerCase(); String dbName = table.getDbName().toLowerCase();
target.add(dgiBridge.registerTable(dbName, table.getTableName())); target.add(dgiBridge.registerTable(dbName, table.getTableName()));
} }
if (writeEntity.getType() == Entity.Type.PARTITION) {
dgiBridge.registerPartition(writeEntity.getPartition());
}
} }
processReferenceable.set("outputTables", target); processReferenceable.set("outputTables", target);
processReferenceable.set("queryText", queryStr); processReferenceable.set("queryText", queryStr);
......
...@@ -26,10 +26,13 @@ import org.apache.hadoop.metadata.MetadataServiceClient; ...@@ -26,10 +26,13 @@ import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge; import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge;
import org.apache.hadoop.metadata.hive.model.HiveDataTypes; import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.io.File;
public class HiveHookIT { public class HiveHookIT {
private static final String DGI_URL = "http://localhost:21000/"; private static final String DGI_URL = "http://localhost:21000/";
private static final String CLUSTER_NAME = "test"; private static final String CLUSTER_NAME = "test";
...@@ -59,6 +62,9 @@ public class HiveHookIT { ...@@ -59,6 +62,9 @@ public class HiveHookIT {
hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true"); hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
hiveConf.set("hive.hook.dgi.synchronous", "true"); hiveConf.set("hive.hook.dgi.synchronous", "true");
hiveConf.set(HiveMetaStoreBridge.HIVE_CLUSTER_NAME, CLUSTER_NAME); hiveConf.set(HiveMetaStoreBridge.HIVE_CLUSTER_NAME, CLUSTER_NAME);
//weird, hive prepends test_ to table name
hiveConf.set("hive.test.mode", "true");
hiveConf.set("fs.pfile.impl", "org.apache.hadoop.fs.ProxyLocalFileSystem");
return hiveConf; return hiveConf;
} }
...@@ -69,7 +75,7 @@ public class HiveHookIT { ...@@ -69,7 +75,7 @@ public class HiveHookIT {
@Test @Test
public void testCreateDatabase() throws Exception { public void testCreateDatabase() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String dbName = "db" + random();
runCommand("create database " + dbName); runCommand("create database " + dbName);
assertDatabaseIsRegistered(dbName); assertDatabaseIsRegistered(dbName);
...@@ -77,15 +83,15 @@ public class HiveHookIT { ...@@ -77,15 +83,15 @@ public class HiveHookIT {
@Test @Test
public void testCreateTable() throws Exception { public void testCreateTable() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String dbName = "db" + random();
runCommand("create database " + dbName); runCommand("create database " + dbName);
String tableName = "table" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String tableName = "table" + random();
runCommand("create table " + dbName + "." + tableName + "(id int, name string)"); runCommand("create table " + dbName + "." + tableName + "(id int, name string)");
assertTableIsRegistered(dbName, tableName); assertTableIsRegistered(dbName, tableName);
tableName = "table" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string)"); runCommand("create table " + tableName + "(id int, name string) partitioned by(dt string)");
assertTableIsRegistered("default", tableName); assertTableIsRegistered("default", tableName);
//Create table where database doesn't exist, will create database instance as well //Create table where database doesn't exist, will create database instance as well
...@@ -94,10 +100,10 @@ public class HiveHookIT { ...@@ -94,10 +100,10 @@ public class HiveHookIT {
@Test @Test
public void testCTAS() throws Exception { public void testCTAS() throws Exception {
String tableName = "table" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string)"); runCommand("create table " + tableName + "(id int, name string)");
String ctasTableName = "table" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String ctasTableName = "table" + random();
String query = "create table " + ctasTableName + " as select * from " + tableName; String query = "create table " + ctasTableName + " as select * from " + tableName;
runCommand(query); runCommand(query);
...@@ -105,24 +111,125 @@ public class HiveHookIT { ...@@ -105,24 +111,125 @@ public class HiveHookIT {
assertProcessIsRegistered(query); assertProcessIsRegistered(query);
} }
@Test
public void testCreateView() throws Exception {
String tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string)");
String viewName = "table" + random();
String query = "create view " + viewName + " as select * from " + tableName;
runCommand(query);
assertTableIsRegistered("default", viewName);
assertProcessIsRegistered(query);
}
@Test
public void testLoadData() throws Exception {
String tableName = "table" + random();
runCommand("create table test_" + tableName + "(id int, name string)");
String loadFile = file("load");
String query = "load data local inpath 'file://" + loadFile + "' into table " + tableName;
runCommand(query);
assertProcessIsRegistered(query);
}
@Test
public void testInsert() throws Exception {
String tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string) partitioned by(dt string)");
String insertTableName = "table" + random();
runCommand("create table test_" + insertTableName + "(name string) partitioned by(dt string)");
String query = "insert into " + insertTableName + " partition(dt = '2015-01-01') select name from "
+ tableName + " where dt = '2015-01-01'";
runCommand(query);
assertProcessIsRegistered(query);
assertPartitionIsRegistered("default", "test_" + insertTableName, "2015-01-01");
}
private String random() {
return RandomStringUtils.randomAlphanumeric(5).toLowerCase();
}
private String file(String tag) throws Exception {
String filename = "./target/" + tag + "-data-" + random();
File file = new File(filename);
file.createNewFile();
return file.getAbsolutePath();
}
private String mkdir(String tag) throws Exception {
String filename = "./target/" + tag + "-data-" + random();
File file = new File(filename);
file.mkdirs();
return file.getAbsolutePath();
}
@Test
public void testExportImport() throws Exception {
String tableName = "table" + random();
runCommand("create table test_" + tableName + "(name string)");
String filename = "pfile://" + mkdir("export");
String query = "export table " + tableName + " to '" + filename + "'";
runCommand(query);
assertProcessIsRegistered(query);
tableName = "table" + random();
runCommand("create table " + tableName + "(name string)");
query = "import table " + tableName + " from '" + filename + "'";
runCommand(query);
assertProcessIsRegistered(query);
}
@Test
public void testSelect() throws Exception {
String tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string)");
String query = "select * from " + tableName;
runCommand(query);
assertProcessIsRegistered(query);
}
private void assertProcessIsRegistered(String queryStr) throws Exception { private void assertProcessIsRegistered(String queryStr) throws Exception {
String dslQuery = String.format("%s where queryText = '%s'", HiveDataTypes.HIVE_PROCESS.getName(), queryStr); String dslQuery = String.format("%s where queryText = \"%s\"", HiveDataTypes.HIVE_PROCESS.getName(), queryStr);
assertInstanceIsRegistered(dslQuery); assertEntityIsRegistered(dslQuery);
} }
private void assertTableIsRegistered(String dbName, String tableName) throws Exception { private void assertTableIsRegistered(String dbName, String tableName) throws Exception {
String query = String.format("%s where name = '%s', dbName where name = '%s' and clusterName = '%s'", String query = String.format("%s where name = '%s', dbName where name = '%s' and clusterName = '%s'",
HiveDataTypes.HIVE_TABLE.getName(), tableName, dbName, CLUSTER_NAME); HiveDataTypes.HIVE_TABLE.getName(), tableName, dbName, CLUSTER_NAME);
assertInstanceIsRegistered(query); assertEntityIsRegistered(query);
} }
private void assertDatabaseIsRegistered(String dbName) throws Exception { private void assertDatabaseIsRegistered(String dbName) throws Exception {
String query = String.format("%s where name = '%s' and clusterName = '%s'", HiveDataTypes.HIVE_DB.getName(), String query = String.format("%s where name = '%s' and clusterName = '%s'", HiveDataTypes.HIVE_DB.getName(),
dbName, CLUSTER_NAME); dbName, CLUSTER_NAME);
assertInstanceIsRegistered(query); assertEntityIsRegistered(query);
}
private void assertPartitionIsRegistered(String dbName, String tableName, String value) throws Exception {
String typeName = HiveDataTypes.HIVE_PARTITION.getName();
String dbType = HiveDataTypes.HIVE_DB.getName();
String tableType = HiveDataTypes.HIVE_TABLE.getName();
String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', ['%s']).as('p')."
+ "out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+ ".has('%s.clusterName', '%s').back('p').toList()", typeName, typeName, value, typeName,
tableType, tableName, tableType, dbType, dbName, dbType, CLUSTER_NAME);
JSONObject response = dgiCLient.searchByGremlin(gremlinQuery);
JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
Assert.assertEquals(results.length(), 1);
} }
private void assertInstanceIsRegistered(String dslQuery) throws Exception{ private void assertEntityIsRegistered(String dslQuery) throws Exception{
JSONArray results = dgiCLient.searchByDSL(dslQuery); JSONArray results = dgiCLient.searchByDSL(dslQuery);
Assert.assertEquals(results.length(), 1); Assert.assertEquals(results.length(), 1);
} }
......
...@@ -200,7 +200,7 @@ public class MetadataServiceClient { ...@@ -200,7 +200,7 @@ public class MetadataServiceClient {
public Referenceable getEntity(String guid) throws MetadataServiceException { public Referenceable getEntity(String guid) throws MetadataServiceException {
JSONObject jsonResponse = callAPI(API.GET_ENTITY, null, guid); JSONObject jsonResponse = callAPI(API.GET_ENTITY, null, guid);
try { try {
String entityInstanceDefinition = jsonResponse.getString(MetadataServiceClient.GUID); String entityInstanceDefinition = jsonResponse.getString(MetadataServiceClient.DEFINITION);
return InstanceSerialization.fromJsonReferenceable(entityInstanceDefinition, true); return InstanceSerialization.fromJsonReferenceable(entityInstanceDefinition, true);
} catch (JSONException e) { } catch (JSONException e) {
throw new MetadataServiceException(e); throw new MetadataServiceException(e);
......
...@@ -101,6 +101,9 @@ ...@@ -101,6 +101,9 @@
<StagingId>apache-staging</StagingId> <StagingId>apache-staging</StagingId>
<StagingName>Apache Release Distribution Repository</StagingName> <StagingName>Apache Release Distribution Repository</StagingName>
<StagingUrl>https://repository.apache.org/content/groups/staging</StagingUrl> <StagingUrl>https://repository.apache.org/content/groups/staging</StagingUrl>
<!-- skips checkstyle and find bugs -->
<skipCheck>false</skipCheck>
</properties> </properties>
<profiles> <profiles>
...@@ -971,6 +974,7 @@ ...@@ -971,6 +974,7 @@
</goals> </goals>
<phase>verify</phase> <phase>verify</phase>
<configuration> <configuration>
<skip>${skipCheck}</skip>
<consoleOutput>true</consoleOutput> <consoleOutput>true</consoleOutput>
<includeTestSourceDirectory>true</includeTestSourceDirectory> <includeTestSourceDirectory>true</includeTestSourceDirectory>
<configLocation>src/build/checkstyle.xml</configLocation> <configLocation>src/build/checkstyle.xml</configLocation>
...@@ -988,6 +992,7 @@ ...@@ -988,6 +992,7 @@
<!--debug>true</debug --> <!--debug>true</debug -->
<xmlOutput>true</xmlOutput> <xmlOutput>true</xmlOutput>
<failOnError>false</failOnError> <failOnError>false</failOnError>
<skip>${skipCheck}</skip>
</configuration> </configuration>
<executions> <executions>
<execution> <execution>
......
...@@ -23,6 +23,8 @@ import com.thinkaurelius.titan.core.TitanIndexQuery; ...@@ -23,6 +23,8 @@ import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.TitanProperty; import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex; import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.gremlin.groovy.Gremlin;
import com.tinkerpop.gremlin.java.GremlinPipeline;
import org.apache.hadoop.metadata.MetadataServiceClient; import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.discovery.DiscoveryException; import org.apache.hadoop.metadata.discovery.DiscoveryException;
import org.apache.hadoop.metadata.discovery.DiscoveryService; import org.apache.hadoop.metadata.discovery.DiscoveryService;
......
...@@ -199,7 +199,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -199,7 +199,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
break; break;
case ENUM: case ENUM:
createVertexMixedIndex(propertyName, Integer.class); createVertexMixedIndex(propertyName, String.class);
break; break;
case ARRAY: case ARRAY:
......
...@@ -85,7 +85,7 @@ ...@@ -85,7 +85,7 @@
</logger> </logger>
<root> <root>
<priority value="debug"/> <priority value="info"/>
<appender-ref ref="console"/> <appender-ref ref="console"/>
</root> </root>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment