Skip to content

A
atlas
Commit 3e695bad by Shwetha GS
Options

handling all hive query operations in hive hook

parent 1abeba45
Showing with 316 additions and 153 deletions
addons/hive-bridge/pom.xml
...@@ -40,25 +40,6 @@ ...@@ -40,25 +40,6 @@
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-client</artifactId>
<version>${version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
<scope>runtime</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId> <artifactId>hadoop-minikdc</artifactId>
<version>${hadoop.version}</version> <version>${hadoop.version}</version>
...@@ -99,6 +80,25 @@ ...@@ -99,6 +80,25 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-client</artifactId>
<version>${version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
<scope>runtime</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId> <artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version> <version>${hadoop.version}</version>
......
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
...@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive; ...@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.metadata.MetadataServiceClient; import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.MetadataServiceException;
import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator; import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator;
import org.apache.hadoop.metadata.hive.model.HiveDataTypes; import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.apache.hadoop.metadata.typesystem.Referenceable; import org.apache.hadoop.metadata.typesystem.Referenceable;
...@@ -91,31 +92,6 @@ public class HiveMetaStoreBridge { ...@@ -91,31 +92,6 @@ public class HiveMetaStoreBridge {
} }
} }
/**
* Gets reference for the database
*
*
* @param databaseName
* @param clusterName cluster name
* @return Reference for database if exists, else null
* @throws Exception
*/
private Referenceable getDatabaseReference(String databaseName, String clusterName) throws Exception {
LOG.debug("Getting reference for database {}", databaseName);
String typeName = HiveDataTypes.HIVE_DB.getName();
MetadataServiceClient dgiClient = getMetadataServiceClient();
String dslQuery = String.format("%s where name = '%s' and clusterName = '%s'",
HiveDataTypes.HIVE_DB.getName(), databaseName, clusterName);
JSONArray results = dgiClient.searchByDSL(dslQuery);
if (results.length() == 0) {
return null;
} else {
String guid = getGuidFromDSLResponse(results.getJSONObject(0));
return new Referenceable(guid, typeName, null);
}
}
public Referenceable registerDatabase(String databaseName) throws Exception { public Referenceable registerDatabase(String databaseName) throws Exception {
Referenceable dbRef = getDatabaseReference(databaseName, clusterName); Referenceable dbRef = getDatabaseReference(databaseName, clusterName);
if (dbRef == null) { if (dbRef == null) {
...@@ -169,6 +145,35 @@ public class HiveMetaStoreBridge { ...@@ -169,6 +145,35 @@ public class HiveMetaStoreBridge {
} }
/** /**
* Gets reference for the database
*
*
* @param databaseName
* @param clusterName cluster name
* @return Reference for database if exists, else null
* @throws Exception
*/
private Referenceable getDatabaseReference(String databaseName, String clusterName) throws Exception {
LOG.debug("Getting reference for database {}", databaseName);
String typeName = HiveDataTypes.HIVE_DB.getName();
String dslQuery = String.format("%s where name = '%s' and clusterName = '%s'", HiveDataTypes.HIVE_DB.getName(),
databaseName, clusterName);
return getEntityReferenceFromDSL(typeName, dslQuery);
}
private Referenceable getEntityReferenceFromDSL(String typeName, String dslQuery) throws Exception {
MetadataServiceClient dgiClient = getMetadataServiceClient();
JSONArray results = dgiClient.searchByDSL(dslQuery);
if (results.length() == 0) {
return null;
} else {
String guid = getGuidFromDSLResponse(results.getJSONObject(0));
return new Referenceable(guid, typeName, null);
}
}
/**
* Gets reference for the table * Gets reference for the table
* *
* @param dbName * @param dbName
...@@ -180,19 +185,47 @@ public class HiveMetaStoreBridge { ...@@ -180,19 +185,47 @@ public class HiveMetaStoreBridge {
LOG.debug("Getting reference for table {}.{}", dbName, tableName); LOG.debug("Getting reference for table {}.{}", dbName, tableName);
String typeName = HiveDataTypes.HIVE_TABLE.getName(); String typeName = HiveDataTypes.HIVE_TABLE.getName();
MetadataServiceClient dgiClient = getMetadataServiceClient();
String query = String.format("%s where name = '%s', dbName where name = '%s' and clusterName = '%s'", // String dslQuery = String.format("%s as t where name = '%s' dbName where name = '%s' and "
HiveDataTypes.HIVE_TABLE.getName(), tableName, dbName, clusterName); // + "clusterName = '%s' select t",
JSONArray results = dgiClient.searchByDSL(query); // HiveDataTypes.HIVE_TABLE.getName(), tableName, dbName, clusterName);
String dbType = HiveDataTypes.HIVE_DB.getName();
String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.name', '%s').as('t').out"
+ "('__%s.dbName').has('%s.name', '%s').has('%s.clusterName', '%s').back('t').toList()",
typeName, typeName, tableName, typeName, dbType, dbName, dbType, clusterName);
return getEntityReferenceFromGremlin(typeName, gremlinQuery);
}
private Referenceable getEntityReferenceFromGremlin(String typeName, String gremlinQuery) throws MetadataServiceException,
JSONException {
MetadataServiceClient client = getMetadataServiceClient();
JSONObject response = client.searchByGremlin(gremlinQuery);
JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
if (results.length() == 0) { if (results.length() == 0) {
return null; return null;
} else {
//There should be just one instance with the given name
String guid = getGuidFromDSLResponse(results.getJSONObject(0));
LOG.debug("Got reference for table {}.{} = {}", dbName, tableName, guid);
return new Referenceable(guid, typeName, null);
} }
String guid = results.getJSONObject(0).getString("__guid");
return new Referenceable(guid, typeName, null);
}
private Referenceable getPartitionReference(String dbName, String tableName, List<String> values) throws Exception {
String valuesStr = "['" + StringUtils.join(values, "', '") + "']";
LOG.debug("Getting reference for partition for {}.{} with values {}", dbName, tableName, valuesStr);
String typeName = HiveDataTypes.HIVE_PARTITION.getName();
// String dslQuery = String.format("%s as p where values = %s, tableName where name = '%s', "
// + "dbName where name = '%s' and clusterName = '%s' select p", typeName, valuesStr, tableName,
// dbName, clusterName);
String dbType = HiveDataTypes.HIVE_DB.getName();
String tableType = HiveDataTypes.HIVE_TABLE.getName();
String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', %s).as('p')."
+ "out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+ ".has('%s.clusterName', '%s').back('p').toList()", typeName, typeName, valuesStr, typeName,
tableType, tableName, tableType, dbType, dbName, dbType, clusterName);
return getEntityReferenceFromGremlin(typeName, gremlinQuery);
} }
private String getGuidFromDSLResponse(JSONObject jsonObject) throws JSONException { private String getGuidFromDSLResponse(JSONObject jsonObject) throws JSONException {
...@@ -292,31 +325,48 @@ public class HiveMetaStoreBridge { ...@@ -292,31 +325,48 @@ public class HiveMetaStoreBridge {
} }
} }
//todo should be idempotent public Referenceable registerPartition(Partition partition) throws Exception {
String dbName = partition.getTable().getDbName();
String tableName = partition.getTable().getTableName();
Referenceable dbRef = registerDatabase(dbName);
Referenceable tableRef = registerTable(dbName, tableName);
Referenceable sdRef = getSDForTable(dbName, tableName);
return importPartition(partition, dbRef, tableRef, sdRef);
}
private Referenceable importPartition(Partition hivePart, private Referenceable importPartition(Partition hivePart,
Referenceable dbReferenceable, Referenceable dbReferenceable,
Referenceable tableReferenceable, Referenceable tableReferenceable,
Referenceable sdReferenceable) throws Exception { Referenceable sdReferenceable) throws Exception {
LOG.info("Importing partition for {}.{} with values {}", dbReferenceable, tableReferenceable, LOG.info("Importing partition for {}.{} with values {}", dbReferenceable, tableReferenceable,
StringUtils.join(hivePart.getValues(), ",")); StringUtils.join(hivePart.getValues(), ","));
Referenceable partRef = new Referenceable(HiveDataTypes.HIVE_PARTITION.getName()); String dbName = hivePart.getTable().getDbName();
partRef.set("values", hivePart.getValues()); String tableName = hivePart.getTable().getTableName();
partRef.set("dbName", dbReferenceable); Referenceable partRef = getPartitionReference(dbName, tableName, hivePart.getValues());
partRef.set("tableName", tableReferenceable); if (partRef == null) {
partRef = new Referenceable(HiveDataTypes.HIVE_PARTITION.getName());
partRef.set("values", hivePart.getValues());
//todo fix partRef.set("dbName", dbReferenceable);
partRef.set("createTime", hivePart.getLastAccessTime()); partRef.set("tableName", tableReferenceable);
partRef.set("lastAccessTime", hivePart.getLastAccessTime());
// sdStruct = fillStorageDescStruct(hivePart.getSd()); //todo fix
// Instead of creating copies of the sdstruct for partitions we are reusing existing partRef.set("createTime", hivePart.getLastAccessTime());
// ones will fix to identify partitions with differing schema. partRef.set("lastAccessTime", hivePart.getLastAccessTime());
partRef.set("sd", sdReferenceable);
partRef.set("parameters", hivePart.getParameters()); // sdStruct = fillStorageDescStruct(hivePart.getSd());
// Instead of creating copies of the sdstruct for partitions we are reusing existing
// ones will fix to identify partitions with differing schema.
partRef.set("sd", sdReferenceable);
return createInstance(partRef); partRef.set("parameters", hivePart.getParameters());
partRef = createInstance(partRef);
} else {
LOG.info("Partition {}.{} with values {} is already registered with id {}", dbName, tableName,
StringUtils.join(hivePart.getValues(), ","), partRef.getId().id);
}
return partRef;
} }
private void importIndexes(String db, String table, private void importIndexes(String db, String table,
......
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
...@@ -189,37 +189,48 @@ public class HiveHook implements ExecuteWithHookContext { ...@@ -189,37 +189,48 @@ public class HiveHook implements ExecuteWithHookContext {
switch (event.operation) { switch (event.operation) {
case CREATEDATABASE: case CREATEDATABASE:
Set<WriteEntity> outputs = event.outputs; handleCreateDB(dgiBridge, event);
for (WriteEntity entity : outputs) {
if (entity.getType() == Entity.Type.DATABASE) {
dgiBridge.registerDatabase(entity.getDatabase().getName());
}
}
break; break;
case CREATETABLE: case CREATETABLE:
outputs = event.outputs; handleCreateTable(dgiBridge, event);
for (WriteEntity entity : outputs) {
if (entity.getType() == Entity.Type.TABLE) {
Table table = entity.getTable();
//TODO table.getDbName().toLowerCase() is required as hive stores in lowercase,
// but table.getDbName() is not lowercase
Referenceable dbReferenceable = dgiBridge.registerDatabase(table.getDbName().toLowerCase());
dgiBridge.registerTable(dbReferenceable, table.getDbName(), table.getTableName());
}
}
break; break;
case CREATETABLE_AS_SELECT: case CREATETABLE_AS_SELECT:
registerCTAS(dgiBridge, event); case CREATEVIEW:
case LOAD:
case EXPORT:
case IMPORT:
case QUERY:
registerProcess(dgiBridge, event);
break; break;
default: default:
} }
} }
private void registerCTAS(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception { private void handleCreateTable(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
for (WriteEntity entity : event.outputs) {
if (entity.getType() == Entity.Type.TABLE) {
Table table = entity.getTable();
//TODO table.getDbName().toLowerCase() is required as hive stores in lowercase,
// but table.getDbName() is not lowercase
Referenceable dbReferenceable = dgiBridge.registerDatabase(table.getDbName().toLowerCase());
dgiBridge.registerTable(dbReferenceable, table.getDbName(), table.getTableName());
}
}
}
private void handleCreateDB(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
for (WriteEntity entity : event.outputs) {
if (entity.getType() == Entity.Type.DATABASE) {
dgiBridge.registerDatabase(entity.getDatabase().getName());
}
}
}
private void registerProcess(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
Set<ReadEntity> inputs = event.inputs; Set<ReadEntity> inputs = event.inputs;
Set<WriteEntity> outputs = event.outputs; Set<WriteEntity> outputs = event.outputs;
...@@ -243,7 +254,7 @@ public class HiveHook implements ExecuteWithHookContext { ...@@ -243,7 +254,7 @@ public class HiveHook implements ExecuteWithHookContext {
processReferenceable.set("userName", event.user); processReferenceable.set("userName", event.user);
List<Referenceable> source = new ArrayList<>(); List<Referenceable> source = new ArrayList<>();
for (ReadEntity readEntity : inputs) { for (ReadEntity readEntity : inputs) {
if (readEntity.getTyp() == Entity.Type.TABLE) { if (readEntity.getType() == Entity.Type.TABLE) {
Table table = readEntity.getTable(); Table table = readEntity.getTable();
String dbName = table.getDbName().toLowerCase(); String dbName = table.getDbName().toLowerCase();
source.add(dgiBridge.registerTable(dbName, table.getTableName())); source.add(dgiBridge.registerTable(dbName, table.getTableName()));
...@@ -252,11 +263,14 @@ public class HiveHook implements ExecuteWithHookContext { ...@@ -252,11 +263,14 @@ public class HiveHook implements ExecuteWithHookContext {
processReferenceable.set("inputTables", source); processReferenceable.set("inputTables", source);
List<Referenceable> target = new ArrayList<>(); List<Referenceable> target = new ArrayList<>();
for (WriteEntity writeEntity : outputs) { for (WriteEntity writeEntity : outputs) {
if (writeEntity.getTyp() == Entity.Type.TABLE) { if (writeEntity.getType() == Entity.Type.TABLE || writeEntity.getType() == Entity.Type.PARTITION) {
Table table = writeEntity.getTable(); Table table = writeEntity.getTable();
String dbName = table.getDbName().toLowerCase(); String dbName = table.getDbName().toLowerCase();
target.add(dgiBridge.registerTable(dbName, table.getTableName())); target.add(dgiBridge.registerTable(dbName, table.getTableName()));
} }
if (writeEntity.getType() == Entity.Type.PARTITION) {
dgiBridge.registerPartition(writeEntity.getPartition());
}
} }
processReferenceable.set("outputTables", target); processReferenceable.set("outputTables", target);
processReferenceable.set("queryText", queryStr); processReferenceable.set("queryText", queryStr);
......
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
...@@ -26,10 +26,13 @@ import org.apache.hadoop.metadata.MetadataServiceClient; ...@@ -26,10 +26,13 @@ import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge; import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge;
import org.apache.hadoop.metadata.hive.model.HiveDataTypes; import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.io.File;
public class HiveHookIT { public class HiveHookIT {
private static final String DGI_URL = "http://localhost:21000/"; private static final String DGI_URL = "http://localhost:21000/";
private static final String CLUSTER_NAME = "test"; private static final String CLUSTER_NAME = "test";
...@@ -59,6 +62,9 @@ public class HiveHookIT { ...@@ -59,6 +62,9 @@ public class HiveHookIT {
hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true"); hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
hiveConf.set("hive.hook.dgi.synchronous", "true"); hiveConf.set("hive.hook.dgi.synchronous", "true");
hiveConf.set(HiveMetaStoreBridge.HIVE_CLUSTER_NAME, CLUSTER_NAME); hiveConf.set(HiveMetaStoreBridge.HIVE_CLUSTER_NAME, CLUSTER_NAME);
//weird, hive prepends test_ to table name
hiveConf.set("hive.test.mode", "true");
hiveConf.set("fs.pfile.impl", "org.apache.hadoop.fs.ProxyLocalFileSystem");
return hiveConf; return hiveConf;
} }
...@@ -69,7 +75,7 @@ public class HiveHookIT { ...@@ -69,7 +75,7 @@ public class HiveHookIT {
@Test @Test
public void testCreateDatabase() throws Exception { public void testCreateDatabase() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String dbName = "db" + random();
runCommand("create database " + dbName); runCommand("create database " + dbName);
assertDatabaseIsRegistered(dbName); assertDatabaseIsRegistered(dbName);
...@@ -77,15 +83,15 @@ public class HiveHookIT { ...@@ -77,15 +83,15 @@ public class HiveHookIT {
@Test @Test
public void testCreateTable() throws Exception { public void testCreateTable() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String dbName = "db" + random();
runCommand("create database " + dbName); runCommand("create database " + dbName);
String tableName = "table" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String tableName = "table" + random();
runCommand("create table " + dbName + "." + tableName + "(id int, name string)"); runCommand("create table " + dbName + "." + tableName + "(id int, name string)");
assertTableIsRegistered(dbName, tableName); assertTableIsRegistered(dbName, tableName);
tableName = "table" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string)"); runCommand("create table " + tableName + "(id int, name string) partitioned by(dt string)");
assertTableIsRegistered("default", tableName); assertTableIsRegistered("default", tableName);
//Create table where database doesn't exist, will create database instance as well //Create table where database doesn't exist, will create database instance as well
...@@ -94,10 +100,10 @@ public class HiveHookIT { ...@@ -94,10 +100,10 @@ public class HiveHookIT {
@Test @Test
public void testCTAS() throws Exception { public void testCTAS() throws Exception {
String tableName = "table" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string)"); runCommand("create table " + tableName + "(id int, name string)");
String ctasTableName = "table" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); String ctasTableName = "table" + random();
String query = "create table " + ctasTableName + " as select * from " + tableName; String query = "create table " + ctasTableName + " as select * from " + tableName;
runCommand(query); runCommand(query);
...@@ -105,24 +111,125 @@ public class HiveHookIT { ...@@ -105,24 +111,125 @@ public class HiveHookIT {
assertProcessIsRegistered(query); assertProcessIsRegistered(query);
} }
@Test
public void testCreateView() throws Exception {
String tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string)");
String viewName = "table" + random();
String query = "create view " + viewName + " as select * from " + tableName;
runCommand(query);
assertTableIsRegistered("default", viewName);
assertProcessIsRegistered(query);
}
@Test
public void testLoadData() throws Exception {
String tableName = "table" + random();
runCommand("create table test_" + tableName + "(id int, name string)");
String loadFile = file("load");
String query = "load data local inpath 'file://" + loadFile + "' into table " + tableName;
runCommand(query);
assertProcessIsRegistered(query);
}
@Test
public void testInsert() throws Exception {
String tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string) partitioned by(dt string)");
String insertTableName = "table" + random();
runCommand("create table test_" + insertTableName + "(name string) partitioned by(dt string)");
String query = "insert into " + insertTableName + " partition(dt = '2015-01-01') select name from "
+ tableName + " where dt = '2015-01-01'";
runCommand(query);
assertProcessIsRegistered(query);
assertPartitionIsRegistered("default", "test_" + insertTableName, "2015-01-01");
}
private String random() {
return RandomStringUtils.randomAlphanumeric(5).toLowerCase();
}
private String file(String tag) throws Exception {
String filename = "./target/" + tag + "-data-" + random();
File file = new File(filename);
file.createNewFile();
return file.getAbsolutePath();
}
private String mkdir(String tag) throws Exception {
String filename = "./target/" + tag + "-data-" + random();
File file = new File(filename);
file.mkdirs();
return file.getAbsolutePath();
}
@Test
public void testExportImport() throws Exception {
String tableName = "table" + random();
runCommand("create table test_" + tableName + "(name string)");
String filename = "pfile://" + mkdir("export");
String query = "export table " + tableName + " to '" + filename + "'";
runCommand(query);
assertProcessIsRegistered(query);
tableName = "table" + random();
runCommand("create table " + tableName + "(name string)");
query = "import table " + tableName + " from '" + filename + "'";
runCommand(query);
assertProcessIsRegistered(query);
}
@Test
public void testSelect() throws Exception {
String tableName = "table" + random();
runCommand("create table " + tableName + "(id int, name string)");
String query = "select * from " + tableName;
runCommand(query);
assertProcessIsRegistered(query);
}
private void assertProcessIsRegistered(String queryStr) throws Exception { private void assertProcessIsRegistered(String queryStr) throws Exception {
String dslQuery = String.format("%s where queryText = '%s'", HiveDataTypes.HIVE_PROCESS.getName(), queryStr); String dslQuery = String.format("%s where queryText = \"%s\"", HiveDataTypes.HIVE_PROCESS.getName(), queryStr);
assertInstanceIsRegistered(dslQuery); assertEntityIsRegistered(dslQuery);
} }
private void assertTableIsRegistered(String dbName, String tableName) throws Exception { private void assertTableIsRegistered(String dbName, String tableName) throws Exception {
String query = String.format("%s where name = '%s', dbName where name = '%s' and clusterName = '%s'", String query = String.format("%s where name = '%s', dbName where name = '%s' and clusterName = '%s'",
HiveDataTypes.HIVE_TABLE.getName(), tableName, dbName, CLUSTER_NAME); HiveDataTypes.HIVE_TABLE.getName(), tableName, dbName, CLUSTER_NAME);
assertInstanceIsRegistered(query); assertEntityIsRegistered(query);
} }
private void assertDatabaseIsRegistered(String dbName) throws Exception { private void assertDatabaseIsRegistered(String dbName) throws Exception {
String query = String.format("%s where name = '%s' and clusterName = '%s'", HiveDataTypes.HIVE_DB.getName(), String query = String.format("%s where name = '%s' and clusterName = '%s'", HiveDataTypes.HIVE_DB.getName(),
dbName, CLUSTER_NAME); dbName, CLUSTER_NAME);
assertInstanceIsRegistered(query); assertEntityIsRegistered(query);
}
private void assertPartitionIsRegistered(String dbName, String tableName, String value) throws Exception {
String typeName = HiveDataTypes.HIVE_PARTITION.getName();
String dbType = HiveDataTypes.HIVE_DB.getName();
String tableType = HiveDataTypes.HIVE_TABLE.getName();
String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', ['%s']).as('p')."
+ "out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+ ".has('%s.clusterName', '%s').back('p').toList()", typeName, typeName, value, typeName,
tableType, tableName, tableType, dbType, dbName, dbType, CLUSTER_NAME);
JSONObject response = dgiCLient.searchByGremlin(gremlinQuery);
JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
Assert.assertEquals(results.length(), 1);
} }
private void assertInstanceIsRegistered(String dslQuery) throws Exception{ private void assertEntityIsRegistered(String dslQuery) throws Exception{
JSONArray results = dgiCLient.searchByDSL(dslQuery); JSONArray results = dgiCLient.searchByDSL(dslQuery);
Assert.assertEquals(results.length(), 1); Assert.assertEquals(results.length(), 1);
} }
......
client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
...@@ -200,7 +200,7 @@ public class MetadataServiceClient { ...@@ -200,7 +200,7 @@ public class MetadataServiceClient {
public Referenceable getEntity(String guid) throws MetadataServiceException { public Referenceable getEntity(String guid) throws MetadataServiceException {
JSONObject jsonResponse = callAPI(API.GET_ENTITY, null, guid); JSONObject jsonResponse = callAPI(API.GET_ENTITY, null, guid);
try { try {
String entityInstanceDefinition = jsonResponse.getString(MetadataServiceClient.GUID); String entityInstanceDefinition = jsonResponse.getString(MetadataServiceClient.DEFINITION);
return InstanceSerialization.fromJsonReferenceable(entityInstanceDefinition, true); return InstanceSerialization.fromJsonReferenceable(entityInstanceDefinition, true);
} catch (JSONException e) { } catch (JSONException e) {
throw new MetadataServiceException(e); throw new MetadataServiceException(e);
......
pom.xml
...@@ -101,6 +101,9 @@ ...@@ -101,6 +101,9 @@
<StagingId>apache-staging</StagingId> <StagingId>apache-staging</StagingId>
<StagingName>Apache Release Distribution Repository</StagingName> <StagingName>Apache Release Distribution Repository</StagingName>
<StagingUrl>https://repository.apache.org/content/groups/staging</StagingUrl> <StagingUrl>https://repository.apache.org/content/groups/staging</StagingUrl>
<!-- skips checkstyle and find bugs -->
<skipCheck>false</skipCheck>
</properties> </properties>
<profiles> <profiles>
...@@ -971,6 +974,7 @@ ...@@ -971,6 +974,7 @@
</goals> </goals>
<phase>verify</phase> <phase>verify</phase>
<configuration> <configuration>
<skip>${skipCheck}</skip>
<consoleOutput>true</consoleOutput> <consoleOutput>true</consoleOutput>
<includeTestSourceDirectory>true</includeTestSourceDirectory> <includeTestSourceDirectory>true</includeTestSourceDirectory>
<configLocation>src/build/checkstyle.xml</configLocation> <configLocation>src/build/checkstyle.xml</configLocation>
...@@ -988,6 +992,7 @@ ...@@ -988,6 +992,7 @@
<!--debug>true</debug --> <!--debug>true</debug -->
<xmlOutput>true</xmlOutput> <xmlOutput>true</xmlOutput>
<failOnError>false</failOnError> <failOnError>false</failOnError>
<skip>${skipCheck}</skip>
</configuration> </configuration>
<executions> <executions>
<execution> <execution>
......
repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java
...@@ -23,6 +23,8 @@ import com.thinkaurelius.titan.core.TitanIndexQuery; ...@@ -23,6 +23,8 @@ import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.TitanProperty; import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex; import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.gremlin.groovy.Gremlin;
import com.tinkerpop.gremlin.java.GremlinPipeline;
import org.apache.hadoop.metadata.MetadataServiceClient; import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.discovery.DiscoveryException; import org.apache.hadoop.metadata.discovery.DiscoveryException;
import org.apache.hadoop.metadata.discovery.DiscoveryService; import org.apache.hadoop.metadata.discovery.DiscoveryService;
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
...@@ -40,6 +40,7 @@ import org.apache.hadoop.metadata.typesystem.persistence.MapIds; ...@@ -40,6 +40,7 @@ import org.apache.hadoop.metadata.typesystem.persistence.MapIds;
import org.apache.hadoop.metadata.typesystem.types.AttributeInfo; import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
import org.apache.hadoop.metadata.typesystem.types.ClassType; import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes; import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.EnumType;
import org.apache.hadoop.metadata.typesystem.types.EnumValue; import org.apache.hadoop.metadata.typesystem.types.EnumValue;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalType; import org.apache.hadoop.metadata.typesystem.types.HierarchicalType;
import org.apache.hadoop.metadata.typesystem.types.IDataType; import org.apache.hadoop.metadata.typesystem.types.IDataType;
...@@ -693,7 +694,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -693,7 +694,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break; break;
case ENUM: case ENUM:
addProperty(instanceVertex, propertyName, typedInstance.getInt(attributeInfo.name)); //handles both int and string for enum
EnumValue enumValue = (EnumValue) dataType.convert(typedInstance.get(attributeInfo.name),
Multiplicity.REQUIRED);
addProperty(instanceVertex, propertyName, enumValue.value);
break; break;
case ARRAY: case ARRAY:
...@@ -745,18 +749,15 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -745,18 +749,15 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
String propertyName = getQualifiedName(typedInstance, attributeInfo); String propertyName = getQualifiedName(typedInstance, attributeInfo);
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType(); IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
StringBuilder buffer = new StringBuilder(); List<String> values = new ArrayList(list.size());
Object[] array = list.toArray(); for (int index = 0; index < list.size(); index++) {
for (int index = 0; index < array.length; index++) {
String entryId = mapCollectionEntryToVertex(id, instanceVertex, String entryId = mapCollectionEntryToVertex(id, instanceVertex,
attributeInfo, idToVertexMap, elementType, array[index], attributeInfo, idToVertexMap, elementType, list.get(index), propertyName);
propertyName, String.valueOf(index)); values.add(entryId);
buffer.append(entryId).append(",");
} }
buffer.setLength(buffer.length() - 1);
// for dereference on way out // for dereference on way out
addProperty(instanceVertex, propertyName, buffer.toString()); addProperty(instanceVertex, propertyName, values);
} }
private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance, private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance,
...@@ -774,33 +775,27 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -774,33 +775,27 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} }
String propertyName = getQualifiedName(typedInstance, attributeInfo); String propertyName = getQualifiedName(typedInstance, attributeInfo);
StringBuilder buffer = new StringBuilder();
IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType(); IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
for (Map.Entry entry : collection.entrySet()) { for (Map.Entry entry : collection.entrySet()) {
String entryId = mapCollectionEntryToVertex(id, instanceVertex, attributeInfo, String myPropertyName = propertyName + "." + entry.getKey().toString();
idToVertexMap, elementType, entry.getValue(), mapCollectionEntryToVertex(id, instanceVertex, attributeInfo,
propertyName, String.valueOf(entry.getKey())); idToVertexMap, elementType, entry.getValue(), myPropertyName);
buffer.append(entryId).append(",");
} }
buffer.setLength(buffer.length() - 1);
// for dereference on way out // for dereference on way out
addProperty(instanceVertex, propertyName, buffer.toString()); addProperty(instanceVertex, propertyName, new ArrayList(collection.keySet()));
} }
private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex, private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex,
AttributeInfo attributeInfo, AttributeInfo attributeInfo,
Map<Id, Vertex> idToVertexMap, Map<Id, Vertex> idToVertexMap,
IDataType elementType, Object value, IDataType elementType, Object value,
String propertyName, String propertyName) throws MetadataException {
String key) throws MetadataException {
final String propertyNameWithSuffix = propertyName + "." + key;
final String edgeLabel = EDGE_LABEL_PREFIX + propertyName; final String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
switch (elementType.getTypeCategory()) { switch (elementType.getTypeCategory()) {
case PRIMITIVE: case PRIMITIVE:
case ENUM: case ENUM:
addProperty(instanceVertex, propertyNameWithSuffix, value); return value.toString();
return propertyNameWithSuffix;
case ARRAY: case ARRAY:
case MAP: case MAP:
...@@ -814,13 +809,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -814,13 +809,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
// add an edge to the newly created vertex from the parent // add an edge to the newly created vertex from the parent
Edge structElementEdge = GraphHelper.addEdge( Edge structElementEdge = GraphHelper.addEdge(
titanGraph, instanceVertex, structInstanceVertex, edgeLabel); titanGraph, instanceVertex, structInstanceVertex, edgeLabel);
return propertyName + "." + key + ":" + structElementEdge.getId(); return structElementEdge.getId().toString();
case CLASS: case CLASS:
Id referenceId = (Id) value; Id referenceId = (Id) value;
String edgeId = mapClassReferenceAsEdge( return mapClassReferenceAsEdge(
instanceVertex, idToVertexMap, edgeLabel, referenceId); instanceVertex, idToVertexMap, edgeLabel, referenceId);
return propertyName + "." + key + ":" + edgeId;
default: default:
throw new IllegalArgumentException("Unknown type category: " throw new IllegalArgumentException("Unknown type category: "
...@@ -1003,8 +997,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -1003,8 +997,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
return; return;
} }
typedInstance.setInt(attributeInfo.name, typedInstance.set(attributeInfo.name, dataType.convert(instanceVertex.<String>getProperty
instanceVertex.<Integer>getProperty(vertexPropertyName)); (vertexPropertyName), Multiplicity.REQUIRED));
break; break;
case ARRAY: case ARRAY:
...@@ -1071,17 +1065,17 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -1071,17 +1065,17 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
AttributeInfo attributeInfo, AttributeInfo attributeInfo,
String propertyName) throws MetadataException { String propertyName) throws MetadataException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name); LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
String keys = instanceVertex.getProperty(propertyName); List list = instanceVertex.getProperty(propertyName);
if (keys == null || keys.length() == 0) { if (list == null || list.size() == 0) {
return; return;
} }
DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType(); DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
final IDataType elementType = arrayType.getElemType(); final IDataType elementType = arrayType.getElemType();
ArrayList values = new ArrayList(); ArrayList values = new ArrayList();
for (String propertyNameWithSuffix : keys.split(",")) { for (Object listElement : list) {
values.add(mapVertexToCollectionEntry(instanceVertex, attributeInfo, values.add(mapVertexToCollectionEntry(instanceVertex, attributeInfo, elementType, listElement,
elementType, propertyName, propertyNameWithSuffix)); propertyName));
} }
typedInstance.set(attributeInfo.name, values); typedInstance.set(attributeInfo.name, values);
...@@ -1089,20 +1083,13 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -1089,20 +1083,13 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public Object mapVertexToCollectionEntry(Vertex instanceVertex, public Object mapVertexToCollectionEntry(Vertex instanceVertex,
AttributeInfo attributeInfo, AttributeInfo attributeInfo,
IDataType elementType, IDataType elementType, Object value, String propertyName)
String propertyName,
String propertyNameWithSuffix)
throws MetadataException { throws MetadataException {
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
final String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
final String edgeId = propertyNameWithSuffix
.substring(propertyNameWithSuffix.lastIndexOf(":") + 1, propertyNameWithSuffix.length());
switch (elementType.getTypeCategory()) { switch (elementType.getTypeCategory()) {
case PRIMITIVE: case PRIMITIVE:
return instanceVertex.getProperty(propertyNameWithSuffix);
case ENUM: case ENUM:
return instanceVertex.<Integer>getProperty(propertyNameWithSuffix); return value;
case ARRAY: case ARRAY:
case MAP: case MAP:
...@@ -1112,11 +1099,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -1112,11 +1099,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
case STRUCT: case STRUCT:
return getStructInstanceFromVertex(instanceVertex, return getStructInstanceFromVertex(instanceVertex,
elementType, attributeInfo.name, edgeLabel, edgeId); elementType, attributeInfo.name, edgeLabel, (String) value);
case CLASS: case CLASS:
return mapClassReferenceToVertex( return mapClassReferenceToVertex(
instanceVertex, attributeInfo, edgeLabel, elementType, edgeId); instanceVertex, attributeInfo, edgeLabel, elementType, (String) value);
default: default:
break; break;
...@@ -1130,8 +1117,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -1130,8 +1117,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
AttributeInfo attributeInfo, AttributeInfo attributeInfo,
String propertyName) throws MetadataException { String propertyName) throws MetadataException {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name); LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
String keys = instanceVertex.getProperty(propertyName); List<String> keys = instanceVertex.getProperty(propertyName);
if (keys == null || keys.length() == 0) { if (keys == null || keys.size() == 0) {
return; return;
} }
DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType(); DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
...@@ -1139,10 +1126,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -1139,10 +1126,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
final IDataType valueType = mapType.getValueType(); final IDataType valueType = mapType.getValueType();
HashMap values = new HashMap(); HashMap values = new HashMap();
for (String propertyNameWithSuffix : keys.split(",")) { for (String key : keys) {
final String key = extractKey(propertyNameWithSuffix, keyType);
values.put(key, mapVertexToCollectionEntry(instanceVertex, attributeInfo, values.put(key, mapVertexToCollectionEntry(instanceVertex, attributeInfo,
valueType, propertyName, propertyNameWithSuffix)); valueType, propertyName, propertyName));
} }
typedInstance.set(attributeInfo.name, values); typedInstance.set(attributeInfo.name, values);
...@@ -1158,8 +1144,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -1158,8 +1144,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex, private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex,
IDataType elemType, IDataType elemType,
String attributeName, String attributeName, String relationshipLabel,
String relationshipLabel,
String edgeId) throws MetadataException { String edgeId) throws MetadataException {
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel); LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) { for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
...@@ -199,7 +199,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -199,7 +199,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
break; break;
case ENUM: case ENUM:
createVertexMixedIndex(propertyName, Integer.class); createVertexMixedIndex(propertyName, String.class);
break; break;
case ARRAY: case ARRAY:
......
typesystem/src/main/resources/log4j.xml
...@@ -85,7 +85,7 @@ ...@@ -85,7 +85,7 @@
</logger> </logger>
<root> <root>
<priority value="debug"/> <priority value="info"/>
<appender-ref ref="console"/> <appender-ref ref="console"/>
</root> </root>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment