Skip to content

A
atlas
Commit 0dd414f2 by Shwetha GS
Options

BUG-38679 change hive lineage api's to require the syntax cluster.db.tablename

parent ec046d19
Showing with 91 additions and 150 deletions
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
......@@ -101,8 +101,8 @@ public class HiveMetaStoreBridge {
Database hiveDB = hiveClient.getDatabase(databaseName);
dbRef = new Referenceable(HiveDataTypes.HIVE_DB.getName());
dbRef.set("name", hiveDB.getName());
dbRef.set("clusterName", clusterName);
dbRef.set(HiveDataModelGenerator.NAME, hiveDB.getName().toLowerCase());
dbRef.set(HiveDataModelGenerator.CLUSTER_NAME, clusterName);
dbRef.set("description", hiveDB.getDescription());
dbRef.set("locationUri", hiveDB.getLocationUri());
dbRef.set("parameters", hiveDB.getParameters());
......@@ -139,7 +139,7 @@ public class HiveMetaStoreBridge {
// Import Partitions
Referenceable sdReferenceable = getSDForTable(databaseName, tableName);
importPartitions(databaseName, tableName, databaseReferenceable, tableReferenceable, sdReferenceable);
registerPartitions(databaseName, tableName, tableReferenceable, sdReferenceable);
// Import Indexes
importIndexes(databaseName, tableName, databaseReferenceable, tableReferenceable);
......@@ -159,8 +159,9 @@ public class HiveMetaStoreBridge {
LOG.debug("Getting reference for database {}", databaseName);
String typeName = HiveDataTypes.HIVE_DB.getName();
String dslQuery = String.format("%s where name = '%s' and clusterName = '%s'", typeName,
databaseName.toLowerCase(), clusterName);
String dslQuery = String.format("%s where %s = '%s' and %s = '%s'", typeName,
HiveDataModelGenerator.NAME, databaseName.toLowerCase(), HiveDataModelGenerator.CLUSTER_NAME,
clusterName);
return getEntityReferenceFromDSL(typeName, dslQuery);
}
......@@ -194,6 +195,10 @@ public class HiveMetaStoreBridge {
}
}
private String getTableName(String dbName, String tableName) {
return String.format("%s/%s.%s", clusterName, dbName.toLowerCase(), tableName.toLowerCase());
}
/**
* Gets reference for the table
*
......@@ -206,17 +211,9 @@ public class HiveMetaStoreBridge {
LOG.debug("Getting reference for table {}.{}", dbName, tableName);
String typeName = HiveDataTypes.HIVE_TABLE.getName();
String dslQuery = String.format(
"%s as t where name = '%s', dbName where name = '%s' and " + "clusterName = '%s' select t",
HiveDataTypes.HIVE_TABLE.getName(), tableName.toLowerCase(), dbName.toLowerCase(), clusterName);
String entityName = getTableName(dbName, tableName);
String dslQuery = String.format("%s as t where name = '%s'", typeName, entityName);
return getEntityReferenceFromDSL(typeName, dslQuery);
// String dbType = HiveDataTypes.HIVE_DB.getName();
// String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.name', '%s').as('t').out"
// + "('__%s.dbName').has('%s.name', '%s').has('%s.clusterName', '%s').back('t').toList()",
// typeName, typeName, tableName, typeName, dbType, dbName, dbType, clusterName);
// return getEntityReferenceFromGremlin(typeName, gremlinQuery);
}
private Referenceable getEntityReferenceFromGremlin(String typeName, String gremlinQuery) throws MetadataServiceException,
......@@ -241,13 +238,12 @@ public class HiveMetaStoreBridge {
// + "dbName where name = '%s' and clusterName = '%s' select p", typeName, valuesStr, tableName,
// dbName, clusterName);
String dbType = HiveDataTypes.HIVE_DB.getName();
String tableType = HiveDataTypes.HIVE_TABLE.getName();
String datasetType = MetadataServiceClient.DATA_SET_SUPER_TYPE;
String tableEntityName = getTableName(dbName, tableName);
String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', %s).as('p')."
+ "out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+ ".has('%s.clusterName', '%s').back('p').toList()", typeName, typeName, valuesStr, typeName,
datasetType, tableName.toLowerCase(), tableType, dbType, dbName.toLowerCase(), dbType, clusterName);
+ "out('__%s.table').has('%s.name', '%s').back('p').toList()", typeName, typeName, valuesStr,
typeName, datasetType, tableEntityName);
return getEntityReferenceFromGremlin(typeName, gremlinQuery);
}
......@@ -278,7 +274,8 @@ public class HiveMetaStoreBridge {
Table hiveTable = hiveClient.getTable(dbName, tableName);
tableRef = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
tableRef.set("name", hiveTable.getTableName());
tableRef.set(HiveDataModelGenerator.NAME, getTableName(hiveTable.getDbName(), hiveTable.getTableName()));
tableRef.set(HiveDataModelGenerator.TABLE_NAME, hiveTable.getTableName().toLowerCase());
tableRef.set("owner", hiveTable.getOwner());
tableRef.set("createTime", hiveTable.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME));
......@@ -288,7 +285,7 @@ public class HiveMetaStoreBridge {
tableRef.set(HiveDataModelGenerator.COMMENT, hiveTable.getParameters().get(HiveDataModelGenerator.COMMENT));
// add reference to the database
tableRef.set("dbName", dbReference);
tableRef.set(HiveDataModelGenerator.DB, dbReference);
List<Referenceable> colList = getColumns(hiveTable.getCols());
tableRef.set("columns", colList);
......@@ -323,15 +320,13 @@ public class HiveMetaStoreBridge {
return tableRef;
}
private void importPartitions(String db, String tableName,
Referenceable dbReferenceable,
Referenceable tableReferenceable,
Referenceable sdReferenceable) throws Exception {
private void registerPartitions(String db, String tableName, Referenceable tableReferenceable,
Referenceable sdReferenceable) throws Exception {
Set<Partition> tableParts = hiveClient.getAllPartitionsOf(new Table(Table.getEmptyTable(db, tableName)));
if (tableParts.size() > 0) {
for (Partition hivePart : tableParts) {
importPartition(hivePart, dbReferenceable, tableReferenceable, sdReferenceable);
registerPartition(hivePart, tableReferenceable, sdReferenceable);
}
}
}
......@@ -339,17 +334,14 @@ public class HiveMetaStoreBridge {
public Referenceable registerPartition(Partition partition) throws Exception {
String dbName = partition.getTable().getDbName();
String tableName = partition.getTable().getTableName();
Referenceable dbRef = registerDatabase(dbName);
Referenceable tableRef = registerTable(dbName, tableName);
Referenceable sdRef = getSDForTable(dbName, tableName);
return importPartition(partition, dbRef, tableRef, sdRef);
return registerPartition(partition, tableRef, sdRef);
}
private Referenceable importPartition(Partition hivePart,
Referenceable dbReferenceable,
Referenceable tableReferenceable,
Referenceable sdReferenceable) throws Exception {
LOG.info("Importing partition for {}.{} with values {}", dbReferenceable, tableReferenceable,
private Referenceable registerPartition(Partition hivePart, Referenceable tableReferenceable,
Referenceable sdReferenceable) throws Exception {
LOG.info("Registering partition for {} with values {}", tableReferenceable,
StringUtils.join(hivePart.getValues(), ","));
String dbName = hivePart.getTable().getDbName();
String tableName = hivePart.getTable().getTableName();
......@@ -359,8 +351,7 @@ public class HiveMetaStoreBridge {
partRef = new Referenceable(HiveDataTypes.HIVE_PARTITION.getName());
partRef.set("values", hivePart.getValues());
partRef.set("dbName", dbReferenceable);
partRef.set("tableName", tableReferenceable);
partRef.set(HiveDataModelGenerator.TABLE, tableReferenceable);
//todo fix
partRef.set("createTime", hivePart.getLastAccessTime());
......@@ -398,15 +389,15 @@ public class HiveMetaStoreBridge {
LOG.info("Importing index {} for {}.{}", index.getIndexName(), dbReferenceable, tableReferenceable);
Referenceable indexRef = new Referenceable(HiveDataTypes.HIVE_INDEX.getName());
indexRef.set("indexName", index.getIndexName());
indexRef.set(HiveDataModelGenerator.NAME, index.getIndexName());
indexRef.set("indexHandlerClass", index.getIndexHandlerClass());
indexRef.set("dbName", dbReferenceable);
indexRef.set(HiveDataModelGenerator.DB, dbReferenceable);
indexRef.set("createTime", index.getCreateTime());
indexRef.set("lastAccessTime", index.getLastAccessTime());
indexRef.set("origTableName", index.getOrigTableName());
indexRef.set("indexTableName", index.getIndexTableName());
indexRef.set("origTable", index.getOrigTableName());
indexRef.set("indexTable", index.getIndexTableName());
Referenceable sdReferenceable = fillStorageDescStruct(index.getSd(), null);
indexRef.set("sd", sdReferenceable);
......@@ -430,7 +421,7 @@ public class HiveMetaStoreBridge {
String serdeInfoName = HiveDataTypes.HIVE_SERDE.getName();
Struct serdeInfoStruct = new Struct(serdeInfoName);
serdeInfoStruct.set("name", serdeInfo.getName());
serdeInfoStruct.set(HiveDataModelGenerator.NAME, serdeInfo.getName());
serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib());
serdeInfoStruct.set("parameters", serdeInfo.getParameters());
......@@ -438,23 +429,6 @@ public class HiveMetaStoreBridge {
sdReferenceable.set(HiveDataModelGenerator.STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
sdReferenceable.set(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
// Will need to revisit this after we fix typesystem.
/*
LOG.info("skewedInfo = " + skewedInfo);
String skewedInfoName = HiveDataTypes.HIVE_SKEWEDINFO.name();
Struct skewedInfoStruct = new Struct(skewedInfoName);
if (skewedInfo.getSkewedColNames().size() > 0) {
skewedInfoStruct.set("skewedColNames", skewedInfo.getSkewedColNames());
skewedInfoStruct.set("skewedColValues", skewedInfo.getSkewedColValues());
skewedInfoStruct.set("skewedColValueLocationMaps",
skewedInfo.getSkewedColValueLocationMaps());
StructType skewedInfotype = (StructType) hiveTypeSystem.getDataType(skewedInfoName);
ITypedStruct skewedInfoStructTyped =
skewedInfotype.convert(skewedInfoStruct, Multiplicity.OPTIONAL);
sdStruct.set("skewedInfo", skewedInfoStructTyped);
}
*/
//Use the passed column list if not null, ex: use same references for table and SD
List<FieldSchema> columns = storageDesc.getCols();
if (columns != null && !columns.isEmpty()) {
......@@ -499,7 +473,7 @@ public class HiveMetaStoreBridge {
for (FieldSchema fs : schemaList) {
LOG.debug("Processing field " + fs);
Referenceable colReferenceable = new Referenceable(HiveDataTypes.HIVE_COLUMN.getName());
colReferenceable.set("name", fs.getName());
colReferenceable.set(HiveDataModelGenerator.NAME, fs.getName());
colReferenceable.set("type", fs.getType());
colReferenceable.set(HiveDataModelGenerator.COMMENT, fs.getComment());
......@@ -526,4 +500,12 @@ public class HiveMetaStoreBridge {
hiveMetaStoreBridge.registerHiveDataModel();
hiveMetaStoreBridge.importHiveMetadata();
}
public void updateTable(Referenceable tableReferenceable, Table newTable) throws MetadataServiceException {
MetadataServiceClient client = getMetadataServiceClient();
client.updateEntity(tableReferenceable.getId()._getId(), HiveDataModelGenerator.TABLE_NAME,
newTable.getTableName().toLowerCase());
client.updateEntity(tableReferenceable.getId()._getId(), HiveDataModelGenerator.NAME,
getTableName(newTable.getDbName(), newTable.getTableName()));
}
}
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
......@@ -37,7 +37,6 @@ package org.apache.hadoop.metadata.hive.hook;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryPlan;
......@@ -248,10 +247,9 @@ public class HiveHook implements ExecuteWithHookContext {
Referenceable dbReferenceable = dgiBridge.registerDatabase(oldTable.getDbName());
Referenceable tableReferenceable =
dgiBridge.registerTable(dbReferenceable, oldTable.getDbName(), oldTable.getTableName());
LOG.info("Updating entity name {}.{} to {}",
oldTable.getDbName(), oldTable.getTableName(), newTable.getTableName());
dgiBridge.getMetadataServiceClient().updateEntity(tableReferenceable.getId()._getId(), "name",
newTable.getTableName().toLowerCase());
LOG.info("Updating entity name {}.{} to {}", oldTable.getDbName(), oldTable.getTableName(),
newTable.getTableName());
dgiBridge.updateTable(tableReferenceable, newTable);
}
private void handleCreateTable(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
......
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataModelGenerator.java
......@@ -57,9 +57,16 @@ public class HiveDataModelGenerator {
private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
public static final String COMMENT = "comment";
public static final String STORAGE_NUM_BUCKETS = "numBuckets";
public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
public static final String NAME = "name";
public static final String TABLE_NAME = "tableName";
public static final String CLUSTER_NAME = "clusterName";
public static final String TABLE = "table";
public static final String DB = "db";
public HiveDataModelGenerator() {
classTypeDefinitions = new HashMap<>();
enumTypeDefinitionMap = new HashMap<>();
......@@ -72,7 +79,6 @@ public class HiveDataModelGenerator {
// enums
createHiveObjectTypeEnum();
createHivePrincipalTypeEnum();
createFunctionTypeEnum();
createResourceTypeEnum();
// structs
......@@ -89,7 +95,6 @@ public class HiveDataModelGenerator {
createPartitionClass();
createTableClass();
createIndexClass();
createFunctionClass();
createRoleClass();
// DDL/DML Process
......@@ -154,17 +159,6 @@ public class HiveDataModelGenerator {
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName());
}
private void createFunctionTypeEnum() throws MetadataException {
EnumValue values[] = {
new EnumValue("JAVA", 1),
};
EnumTypeDefinition definition = new EnumTypeDefinition(
HiveDataTypes.HIVE_FUNCTION_TYPE.getName(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_FUNCTION_TYPE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_FUNCTION_TYPE.getName());
}
private void createResourceTypeEnum() throws MetadataException {
EnumValue values[] = {
new EnumValue("JAR", 1),
......@@ -179,7 +173,7 @@ public class HiveDataModelGenerator {
private void createSerDeStruct() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(),
new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serializationLib", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
......@@ -192,29 +186,6 @@ public class HiveDataModelGenerator {
LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.getName());
}
/*
private static final DataTypes.ArrayType STRING_ARRAY_TYPE =
new DataTypes.ArrayType(DataTypes.STRING_TYPE);
private static Multiplicity ZeroOrMore = new Multiplicity(0, Integer.MAX_VALUE, true);
private void createSkewedInfoStruct() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("skewedColNames",
String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
ZeroOrMore, false, null),
new AttributeDefinition("skewedColValues",
String.format("array<%s>", STRING_ARRAY_TYPE.getName()),
ZeroOrMore, false, null),
new AttributeDefinition("skewedColValueLocationMaps", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
};
StructTypeDefinition definition = new StructTypeDefinition(
DefinedTypes.HIVE_SKEWEDINFO.getName(), attributeDefinitions);
structTypeDefinitionMap.put(DefinedTypes.HIVE_SKEWEDINFO.getName(), definition);
LOG.debug("Created definition for " + DefinedTypes.HIVE_SKEWEDINFO.getName());
}
*/
private void createOrderStruct() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("col", DataTypes.STRING_TYPE.getName(),
......@@ -283,9 +254,9 @@ public class HiveDataModelGenerator {
private void createDBClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(),
new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("clusterName", DataTypes.STRING_TYPE.getName(),
new AttributeDefinition(CLUSTER_NAME, DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("description", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
......@@ -308,7 +279,7 @@ public class HiveDataModelGenerator {
private void createTypeClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(),
new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("type1", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
......@@ -327,7 +298,7 @@ public class HiveDataModelGenerator {
private void createColumnClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(),
new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
......@@ -343,13 +314,10 @@ public class HiveDataModelGenerator {
}
private void createPartitionClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("tableName", HiveDataTypes.HIVE_TABLE.getName(),
new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
......@@ -372,7 +340,9 @@ public class HiveDataModelGenerator {
private void createTableClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.getName(),
new AttributeDefinition(TABLE_NAME, DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
......@@ -412,19 +382,19 @@ public class HiveDataModelGenerator {
private void createIndexClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("indexName", DataTypes.STRING_TYPE.getName(),
new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("indexHandlerClass", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.getName(),
new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("origTableName", HiveDataTypes.HIVE_TABLE.getName(),
new AttributeDefinition("origTable", HiveDataTypes.HIVE_TABLE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("indexTableName", HiveDataTypes.HIVE_TABLE.getName(),
new AttributeDefinition("indexTable", HiveDataTypes.HIVE_TABLE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
Multiplicity.REQUIRED, false, null),
......@@ -441,33 +411,6 @@ public class HiveDataModelGenerator {
LOG.debug("Created definition for " + HiveDataTypes.HIVE_INDEX.getName());
}
private void createFunctionClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("functionName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("className", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerName", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("functionType", HiveDataTypes.HIVE_FUNCTION_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("resourceUris",
DataTypes.arrayTypeName(HiveDataTypes.HIVE_RESOURCEURI.getName()), Multiplicity.OPTIONAL, false,
null),
};
HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
ClassType.class, HiveDataTypes.HIVE_FUNCTION.getName(), null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_FUNCTION.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_FUNCTION.getName());
}
private void createRoleClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("roleName", DataTypes.STRING_TYPE.getName(),
......
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataTypes.java
......@@ -27,11 +27,9 @@ public enum HiveDataTypes {
HIVE_OBJECT_TYPE,
HIVE_PRINCIPAL_TYPE,
HIVE_RESOURCE_TYPE,
HIVE_FUNCTION_TYPE,
// Structs
HIVE_SERDE,
HIVE_SKEWEDINFO,
HIVE_ORDER,
HIVE_RESOURCEURI,
......@@ -42,7 +40,6 @@ public enum HiveDataTypes {
HIVE_COLUMN,
HIVE_PARTITION,
HIVE_INDEX,
HIVE_FUNCTION,
HIVE_ROLE,
HIVE_TYPE,
HIVE_PROCESS,
......
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
......@@ -121,8 +121,8 @@ public class HiveHookIT {
private String createTable(boolean partition) throws Exception {
String tableName = tableName();
runCommand("create table " + tableName + "(id int, name string) comment 'table comment' " + (partition ? " partitioned by(dt string)"
: ""));
runCommand("create table " + tableName + "(id int, name string) comment 'table comment' "
+ (partition ? " partitioned by(dt string)" : ""));
return tableName;
}
......@@ -304,7 +304,7 @@ public class HiveHookIT {
private String assertTableIsRegistered(String dbName, String tableName, boolean registered) throws Exception {
LOG.debug("Searching for table {}.{}", dbName, tableName);
String query = String.format("%s as t where name = '%s', dbName where name = '%s' and clusterName = '%s'"
String query = String.format("%s as t where tableName = '%s', db where name = '%s' and clusterName = '%s'"
+ " select t", HiveDataTypes.HIVE_TABLE.getName(), tableName.toLowerCase(), dbName.toLowerCase(),
CLUSTER_NAME);
return assertEntityIsRegistered(query, registered);
......@@ -321,14 +321,13 @@ public class HiveHookIT {
String typeName = HiveDataTypes.HIVE_PARTITION.getName();
String dbType = HiveDataTypes.HIVE_DB.getName();
String tableType = HiveDataTypes.HIVE_TABLE.getName();
String datasetType = MetadataServiceClient.DATA_SET_SUPER_TYPE;
LOG.debug("Searching for partition of {}.{} with values {}", dbName, tableName, value);
//todo replace with DSL
String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', ['%s']).as('p')."
+ "out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+ "out('__%s.table').has('%s.tableName', '%s').out('__%s.db').has('%s.name', '%s')"
+ ".has('%s.clusterName', '%s').back('p').toList()", typeName, typeName, value, typeName,
datasetType, tableName.toLowerCase(), tableType, dbType, dbName.toLowerCase(), dbType, CLUSTER_NAME);
tableType, tableName.toLowerCase(), tableType, dbType, dbName.toLowerCase(), dbType, CLUSTER_NAME);
JSONObject response = dgiCLient.searchByGremlin(gremlinQuery);
JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
Assert.assertEquals(results.length(), 1);
......@@ -349,4 +348,21 @@ public class HiveHookIT {
return null;
}
}
@Test(enabled = false)
public void testLineage() throws Exception {
String table1 = createTable(false);
String db2 = createDatabase();
String table2 = tableName();
String db3 = createDatabase();
String table3 = tableName();
String query = String.format("create table %s.%s as select * from %s", db2, table2, table1);
runCommand(query);
query = String.format("create table %s.%s as select * from %s.%s", db3, table3, db2, table2);
runCommand(query);
}
}
client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
......@@ -64,6 +64,7 @@ public class MetadataServiceClient {
public static final String URI_ENTITIES = "entities";
public static final String URI_TRAITS = "traits";
public static final String URI_SEARCH = "discovery/search";
public static final String URI_LINEAGE = "lineage/hive";
public static final String QUERY = "query";
public static final String QUERY_TYPE = "queryType";
......@@ -128,6 +129,8 @@ public class MetadataServiceClient {
SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET),
SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET);
//Lineage operations
private final String method;
private final String path;
......@@ -255,6 +258,7 @@ public class MetadataServiceClient {
* @throws MetadataServiceException
*/
public JSONArray searchByDSL(String query) throws MetadataServiceException {
LOG.debug("DSL query: {}", query);
WebResource resource = getResource(API.SEARCH_DSL);
resource = resource.queryParam(QUERY, query);
JSONObject result = callAPIWithResource(API.SEARCH_DSL, resource);
......@@ -272,6 +276,7 @@ public class MetadataServiceClient {
* @throws MetadataServiceException
*/
public JSONObject searchByGremlin(String gremlinQuery) throws MetadataServiceException {
LOG.debug("Gremlin query: " + gremlinQuery);
WebResource resource = getResource(API.SEARCH_GREMLIN);
resource = resource.queryParam(QUERY, gremlinQuery);
return callAPIWithResource(API.SEARCH_GREMLIN, resource);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment