Skip to content

A
atlas
Commit a86f0f3e by Venkatesh Seetharam
Options

Add Hive Bridge with major refactoring

parent e747e2ae
Showing with 698 additions and 344 deletions
addons/hive-bridge/pom.xml
...@@ -34,10 +34,22 @@ ...@@ -34,10 +34,22 @@
<properties> <properties>
<hive.version>0.14.0</hive.version> <hive.version>0.14.0</hive.version>
<hadoop.version>2.5.0</hadoop.version>
</properties> </properties>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.apache.hadoop.metadata</groupId>
<artifactId>metadata-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId> <artifactId>hive-metastore</artifactId>
<version>${hive.version}</version> <version>${hive.version}</version>
...@@ -96,11 +108,6 @@ ...@@ -96,11 +108,6 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
<groupId>log4j</groupId> <groupId>log4j</groupId>
<artifactId>log4j</artifactId> <artifactId>log4j</artifactId>
</dependency> </dependency>
......
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.hive.bridge;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.metadata.MetadataServiceClient;
import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.apache.hadoop.metadata.typesystem.Struct;
import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
/**
* A Bridge Utility that imports metadata from the Hive Meta Store
* and registers then in DGI.
*/
public class HiveMetaStoreBridge {
private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
private final HiveMetaStoreClient hiveMetaStoreClient;
private final MetadataServiceClient metadataServiceClient;
/**
* Construct a HiveMetaStoreBridge.
* @param baseUrl metadata service url
*/
public HiveMetaStoreBridge(String baseUrl) throws Exception {
hiveMetaStoreClient = createHiveMetaStoreClient();
metadataServiceClient = new MetadataServiceClient(baseUrl);
}
private HiveMetaStoreClient createHiveMetaStoreClient() throws Exception {
HiveConf conf = new HiveConf();
return new HiveMetaStoreClient(conf);
}
public void importHiveMetadata() throws Exception {
LOG.info("Importing hive metadata");
importDatabases();
}
private void importDatabases() throws Exception {
List<String> databases = hiveMetaStoreClient.getAllDatabases();
for (String databaseName : databases) {
importDatabase(databaseName);
}
}
private void importDatabase(String databaseName) throws Exception {
LOG.info("Importing objects from databaseName : " + databaseName);
Database hiveDB = hiveMetaStoreClient.getDatabase(databaseName);
Referenceable dbRef = new Referenceable(HiveDataTypes.HIVE_DB.name());
dbRef.set("name", hiveDB.getName());
dbRef.set("description", hiveDB.getDescription());
dbRef.set("locationUri", hiveDB.getLocationUri());
dbRef.set("parameters", hiveDB.getParameters());
dbRef.set("ownerName", hiveDB.getOwnerName());
dbRef.set("ownerType", hiveDB.getOwnerType().getValue());
Referenceable databaseReferenceable = createInstance(dbRef);
importTables(databaseName, databaseReferenceable);
}
private Referenceable createInstance(Referenceable referenceable) throws Exception {
String typeName = referenceable.getTypeName();
LOG.debug("creating instance of type " + typeName);
String entityJSON = InstanceSerialization.toJson(referenceable, true);
LOG.debug("Submitting new entity= " + entityJSON);
JSONObject jsonObject = metadataServiceClient.createEntity(entityJSON);
String guid = jsonObject.getString(MetadataServiceClient.RESULTS);
LOG.debug("created instance for type " + typeName + ", guid: " + guid);
return new Referenceable(guid, referenceable.getTypeName(), referenceable.getValuesMap());
}
private void importTables(String databaseName,
Referenceable databaseReferenceable) throws Exception {
List<String> hiveTables = hiveMetaStoreClient.getAllTables(databaseName);
for (String tableName : hiveTables) {
importTable(databaseName, tableName, databaseReferenceable);
}
}
private void importTable(String db, String tableName,
Referenceable databaseReferenceable) throws Exception {
LOG.info("Importing objects from " + db + "." + tableName);
Table hiveTable = hiveMetaStoreClient.getTable(db, tableName);
Referenceable tableRef = new Referenceable(HiveDataTypes.HIVE_TABLE.name());
tableRef.set("tableName", hiveTable.getTableName());
tableRef.set("owner", hiveTable.getOwner());
tableRef.set("createTime", hiveTable.getCreateTime());
tableRef.set("lastAccessTime", hiveTable.getLastAccessTime());
tableRef.set("retention", hiveTable.getRetention());
// add reference to the database
tableRef.set("dbName", databaseReferenceable);
// add reference to the StorageDescriptor
StorageDescriptor storageDesc = hiveTable.getSd();
Referenceable sdReferenceable = fillStorageDescStruct(storageDesc);
tableRef.set("sd", sdReferenceable);
// add reference to the Partition Keys
List<Referenceable> partKeys = new ArrayList<>();
Referenceable colRef;
if (hiveTable.getPartitionKeysSize() > 0) {
for (FieldSchema fs : hiveTable.getPartitionKeys()) {
colRef = new Referenceable(HiveDataTypes.HIVE_COLUMN.name());
colRef.set("name", fs.getName());
colRef.set("type", fs.getType());
colRef.set("comment", fs.getComment());
Referenceable colRefTyped = createInstance(colRef);
partKeys.add(colRefTyped);
}
tableRef.set("partitionKeys", partKeys);
}
tableRef.set("parameters", hiveTable.getParameters());
if (hiveTable.isSetViewOriginalText()) {
tableRef.set("viewOriginalText", hiveTable.getViewOriginalText());
}
if (hiveTable.isSetViewExpandedText()) {
tableRef.set("viewExpandedText", hiveTable.getViewExpandedText());
}
tableRef.set("tableType", hiveTable.getTableType());
tableRef.set("temporary", hiveTable.isTemporary());
Referenceable tableReferenceable = createInstance(tableRef);
// Import Partitions
importPartitions(db, tableName, databaseReferenceable, tableReferenceable, sdReferenceable);
// Import Indexes
importIndexes(db, tableName, databaseReferenceable, tableRef);
}
private void importPartitions(String db, String table,
Referenceable dbReferenceable,
Referenceable tableReferenceable,
Referenceable sdReferenceable) throws Exception {
List<Partition> tableParts = hiveMetaStoreClient.listPartitions(
db, table, Short.MAX_VALUE);
if (tableParts.size() > 0) {
for (Partition hivePart : tableParts) {
importPartition(hivePart, dbReferenceable, tableReferenceable, sdReferenceable);
}
}
}
private Referenceable importPartition(Partition hivePart,
Referenceable dbReferenceable,
Referenceable tableReferenceable,
Referenceable sdReferenceable) throws Exception {
Referenceable partRef = new Referenceable(HiveDataTypes.HIVE_PARTITION.name());
partRef.set("values", hivePart.getValues());
partRef.set("dbName", dbReferenceable);
partRef.set("tableName", tableReferenceable);
partRef.set("createTime", hivePart.getCreateTime());
partRef.set("lastAccessTime", hivePart.getLastAccessTime());
// sdStruct = fillStorageDescStruct(hivePart.getSd());
// Instead of creating copies of the sdstruct for partitions we are reusing existing
// ones will fix to identify partitions with differing schema.
partRef.set("sd", sdReferenceable);
partRef.set("parameters", hivePart.getParameters());
return createInstance(partRef);
}
private void importIndexes(String db, String table,
Referenceable dbReferenceable,
Referenceable tableReferenceable) throws Exception {
List<Index> indexes = hiveMetaStoreClient.listIndexes(db, table, Short.MAX_VALUE);
if (indexes.size() > 0) {
for (Index index : indexes) {
importIndex(index, dbReferenceable, tableReferenceable);
}
}
}
private void importIndex(Index index,
Referenceable dbReferenceable,
Referenceable tableReferenceable) throws Exception {
Referenceable indexRef = new Referenceable(HiveDataTypes.HIVE_INDEX.name());
indexRef.set("indexName", index.getIndexName());
indexRef.set("indexHandlerClass", index.getIndexHandlerClass());
indexRef.set("dbName", dbReferenceable);
indexRef.set("createTime", index.getCreateTime());
indexRef.set("lastAccessTime", index.getLastAccessTime());
indexRef.set("origTableName", index.getOrigTableName());
indexRef.set("indexTableName", index.getIndexTableName());
Referenceable sdReferenceable = fillStorageDescStruct(index.getSd());
indexRef.set("sd", sdReferenceable);
indexRef.set("parameters", index.getParameters());
tableReferenceable.set("deferredRebuild", index.isDeferredRebuild());
createInstance(indexRef);
}
private Referenceable fillStorageDescStruct(StorageDescriptor storageDesc) throws Exception {
LOG.debug("Filling storage descriptor information for " + storageDesc);
Referenceable sdReferenceable = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.name());
SerDeInfo serdeInfo = storageDesc.getSerdeInfo();
LOG.debug("serdeInfo = " + serdeInfo);
// SkewedInfo skewedInfo = storageDesc.getSkewedInfo();
String serdeInfoName = HiveDataTypes.HIVE_SERDE.name();
Struct serdeInfoStruct = new Struct(serdeInfoName);
serdeInfoStruct.set("name", serdeInfo.getName());
serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib());
serdeInfoStruct.set("parameters", serdeInfo.getParameters());
sdReferenceable.set("serdeInfo", serdeInfoStruct);
// Will need to revisit this after we fix typesystem.
/*
LOG.info("skewedInfo = " + skewedInfo);
String skewedInfoName = HiveDataTypes.HIVE_SKEWEDINFO.name();
Struct skewedInfoStruct = new Struct(skewedInfoName);
if (skewedInfo.getSkewedColNames().size() > 0) {
skewedInfoStruct.set("skewedColNames", skewedInfo.getSkewedColNames());
skewedInfoStruct.set("skewedColValues", skewedInfo.getSkewedColValues());
skewedInfoStruct.set("skewedColValueLocationMaps",
skewedInfo.getSkewedColValueLocationMaps());
StructType skewedInfotype = (StructType) hiveTypeSystem.getDataType(skewedInfoName);
ITypedStruct skewedInfoStructTyped =
skewedInfotype.convert(skewedInfoStruct, Multiplicity.OPTIONAL);
sdStruct.set("skewedInfo", skewedInfoStructTyped);
}
*/
List<Referenceable> fieldsList = new ArrayList<>();
Referenceable colReferenceable;
for (FieldSchema fs : storageDesc.getCols()) {
LOG.debug("Processing field " + fs);
colReferenceable = new Referenceable(HiveDataTypes.HIVE_COLUMN.name());
colReferenceable.set("name", fs.getName());
colReferenceable.set("type", fs.getType());
colReferenceable.set("comment", fs.getComment());
fieldsList.add(createInstance(colReferenceable));
}
sdReferenceable.set("cols", fieldsList);
List<Struct> sortColsStruct = new ArrayList<>();
for (Order sortcol : storageDesc.getSortCols()) {
String hiveOrderName = HiveDataTypes.HIVE_ORDER.name();
Struct colStruct = new Struct(hiveOrderName);
colStruct.set("col", sortcol.getCol());
colStruct.set("order", sortcol.getOrder());
sortColsStruct.add(colStruct);
}
if (sortColsStruct.size() > 0) {
sdReferenceable.set("sortCols", sortColsStruct);
}
sdReferenceable.set("location", storageDesc.getLocation());
sdReferenceable.set("inputFormat", storageDesc.getInputFormat());
sdReferenceable.set("outputFormat", storageDesc.getOutputFormat());
sdReferenceable.set("compressed", storageDesc.isCompressed());
if (storageDesc.getBucketCols().size() > 0) {
sdReferenceable.set("bucketCols", storageDesc.getBucketCols());
}
sdReferenceable.set("parameters", storageDesc.getParameters());
sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories());
return createInstance(sdReferenceable);
}
static String getServerUrl(String[] args) {
String baseUrl = "http://localhost:21000";
if (args.length > 0) {
baseUrl = args[0];
}
return baseUrl;
}
public static void main(String[] argv) throws Exception {
String baseUrl = getServerUrl(argv);
HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(baseUrl);
hiveMetaStoreBridge.importHiveMetadata();
}
}
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataModelGenerator.java
...@@ -127,9 +127,9 @@ public class HiveDataModelGenerator { ...@@ -127,9 +127,9 @@ public class HiveDataModelGenerator {
}; };
EnumTypeDefinition definition = new EnumTypeDefinition( EnumTypeDefinition definition = new EnumTypeDefinition(
HiveDataTypes.HIVE_OBJECTTYPE.name(), values); HiveDataTypes.HIVE_OBJECT_TYPE.getName(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_OBJECTTYPE.name(), definition); enumTypeDefinitionMap.put(HiveDataTypes.HIVE_OBJECT_TYPE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_OBJECTTYPE.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_OBJECT_TYPE.getName());
} }
private void createHivePrincipalTypeEnum() throws MetadataException { private void createHivePrincipalTypeEnum() throws MetadataException {
...@@ -140,10 +140,10 @@ public class HiveDataModelGenerator { ...@@ -140,10 +140,10 @@ public class HiveDataModelGenerator {
}; };
EnumTypeDefinition definition = new EnumTypeDefinition( EnumTypeDefinition definition = new EnumTypeDefinition(
HiveDataTypes.HIVE_PRINCIPALTYPE.name(), values); HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPALTYPE.name(), definition); enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPALTYPE.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName());
} }
private void createFunctionTypeEnum() throws MetadataException { private void createFunctionTypeEnum() throws MetadataException {
...@@ -152,9 +152,9 @@ public class HiveDataModelGenerator { ...@@ -152,9 +152,9 @@ public class HiveDataModelGenerator {
}; };
EnumTypeDefinition definition = new EnumTypeDefinition( EnumTypeDefinition definition = new EnumTypeDefinition(
HiveDataTypes.HIVE_FUNCTIONTYPE.name(), values); HiveDataTypes.HIVE_FUNCTION_TYPE.getName(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_FUNCTIONTYPE.name(), definition); enumTypeDefinitionMap.put(HiveDataTypes.HIVE_FUNCTION_TYPE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_FUNCTIONTYPE.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_FUNCTION_TYPE.getName());
} }
private void createResourceTypeEnum() throws MetadataException { private void createResourceTypeEnum() throws MetadataException {
...@@ -164,9 +164,9 @@ public class HiveDataModelGenerator { ...@@ -164,9 +164,9 @@ public class HiveDataModelGenerator {
new EnumValue("ARCHIVE", 3), new EnumValue("ARCHIVE", 3),
}; };
EnumTypeDefinition definition = new EnumTypeDefinition( EnumTypeDefinition definition = new EnumTypeDefinition(
HiveDataTypes.HIVE_RESOURCETYPE.name(), values); HiveDataTypes.HIVE_RESOURCE_TYPE.getName(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCETYPE.name(), definition); enumTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCE_TYPE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCETYPE.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCE_TYPE.getName());
} }
private void createSerDeStruct() throws MetadataException { private void createSerDeStruct() throws MetadataException {
...@@ -178,10 +178,10 @@ public class HiveDataModelGenerator { ...@@ -178,10 +178,10 @@ public class HiveDataModelGenerator {
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
}; };
StructTypeDefinition definition = new StructTypeDefinition(HiveDataTypes.HIVE_SERDE.name(), StructTypeDefinition definition = new StructTypeDefinition(
attributeDefinitions); HiveDataTypes.HIVE_SERDE.getName(), attributeDefinitions);
structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.name(), definition); structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.getName());
} }
/* /*
...@@ -200,10 +200,10 @@ public class HiveDataModelGenerator { ...@@ -200,10 +200,10 @@ public class HiveDataModelGenerator {
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
}; };
StructTypeDefinition definition = new StructTypeDefinition( StructTypeDefinition definition = new StructTypeDefinition(
DefinedTypes.HIVE_SKEWEDINFO.name(), attributeDefinitions); DefinedTypes.HIVE_SKEWEDINFO.getName(), attributeDefinitions);
structTypeDefinitionMap.put(DefinedTypes.HIVE_SKEWEDINFO.name(), definition); structTypeDefinitionMap.put(DefinedTypes.HIVE_SKEWEDINFO.getName(), definition);
LOG.debug("Created definition for " + DefinedTypes.HIVE_SKEWEDINFO.name()); LOG.debug("Created definition for " + DefinedTypes.HIVE_SKEWEDINFO.getName());
} }
*/ */
...@@ -216,15 +216,15 @@ public class HiveDataModelGenerator { ...@@ -216,15 +216,15 @@ public class HiveDataModelGenerator {
}; };
StructTypeDefinition definition = new StructTypeDefinition( StructTypeDefinition definition = new StructTypeDefinition(
HiveDataTypes.HIVE_ORDER.name(), attributeDefinitions); HiveDataTypes.HIVE_ORDER.getName(), attributeDefinitions);
structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.name(), definition); structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.getName());
} }
private void createStorageDescClass() throws MetadataException { private void createStorageDescClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("cols", new AttributeDefinition("cols",
String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.name()), String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.getName()),
Multiplicity.COLLECTION, false, null), Multiplicity.COLLECTION, false, null),
new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
...@@ -236,41 +236,41 @@ public class HiveDataModelGenerator { ...@@ -236,41 +236,41 @@ public class HiveDataModelGenerator {
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("numBuckets", DataTypes.INT_TYPE.getName(), new AttributeDefinition("numBuckets", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.name(), new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("bucketCols", new AttributeDefinition("bucketCols",
String.format("array<%s>", DataTypes.STRING_TYPE.getName()), String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sortCols", new AttributeDefinition("sortCols",
String.format("array<%s>", HiveDataTypes.HIVE_ORDER.name()), String.format("array<%s>", HiveDataTypes.HIVE_ORDER.getName()),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
//new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.name(), //new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.getName(),
// Multiplicity.OPTIONAL, false, null), // Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("storedAsSubDirectories", DataTypes.BOOLEAN_TYPE.getName(), new AttributeDefinition("storedAsSubDirectories", DataTypes.BOOLEAN_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
}; };
HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>( HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.name(), null, attributeDefinitions); ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.name(), definition); classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName());
} }
/** Revisit later after nested array types are handled by the typesystem **/ /** Revisit later after nested array types are handled by the typesystem **/
private void createResourceUriStruct() throws MetadataException { private void createResourceUriStruct() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("resourceType", HiveDataTypes.HIVE_RESOURCETYPE.name(), new AttributeDefinition("resourceType", HiveDataTypes.HIVE_RESOURCE_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("uri", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("uri", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
}; };
StructTypeDefinition definition = new StructTypeDefinition( StructTypeDefinition definition = new StructTypeDefinition(
HiveDataTypes.HIVE_RESOURCEURI.name(), attributeDefinitions); HiveDataTypes.HIVE_RESOURCEURI.getName(), attributeDefinitions);
structTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCEURI.name(), definition); structTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCEURI.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCEURI.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCEURI.getName());
} }
private void createDBClass() throws MetadataException { private void createDBClass() throws MetadataException {
...@@ -285,15 +285,15 @@ public class HiveDataModelGenerator { ...@@ -285,15 +285,15 @@ public class HiveDataModelGenerator {
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPALTYPE.name(), new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
}; };
HierarchicalTypeDefinition<ClassType> definition = HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.name(), new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(),
null, attributeDefinitions); null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_DB.name(), definition); classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName());
} }
private void createTypeClass() throws MetadataException { private void createTypeClass() throws MetadataException {
...@@ -305,21 +305,21 @@ public class HiveDataModelGenerator { ...@@ -305,21 +305,21 @@ public class HiveDataModelGenerator {
new AttributeDefinition("type2", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("type2", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("fields", String.format("array<%s>", new AttributeDefinition("fields", String.format("array<%s>",
HiveDataTypes.HIVE_COLUMN.name()), Multiplicity.OPTIONAL, false, null), HiveDataTypes.HIVE_COLUMN.getName()), Multiplicity.OPTIONAL, false, null),
}; };
HierarchicalTypeDefinition<ClassType> definition = HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TYPE.name(), new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TYPE.getName(),
null, attributeDefinitions); null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_TYPE.name(), definition); classTypeDefinitions.put(HiveDataTypes.HIVE_TYPE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_TYPE.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_TYPE.getName());
} }
private void createColumnClass() throws MetadataException { private void createColumnClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
//new AttributeDefinition("type", DefinedTypes.HIVE_TYPE.name(), Multiplicity //new AttributeDefinition("type", DefinedTypes.HIVE_TYPE.getName(), Multiplicity
// .REQUIRED, false, null), // .REQUIRED, false, null),
new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
...@@ -327,11 +327,11 @@ public class HiveDataModelGenerator { ...@@ -327,11 +327,11 @@ public class HiveDataModelGenerator {
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
}; };
HierarchicalTypeDefinition<ClassType> definition = HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.name(), new HierarchicalTypeDefinition<>(
ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(),
null, attributeDefinitions); null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.name(), definition); classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName());
LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.name());
} }
private void createPartitionClass() throws MetadataException { private void createPartitionClass() throws MetadataException {
...@@ -339,36 +339,34 @@ public class HiveDataModelGenerator { ...@@ -339,36 +339,34 @@ public class HiveDataModelGenerator {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("values", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("values", DataTypes.STRING_TYPE.getName(),
Multiplicity.COLLECTION, false, null), Multiplicity.COLLECTION, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.name(), new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("tableName", HiveDataTypes.HIVE_TABLE.name(), new AttributeDefinition("tableName", HiveDataTypes.HIVE_TABLE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(), new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("lastAccessTime", DataTypes.INT_TYPE.getName(), new AttributeDefinition("lastAccessTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.name(), new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
//new AttributeDefinition("columns", String.format("array<%s>", DefinedTypes //new AttributeDefinition("columns", String.format("array<%s>", DefinedTypes
// .HIVE_COLUMN.name()), // .HIVE_COLUMN.getName()), Multiplicity.COLLECTION, true, null),
// Multiplicity.COLLECTION, true, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
}; };
HierarchicalTypeDefinition<ClassType> definition = HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, new HierarchicalTypeDefinition<>(ClassType.class,
HiveDataTypes.HIVE_PARTITION.name(), HiveDataTypes.HIVE_PARTITION.getName(), null, attributeDefinitions);
null, attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_PARTITION.getName(), definition);
classTypeDefinitions.put(HiveDataTypes.HIVE_PARTITION.name(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_PARTITION.getName());
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PARTITION.name());
} }
private void createTableClass() throws MetadataException { private void createTableClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("tableName", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("tableName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.name(), new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
...@@ -378,13 +376,13 @@ public class HiveDataModelGenerator { ...@@ -378,13 +376,13 @@ public class HiveDataModelGenerator {
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(), new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.name(), new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("partitionKeys", new AttributeDefinition("partitionKeys",
String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.name()), String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.getName()),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
//new AttributeDefinition("columns", String.format("array<%s>", DefinedTypes // new AttributeDefinition("columns", // todo - ask venkat
// .HIVE_COLUMN.name()), // String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.getName()),
// Multiplicity.COLLECTION, true, null), // Multiplicity.COLLECTION, true, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
...@@ -398,10 +396,10 @@ public class HiveDataModelGenerator { ...@@ -398,10 +396,10 @@ public class HiveDataModelGenerator {
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
}; };
HierarchicalTypeDefinition<ClassType> definition = HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.name(), new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(),
null, attributeDefinitions); null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.name(), definition); classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName());
} }
private void createIndexClass() throws MetadataException { private void createIndexClass() throws MetadataException {
...@@ -410,17 +408,17 @@ public class HiveDataModelGenerator { ...@@ -410,17 +408,17 @@ public class HiveDataModelGenerator {
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("indexHandlerClass", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("indexHandlerClass", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.name(), new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(), new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("lastAccessTime", DataTypes.INT_TYPE.getName(), new AttributeDefinition("lastAccessTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("origTableName", HiveDataTypes.HIVE_TABLE.name(), new AttributeDefinition("origTableName", HiveDataTypes.HIVE_TABLE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("indexTableName", HiveDataTypes.HIVE_TABLE.name(), new AttributeDefinition("indexTableName", HiveDataTypes.HIVE_TABLE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.name(), new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
...@@ -429,37 +427,36 @@ public class HiveDataModelGenerator { ...@@ -429,37 +427,36 @@ public class HiveDataModelGenerator {
}; };
HierarchicalTypeDefinition<ClassType> definition = HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_INDEX.name(), new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_INDEX.getName(),
null, attributeDefinitions); null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_INDEX.name(), definition); classTypeDefinitions.put(HiveDataTypes.HIVE_INDEX.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_INDEX.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_INDEX.getName());
} }
private void createFunctionClass() throws MetadataException { private void createFunctionClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("functionName", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("functionName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.name(), new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("className", DataTypes.INT_TYPE.getName(), new AttributeDefinition("className", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerName", DataTypes.INT_TYPE.getName(), new AttributeDefinition("ownerName", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPALTYPE.name(), new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(), new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("functionType", HiveDataTypes.HIVE_FUNCTIONTYPE.name(), new AttributeDefinition("functionType", HiveDataTypes.HIVE_FUNCTION_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("resourceUris", HiveDataTypes.HIVE_RESOURCEURI.name(), new AttributeDefinition("resourceUris", HiveDataTypes.HIVE_RESOURCEURI.getName(),
Multiplicity.COLLECTION, false, null), Multiplicity.COLLECTION, false, null),
}; };
HierarchicalTypeDefinition<ClassType> definition = HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_FUNCTION.name(), ClassType.class, HiveDataTypes.HIVE_FUNCTION.getName(), null, attributeDefinitions);
null, attributeDefinitions); classTypeDefinitions.put(HiveDataTypes.HIVE_FUNCTION.getName(), definition);
classTypeDefinitions.put(HiveDataTypes.HIVE_FUNCTION.name(), definition); LOG.debug("Created definition for " + HiveDataTypes.HIVE_FUNCTION.getName());
LOG.debug("Created definition for " + HiveDataTypes.HIVE_FUNCTION.name());
} }
private void createRoleClass() throws MetadataException { private void createRoleClass() throws MetadataException {
...@@ -471,12 +468,11 @@ public class HiveDataModelGenerator { ...@@ -471,12 +468,11 @@ public class HiveDataModelGenerator {
new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
}; };
HierarchicalTypeDefinition<ClassType> definition = HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_ROLE.name(), ClassType.class, HiveDataTypes.HIVE_ROLE.getName(), null, attributeDefinitions);
null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_ROLE.name(), definition); classTypeDefinitions.put(HiveDataTypes.HIVE_ROLE.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_ROLE.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_ROLE.getName());
} }
private void createProcessClass() throws MetadataException { private void createProcessClass() throws MetadataException {
...@@ -490,10 +486,10 @@ public class HiveDataModelGenerator { ...@@ -490,10 +486,10 @@ public class HiveDataModelGenerator {
new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("sourceTableNames", new AttributeDefinition("sourceTableNames",
String.format("array<%s>", HiveDataTypes.HIVE_TABLE.name()), String.format("array<%s>", HiveDataTypes.HIVE_TABLE.getName()),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("targetTableNames", new AttributeDefinition("targetTableNames",
String.format("array<%s>", HiveDataTypes.HIVE_TABLE.name()), String.format("array<%s>", HiveDataTypes.HIVE_TABLE.getName()),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null), Multiplicity.REQUIRED, false, null),
...@@ -506,9 +502,9 @@ public class HiveDataModelGenerator { ...@@ -506,9 +502,9 @@ public class HiveDataModelGenerator {
}; };
HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>( HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
ClassType.class, HiveDataTypes.HIVE_PROCESS.name(), null, attributeDefinitions); ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.name(), definition); classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.name()); LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName());
} }
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
......
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataTypes.java
...@@ -24,10 +24,10 @@ package org.apache.hadoop.metadata.hive.model; ...@@ -24,10 +24,10 @@ package org.apache.hadoop.metadata.hive.model;
public enum HiveDataTypes { public enum HiveDataTypes {
// Enums // Enums
HIVE_OBJECTTYPE, HIVE_OBJECT_TYPE,
HIVE_PRINCIPALTYPE, HIVE_PRINCIPAL_TYPE,
HIVE_RESOURCETYPE, HIVE_RESOURCE_TYPE,
HIVE_FUNCTIONTYPE, HIVE_FUNCTION_TYPE,
// Structs // Structs
HIVE_SERDE, HIVE_SERDE,
...@@ -47,4 +47,9 @@ public enum HiveDataTypes { ...@@ -47,4 +47,9 @@ public enum HiveDataTypes {
HIVE_TYPE, HIVE_TYPE,
HIVE_PROCESS, HIVE_PROCESS,
// HIVE_VIEW, // HIVE_VIEW,
;
public String getName() {
return name().toLowerCase();
}
} }
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hivetypes/HiveImporter.java
...@@ -46,6 +46,10 @@ import org.slf4j.LoggerFactory; ...@@ -46,6 +46,10 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
/**
* todo - this needs to be removed.
*/
@Deprecated
public class HiveImporter { public class HiveImporter {
private static final Logger LOG = private static final Logger LOG =
...@@ -158,7 +162,7 @@ public class HiveImporter { ...@@ -158,7 +162,7 @@ public class HiveImporter {
LOG.debug("creating instance of type " + typeName + " dataType " + dataType); LOG.debug("creating instance of type " + typeName + " dataType " + dataType);
ITypedReferenceableInstance instance = ITypedReferenceableInstance instance =
(ITypedReferenceableInstance) dataType.convert(ref, Multiplicity.OPTIONAL); (ITypedReferenceableInstance) dataType.convert(ref, Multiplicity.OPTIONAL);
String guid = graphRepository.createEntity(instance, typeName); String guid = graphRepository.createEntity(instance);
System.out.println("creating instance of type " + typeName + " dataType " + dataType System.out.println("creating instance of type " + typeName + " dataType " + dataType
+ ", guid: " + guid); + ", guid: " + guid);
......
src/main/examples/hive-data-model.json
{ {
"enumTypes":[ "enumTypes":[
{ {
"name":"HIVE_FUNCTIONTYPE", "name":"hive_object_type",
"enumValues":[
{
"value":"JAVA",
"ordinal":1
}
]
},
{
"name":"HIVE_PRINCIPALTYPE",
"enumValues":[
{
"value":"USER",
"ordinal":1
},
{
"value":"ROLE",
"ordinal":2
},
{
"value":"GROUP",
"ordinal":3
}
]
},
{
"name":"HIVE_OBJECTTYPE",
"enumValues":[ "enumValues":[
{ {
"value":"GLOBAL", "value":"GLOBAL",
...@@ -52,7 +26,7 @@ ...@@ -52,7 +26,7 @@
] ]
}, },
{ {
"name":"HIVE_RESOURCETYPE", "name":"hive_resource_type",
"enumValues":[ "enumValues":[
{ {
"value":"JAR", "value":"JAR",
...@@ -67,15 +41,41 @@ ...@@ -67,15 +41,41 @@
"ordinal":3 "ordinal":3
} }
] ]
},
{
"name":"hive_principal_type",
"enumValues":[
{
"value":"USER",
"ordinal":1
},
{
"value":"ROLE",
"ordinal":2
},
{
"value":"GROUP",
"ordinal":3
}
]
},
{
"name":"hive_function_type",
"enumValues":[
{
"value":"JAVA",
"ordinal":1
}
]
} }
], ],
"structTypes":[ "structTypes":[
{ {
"typeName":"HIVE_RESOURCEURI", "typeName":"hive_order",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"resourceType", "name":"col",
"dataTypeName":"HIVE_RESOURCETYPE", "dataTypeName":"string",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -83,8 +83,8 @@ ...@@ -83,8 +83,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"uri", "name":"order",
"dataTypeName":"string", "dataTypeName":"int",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -94,11 +94,11 @@ ...@@ -94,11 +94,11 @@
] ]
}, },
{ {
"typeName":"HIVE_ORDER", "typeName":"hive_resourceuri",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"col", "name":"resourceType",
"dataTypeName":"string", "dataTypeName":"hive_resource_type",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -106,8 +106,8 @@ ...@@ -106,8 +106,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"order", "name":"uri",
"dataTypeName":"int", "dataTypeName":"string",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -117,7 +117,7 @@ ...@@ -117,7 +117,7 @@
] ]
}, },
{ {
"typeName":"HIVE_SERDE", "typeName":"hive_serde",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"name", "name":"name",
...@@ -158,88 +158,7 @@ ...@@ -158,88 +158,7 @@
], ],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType", "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_FUNCTION", "typeName":"hive_process",
"attributeDefinitions":[
{
"name":"functionName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"dbName",
"dataTypeName":"HIVE_DB",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"className",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"ownerName",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"ownerType",
"dataTypeName":"HIVE_PRINCIPALTYPE",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"createTime",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"functionType",
"dataTypeName":"HIVE_FUNCTIONTYPE",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"resourceUris",
"dataTypeName":"HIVE_RESOURCEURI",
"multiplicity":"collection",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_PROCESS",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"processName", "name":"processName",
...@@ -279,7 +198,7 @@ ...@@ -279,7 +198,7 @@
}, },
{ {
"name":"sourceTableNames", "name":"sourceTableNames",
"dataTypeName":"array<HIVE_TABLE>", "dataTypeName":"array<hive_table>",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -288,7 +207,7 @@ ...@@ -288,7 +207,7 @@
}, },
{ {
"name":"targetTableNames", "name":"targetTableNames",
"dataTypeName":"array<HIVE_TABLE>", "dataTypeName":"array<hive_table>",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -338,12 +257,12 @@ ...@@ -338,12 +257,12 @@
], ],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType", "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_PARTITION", "typeName":"hive_function",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"values", "name":"functionName",
"dataTypeName":"string", "dataTypeName":"string",
"multiplicity":"collection", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
...@@ -351,7 +270,7 @@ ...@@ -351,7 +270,7 @@
}, },
{ {
"name":"dbName", "name":"dbName",
"dataTypeName":"HIVE_DB", "dataTypeName":"hive_db",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -359,16 +278,16 @@ ...@@ -359,16 +278,16 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"tableName", "name":"className",
"dataTypeName":"HIVE_TABLE", "dataTypeName":"int",
"multiplicity":"required", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"createTime", "name":"ownerName",
"dataTypeName":"int", "dataTypeName":"int",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
...@@ -377,17 +296,26 @@ ...@@ -377,17 +296,26 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"lastAccessTime", "name":"ownerType",
"dataTypeName":"hive_principal_type",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"createTime",
"dataTypeName":"int", "dataTypeName":"int",
"multiplicity":"optional", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"sd", "name":"functionType",
"dataTypeName":"HIVE_STORAGEDESC", "dataTypeName":"hive_function_type",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -395,9 +323,9 @@ ...@@ -395,9 +323,9 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"parameters", "name":"resourceUris",
"dataTypeName":"map<string,string>", "dataTypeName":"hive_resourceuri",
"multiplicity":"optional", "multiplicity":"collection",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
...@@ -410,19 +338,19 @@ ...@@ -410,19 +338,19 @@
], ],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType", "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_STORAGEDESC", "typeName":"hive_type",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"cols", "name":"name",
"dataTypeName":"array<HIVE_COLUMN>", "dataTypeName":"string",
"multiplicity":"collection", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"location", "name":"type1",
"dataTypeName":"string", "dataTypeName":"string",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
...@@ -431,7 +359,7 @@ ...@@ -431,7 +359,7 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"inputFormat", "name":"type2",
"dataTypeName":"string", "dataTypeName":"string",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
...@@ -440,17 +368,26 @@ ...@@ -440,17 +368,26 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"outputFormat", "name":"fields",
"dataTypeName":"string", "dataTypeName":"array<hive_column>",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}
]
}, },
{ {
"name":"compressed", "superTypes":[
"dataTypeName":"boolean",
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"hive_table",
"attributeDefinitions":[
{
"name":"tableName",
"dataTypeName":"string",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -458,8 +395,17 @@ ...@@ -458,8 +395,17 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"numBuckets", "name":"dbName",
"dataTypeName":"int", "dataTypeName":"hive_db",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"owner",
"dataTypeName":"string",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -467,8 +413,8 @@ ...@@ -467,8 +413,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"serdeInfo", "name":"createTime",
"dataTypeName":"HIVE_SERDE", "dataTypeName":"int",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -476,8 +422,8 @@ ...@@ -476,8 +422,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"bucketCols", "name":"lastAccessTime",
"dataTypeName":"array<string>", "dataTypeName":"int",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -485,8 +431,8 @@ ...@@ -485,8 +431,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"sortCols", "name":"retention",
"dataTypeName":"array<HIVE_ORDER>", "dataTypeName":"int",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -494,8 +440,8 @@ ...@@ -494,8 +440,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"parameters", "name":"sd",
"dataTypeName":"map<string,string>", "dataTypeName":"hive_storagedesc",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -503,34 +449,34 @@ ...@@ -503,34 +449,34 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"storedAsSubDirectories", "name":"partitionKeys",
"dataTypeName":"boolean", "dataTypeName":"array<hive_column>",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}
]
}, },
{ {
"superTypes":[ "name":"parameters",
"dataTypeName":"map<string,string>",
], "multiplicity":"optional",
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType", "isComposite":false,
"typeName":"HIVE_TYPE", "isUnique":false,
"attributeDefinitions":[ "isIndexable":true,
"reverseAttributeName":null
},
{ {
"name":"name", "name":"viewOriginalText",
"dataTypeName":"string", "dataTypeName":"string",
"multiplicity":"required", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"type1", "name":"viewExpandedText",
"dataTypeName":"string", "dataTypeName":"string",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
...@@ -539,7 +485,7 @@ ...@@ -539,7 +485,7 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"type2", "name":"tableType",
"dataTypeName":"string", "dataTypeName":"string",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
...@@ -548,8 +494,8 @@ ...@@ -548,8 +494,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"fields", "name":"temporary",
"dataTypeName":"array<HIVE_COLUMN>", "dataTypeName":"boolean",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -563,11 +509,20 @@ ...@@ -563,11 +509,20 @@
], ],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType", "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_COLUMN", "typeName":"hive_partition",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"name", "name":"values",
"dataTypeName":"string", "dataTypeName":"string",
"multiplicity":"collection",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"dbName",
"dataTypeName":"hive_db",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -575,8 +530,8 @@ ...@@ -575,8 +530,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"type", "name":"tableName",
"dataTypeName":"string", "dataTypeName":"hive_table",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -584,35 +539,26 @@ ...@@ -584,35 +539,26 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"comment", "name":"createTime",
"dataTypeName":"string", "dataTypeName":"int",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}
]
}, },
{ {
"superTypes":[ "name":"lastAccessTime",
"dataTypeName":"int",
], "multiplicity":"optional",
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_ROLE",
"attributeDefinitions":[
{
"name":"roleName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"createTime", "name":"sd",
"dataTypeName":"int", "dataTypeName":"hive_storagedesc",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -620,9 +566,9 @@ ...@@ -620,9 +566,9 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"ownerName", "name":"parameters",
"dataTypeName":"string", "dataTypeName":"map<string,string>",
"multiplicity":"required", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
...@@ -635,28 +581,19 @@ ...@@ -635,28 +581,19 @@
], ],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType", "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_TABLE", "typeName":"hive_storagedesc",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"tableName", "name":"cols",
"dataTypeName":"string", "dataTypeName":"array<hive_column>",
"multiplicity":"required", "multiplicity":"collection",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"dbName",
"dataTypeName":"HIVE_DB",
"multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"owner", "name":"location",
"dataTypeName":"string", "dataTypeName":"string",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
...@@ -665,17 +602,8 @@ ...@@ -665,17 +602,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"createTime", "name":"inputFormat",
"dataTypeName":"int", "dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"lastAccessTime",
"dataTypeName":"int",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -683,8 +611,8 @@ ...@@ -683,8 +611,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"retention", "name":"outputFormat",
"dataTypeName":"int", "dataTypeName":"string",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -692,17 +620,17 @@ ...@@ -692,17 +620,17 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"sd", "name":"compressed",
"dataTypeName":"HIVE_STORAGEDESC", "dataTypeName":"boolean",
"multiplicity":"optional", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
"isIndexable":true, "isIndexable":true,
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"partitionKeys", "name":"numBuckets",
"dataTypeName":"array<HIVE_COLUMN>", "dataTypeName":"int",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -710,8 +638,8 @@ ...@@ -710,8 +638,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"parameters", "name":"serdeInfo",
"dataTypeName":"map<string,string>", "dataTypeName":"hive_serde",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -719,8 +647,8 @@ ...@@ -719,8 +647,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"viewOriginalText", "name":"bucketCols",
"dataTypeName":"string", "dataTypeName":"array<string>",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -728,8 +656,8 @@ ...@@ -728,8 +656,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"viewExpandedText", "name":"sortCols",
"dataTypeName":"string", "dataTypeName":"array<hive_order>",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -737,8 +665,8 @@ ...@@ -737,8 +665,8 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"tableType", "name":"parameters",
"dataTypeName":"string", "dataTypeName":"map<string,string>",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -746,7 +674,7 @@ ...@@ -746,7 +674,7 @@
"reverseAttributeName":null "reverseAttributeName":null
}, },
{ {
"name":"temporary", "name":"storedAsSubDirectories",
"dataTypeName":"boolean", "dataTypeName":"boolean",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
...@@ -761,7 +689,7 @@ ...@@ -761,7 +689,7 @@
], ],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType", "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_INDEX", "typeName":"hive_index",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"indexName", "name":"indexName",
...@@ -783,7 +711,7 @@ ...@@ -783,7 +711,7 @@
}, },
{ {
"name":"dbName", "name":"dbName",
"dataTypeName":"HIVE_DB", "dataTypeName":"hive_db",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -810,7 +738,7 @@ ...@@ -810,7 +738,7 @@
}, },
{ {
"name":"origTableName", "name":"origTableName",
"dataTypeName":"HIVE_TABLE", "dataTypeName":"hive_table",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -819,7 +747,7 @@ ...@@ -819,7 +747,7 @@
}, },
{ {
"name":"indexTableName", "name":"indexTableName",
"dataTypeName":"HIVE_TABLE", "dataTypeName":"hive_table",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -828,7 +756,7 @@ ...@@ -828,7 +756,7 @@
}, },
{ {
"name":"sd", "name":"sd",
"dataTypeName":"HIVE_STORAGEDESC", "dataTypeName":"hive_storagedesc",
"multiplicity":"required", "multiplicity":"required",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
...@@ -860,7 +788,43 @@ ...@@ -860,7 +788,43 @@
], ],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType", "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_DB", "typeName":"hive_role",
"attributeDefinitions":[
{
"name":"roleName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"createTime",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"ownerName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"hive_db",
"attributeDefinitions":[ "attributeDefinitions":[
{ {
"name":"name", "name":"name",
...@@ -909,7 +873,43 @@ ...@@ -909,7 +873,43 @@
}, },
{ {
"name":"ownerType", "name":"ownerType",
"dataTypeName":"HIVE_PRINCIPALTYPE", "dataTypeName":"hive_principal_type",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"hive_column",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"type",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"comment",
"dataTypeName":"string",
"multiplicity":"optional", "multiplicity":"optional",
"isComposite":false, "isComposite":false,
"isUnique":false, "isUnique":false,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment