Skip to content

A
atlas
Commit 858348b0 by Venkatesh Seetharam
Options

Add Hive Data Model Generator with minor refactoring

parent a2e16bb9
Showing with 1515 additions and 20 deletions
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hivetypes/HiveHook.java → addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
......@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.metadata.hivetypes;
package org.apache.hadoop.metadata.hive.hook;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
......@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook;
import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.metadata.hivetypes.HiveTypeSystem;
import org.apache.hadoop.util.StringUtils;
import org.json.JSONException;
import org.json.JSONObject;
......@@ -61,7 +62,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* DgiHook sends lineage information to the DgiSever
* DgiHook sends lineage information to the DgiSever.
*/
public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHook {
......@@ -75,9 +76,8 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
private static final String user = "postgres";
private static final String password = "postgres";
private static final String insertQuery =
"insert into query_info(query_id, query_text, query_plan, start_time, user_name, " +
"query_graph) "
+ "values (?, ?, ?, ?, ?, ?";
"insert into query_info(query_id, query_text, query_plan, start_time, user_name, "
+ "query_graph) values (?, ?, ?, ?, ?, ?";
private static final String updateQuery =
"update query_info set end_time = ? where query_id = ?";
private static ExecutorService executor;
......
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataModelGenerator.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.hive.model;
import com.google.common.collect.ImmutableList;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.typesystem.TypesDef;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.EnumValue;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
/**
* Utility that generates hive data model for both metastore entities and DDL/DML queries.
*/
public class HiveDataModelGenerator {
private static final Logger LOG = LoggerFactory.getLogger(HiveDataModelGenerator.class);
private static final DataTypes.MapType STRING_MAP_TYPE =
new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
public HiveDataModelGenerator() {
classTypeDefinitions = new HashMap<>();
enumTypeDefinitionMap = new HashMap<>();
structTypeDefinitionMap = new HashMap<>();
}
public void createDataModel() throws MetadataException {
LOG.info("Generating the Hive Data Model....");
// enums
createHiveObjectTypeEnum();
createHivePrincipalTypeEnum();
createFunctionTypeEnum();
createResourceTypeEnum();
// structs
createSerDeStruct();
//createSkewedInfoStruct();
createOrderStruct();
createResourceUriStruct();
createStorageDescClass();
// classes
createDBClass();
createTypeClass();
createColumnClass();
createPartitionClass();
createTableClass();
createIndexClass();
createFunctionClass();
createRoleClass();
// DDL/DML Process
createProcessClass();
}
public TypesDef getTypesDef() {
return TypeUtils.getTypesDef(
getEnumTypeDefinitions(),
getStructTypeDefinitions(),
getTraitTypeDefinitions(),
getClassTypeDefinitions()
);
}
public String getDataModelAsJSON() {
return TypesSerialization.toJson(getTypesDef());
}
public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
return ImmutableList.copyOf(enumTypeDefinitionMap.values());
}
public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
return ImmutableList.copyOf(structTypeDefinitionMap.values());
}
public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
return ImmutableList.copyOf(classTypeDefinitions.values());
}
public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
return ImmutableList.of();
}
private void createHiveObjectTypeEnum() throws MetadataException {
EnumValue values[] = {
new EnumValue("GLOBAL", 1),
new EnumValue("DATABASE", 2),
new EnumValue("TABLE", 3),
new EnumValue("PARTITION", 4),
new EnumValue("COLUMN", 5),
};
EnumTypeDefinition definition = new EnumTypeDefinition(
HiveDataTypes.HIVE_OBJECTTYPE.name(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_OBJECTTYPE.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_OBJECTTYPE.name());
}
private void createHivePrincipalTypeEnum() throws MetadataException {
EnumValue values[] = {
new EnumValue("USER", 1),
new EnumValue("ROLE", 2),
new EnumValue("GROUP", 3),
};
EnumTypeDefinition definition = new EnumTypeDefinition(
HiveDataTypes.HIVE_PRINCIPALTYPE.name(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPALTYPE.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPALTYPE.name());
}
private void createFunctionTypeEnum() throws MetadataException {
EnumValue values[] = {
new EnumValue("JAVA", 1),
};
EnumTypeDefinition definition = new EnumTypeDefinition(
HiveDataTypes.HIVE_FUNCTIONTYPE.name(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_FUNCTIONTYPE.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_FUNCTIONTYPE.name());
}
private void createResourceTypeEnum() throws MetadataException {
EnumValue values[] = {
new EnumValue("JAR", 1),
new EnumValue("FILE", 2),
new EnumValue("ARCHIVE", 3),
};
EnumTypeDefinition definition = new EnumTypeDefinition(
HiveDataTypes.HIVE_RESOURCETYPE.name(), values);
enumTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCETYPE.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCETYPE.name());
}
private void createSerDeStruct() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serializationLib", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
};
StructTypeDefinition definition = new StructTypeDefinition(HiveDataTypes.HIVE_SERDE.name(),
attributeDefinitions);
structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.name());
}
/*
private static final DataTypes.ArrayType STRING_ARRAY_TYPE =
new DataTypes.ArrayType(DataTypes.STRING_TYPE);
private static Multiplicity ZeroOrMore = new Multiplicity(0, Integer.MAX_VALUE, true);
private void createSkewedInfoStruct() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("skewedColNames",
String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
ZeroOrMore, false, null),
new AttributeDefinition("skewedColValues",
String.format("array<%s>", STRING_ARRAY_TYPE.getName()),
ZeroOrMore, false, null),
new AttributeDefinition("skewedColValueLocationMaps", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
};
StructTypeDefinition definition = new StructTypeDefinition(
DefinedTypes.HIVE_SKEWEDINFO.name(), attributeDefinitions);
structTypeDefinitionMap.put(DefinedTypes.HIVE_SKEWEDINFO.name(), definition);
LOG.debug("Created definition for " + DefinedTypes.HIVE_SKEWEDINFO.name());
}
*/
private void createOrderStruct() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("col", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("order", DataTypes.INT_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
};
StructTypeDefinition definition = new StructTypeDefinition(
HiveDataTypes.HIVE_ORDER.name(), attributeDefinitions);
structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.name());
}
private void createStorageDescClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("cols",
String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.name()),
Multiplicity.COLLECTION, false, null),
new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("numBuckets", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.name(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("bucketCols",
String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sortCols",
String.format("array<%s>", HiveDataTypes.HIVE_ORDER.name()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
//new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.name(),
// Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("storedAsSubDirectories", DataTypes.BOOLEAN_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
};
HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.name(), null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.name());
}
/** Revisit later after nested array types are handled by the typesystem **/
private void createResourceUriStruct() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("resourceType", HiveDataTypes.HIVE_RESOURCETYPE.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("uri", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
};
StructTypeDefinition definition = new StructTypeDefinition(
HiveDataTypes.HIVE_RESOURCEURI.name(), attributeDefinitions);
structTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCEURI.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCEURI.name());
}
private void createDBClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("description", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("locationUri", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPALTYPE.name(),
Multiplicity.OPTIONAL, false, null),
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.name(),
null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_DB.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.name());
}
private void createTypeClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("type1", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("type2", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("fields", String.format("array<%s>",
HiveDataTypes.HIVE_COLUMN.name()), Multiplicity.OPTIONAL, false, null),
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TYPE.name(),
null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_TYPE.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_TYPE.name());
}
private void createColumnClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
//new AttributeDefinition("type", DefinedTypes.HIVE_TYPE.name(), Multiplicity
// .REQUIRED, false, null),
new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("comment", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.name(),
null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.name());
}
private void createPartitionClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("values", DataTypes.STRING_TYPE.getName(),
Multiplicity.COLLECTION, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("tableName", HiveDataTypes.HIVE_TABLE.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("lastAccessTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.name(),
Multiplicity.REQUIRED, false, null),
//new AttributeDefinition("columns", String.format("array<%s>", DefinedTypes
// .HIVE_COLUMN.name()),
// Multiplicity.COLLECTION, true, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class,
HiveDataTypes.HIVE_PARTITION.name(),
null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_PARTITION.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PARTITION.name());
}
private void createTableClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("tableName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("lastAccessTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.name(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("partitionKeys",
String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.name()),
Multiplicity.OPTIONAL, false, null),
//new AttributeDefinition("columns", String.format("array<%s>", DefinedTypes
// .HIVE_COLUMN.name()),
// Multiplicity.COLLECTION, true, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("tableType", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.name(),
null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.name());
}
private void createIndexClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("indexName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("indexHandlerClass", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("lastAccessTime", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("origTableName", HiveDataTypes.HIVE_TABLE.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("indexTableName", HiveDataTypes.HIVE_TABLE.name(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("deferredRebuild", DataTypes.BOOLEAN_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_INDEX.name(),
null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_INDEX.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_INDEX.name());
}
private void createFunctionClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("functionName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("dbName", HiveDataTypes.HIVE_DB.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("className", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerName", DataTypes.INT_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPALTYPE.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("functionType", HiveDataTypes.HIVE_FUNCTIONTYPE.name(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("resourceUris", HiveDataTypes.HIVE_RESOURCEURI.name(),
Multiplicity.COLLECTION, false, null),
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_FUNCTION.name(),
null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_FUNCTION.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_FUNCTION.name());
}
private void createRoleClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("roleName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_ROLE.name(),
null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_ROLE.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_ROLE.name());
}
private void createProcessClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("processName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("startTime", DataTypes.INT_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("endTime", DataTypes.INT_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("sourceTableNames",
String.format("array<%s>", HiveDataTypes.HIVE_TABLE.name()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("targetTableNames",
String.format("array<%s>", HiveDataTypes.HIVE_TABLE.name()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("queryId", DataTypes.STRING_TYPE.getName(),
Multiplicity.REQUIRED, false, null),
new AttributeDefinition("queryGraph", DataTypes.STRING_TYPE.getName(),
Multiplicity.OPTIONAL, false, null),
};
HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
ClassType.class, HiveDataTypes.HIVE_PROCESS.name(), null, attributeDefinitions);
classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.name(), definition);
LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.name());
}
public static void main(String[] args) throws Exception {
HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator();
hiveDataModelGenerator.createDataModel();
String hiveDataModelAsJSON = hiveDataModelGenerator.getDataModelAsJSON();
System.out.println("hiveDataModelAsJSON = " + hiveDataModelAsJSON);
}
}
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataTypes.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.hive.model;
/**
* Hive Data Types for model and bridge.
*/
public enum HiveDataTypes {
// Enums
HIVE_OBJECTTYPE,
HIVE_PRINCIPALTYPE,
HIVE_RESOURCETYPE,
HIVE_FUNCTIONTYPE,
// Structs
HIVE_SERDE,
HIVE_SKEWEDINFO,
HIVE_ORDER,
HIVE_RESOURCEURI,
// Classes
HIVE_DB,
HIVE_STORAGEDESC,
HIVE_TABLE,
HIVE_COLUMN,
HIVE_PARTITION,
HIVE_INDEX,
HIVE_FUNCTION,
HIVE_ROLE,
HIVE_TYPE,
HIVE_PROCESS,
// HIVE_VIEW,
}
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hivetypes/HiveTypeSystem.java
......@@ -40,6 +40,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* todo: remove this.
*/
@Deprecated
public class HiveTypeSystem {
......
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hivetypes/HiveGraphRepositoryTest.java
......@@ -25,7 +25,7 @@ import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
import org.apache.hadoop.metadata.repository.graph.GraphHelper;
......@@ -46,8 +46,6 @@ import java.util.List;
@Test(enabled = false)
public class HiveGraphRepositoryTest {
public static final String HIVE_L4J_PROPS = "target/hive-log4j.properties";
public static final String HIVE_EXEC_L4J_PROPS = "target/hive-exec-log4j.properties";
private static final Logger LOG =
LoggerFactory.getLogger(HiveGraphRepositoryTest.class);
protected HiveTypeSystem hts;
......
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hivetypes/HiveTypeSystemTest.java
......@@ -21,10 +21,10 @@ package org.apache.hadoop.metadata.hivetypes;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.typesystem.persistence.Id;
import org.apache.hadoop.metadata.typesystem.persistence.memory.MemRepository;
import org.apache.hadoop.metadata.repository.memory.MemRepository;
import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -105,7 +105,7 @@ public class HiveTypeSystemTest {
}
@Test(enabled = true)
public void testHiveLineage() throws MetaException, MetadataException, IOException, Exception {
public void testHiveLineage() throws Exception {
Class.forName("org.apache.hive.jdbc.HiveDriver");
String url = "jdbc:hive2://" + hiveHost + ":" + hivePort;
Connection con = DriverManager.getConnection(url, "ambari-qa", "");
......@@ -114,8 +114,5 @@ public class HiveTypeSystemTest {
stmt.execute("create table t(a int, b string)");
stmt.execute("drop table if exists t2");
stmt.execute("create table t2 as select * from t");
}
}
\ No newline at end of file
pom.xml
......@@ -112,6 +112,7 @@
<excludes>
<exclude>*.txt</exclude>
<exclude>**/*.txt</exclude>
<exclude>**/*.json</exclude>
<exclude>.git/**</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.idea/**</exclude>
......
src/main/examples/hive-data-model.json 0 → 100644
{
"enumTypes":[
{
"name":"HIVE_FUNCTIONTYPE",
"enumValues":[
{
"value":"JAVA",
"ordinal":1
}
]
},
{
"name":"HIVE_PRINCIPALTYPE",
"enumValues":[
{
"value":"USER",
"ordinal":1
},
{
"value":"ROLE",
"ordinal":2
},
{
"value":"GROUP",
"ordinal":3
}
]
},
{
"name":"HIVE_OBJECTTYPE",
"enumValues":[
{
"value":"GLOBAL",
"ordinal":1
},
{
"value":"DATABASE",
"ordinal":2
},
{
"value":"TABLE",
"ordinal":3
},
{
"value":"PARTITION",
"ordinal":4
},
{
"value":"COLUMN",
"ordinal":5
}
]
},
{
"name":"HIVE_RESOURCETYPE",
"enumValues":[
{
"value":"JAR",
"ordinal":1
},
{
"value":"FILE",
"ordinal":2
},
{
"value":"ARCHIVE",
"ordinal":3
}
]
}
],
"structTypes":[
{
"typeName":"HIVE_RESOURCEURI",
"attributeDefinitions":[
{
"name":"resourceType",
"dataTypeName":"HIVE_RESOURCETYPE",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"uri",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"typeName":"HIVE_ORDER",
"attributeDefinitions":[
{
"name":"col",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"order",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"typeName":"HIVE_SERDE",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"serializationLib",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"parameters",
"dataTypeName":"map<string,string>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
}
],
"traitTypes":[
],
"classTypes":[
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_FUNCTION",
"attributeDefinitions":[
{
"name":"functionName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"dbName",
"dataTypeName":"HIVE_DB",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"className",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"ownerName",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"ownerType",
"dataTypeName":"HIVE_PRINCIPALTYPE",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"createTime",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"functionType",
"dataTypeName":"HIVE_FUNCTIONTYPE",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"resourceUris",
"dataTypeName":"HIVE_RESOURCEURI",
"multiplicity":"collection",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_PROCESS",
"attributeDefinitions":[
{
"name":"processName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"startTime",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"endTime",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"userName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"sourceTableNames",
"dataTypeName":"array<HIVE_TABLE>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"targetTableNames",
"dataTypeName":"array<HIVE_TABLE>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"queryText",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"queryPlan",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"queryId",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"queryGraph",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_PARTITION",
"attributeDefinitions":[
{
"name":"values",
"dataTypeName":"string",
"multiplicity":"collection",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"dbName",
"dataTypeName":"HIVE_DB",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"tableName",
"dataTypeName":"HIVE_TABLE",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"createTime",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"lastAccessTime",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"sd",
"dataTypeName":"HIVE_STORAGEDESC",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"parameters",
"dataTypeName":"map<string,string>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_STORAGEDESC",
"attributeDefinitions":[
{
"name":"cols",
"dataTypeName":"array<HIVE_COLUMN>",
"multiplicity":"collection",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"location",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"inputFormat",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"outputFormat",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"compressed",
"dataTypeName":"boolean",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"numBuckets",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"serdeInfo",
"dataTypeName":"HIVE_SERDE",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"bucketCols",
"dataTypeName":"array<string>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"sortCols",
"dataTypeName":"array<HIVE_ORDER>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"parameters",
"dataTypeName":"map<string,string>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"storedAsSubDirectories",
"dataTypeName":"boolean",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_TYPE",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"type1",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"type2",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"fields",
"dataTypeName":"array<HIVE_COLUMN>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_COLUMN",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"type",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"comment",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_ROLE",
"attributeDefinitions":[
{
"name":"roleName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"createTime",
"dataTypeName":"int",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"ownerName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_TABLE",
"attributeDefinitions":[
{
"name":"tableName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"dbName",
"dataTypeName":"HIVE_DB",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"owner",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"createTime",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"lastAccessTime",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"retention",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"sd",
"dataTypeName":"HIVE_STORAGEDESC",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"partitionKeys",
"dataTypeName":"array<HIVE_COLUMN>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"parameters",
"dataTypeName":"map<string,string>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"viewOriginalText",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"viewExpandedText",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"tableType",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"temporary",
"dataTypeName":"boolean",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_INDEX",
"attributeDefinitions":[
{
"name":"indexName",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"indexHandlerClass",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"dbName",
"dataTypeName":"HIVE_DB",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"createTime",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"lastAccessTime",
"dataTypeName":"int",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"origTableName",
"dataTypeName":"HIVE_TABLE",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"indexTableName",
"dataTypeName":"HIVE_TABLE",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"sd",
"dataTypeName":"HIVE_STORAGEDESC",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"parameters",
"dataTypeName":"map<string,string>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"deferredRebuild",
"dataTypeName":"boolean",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
},
{
"superTypes":[
],
"hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
"typeName":"HIVE_DB",
"attributeDefinitions":[
{
"name":"name",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"description",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"locationUri",
"dataTypeName":"string",
"multiplicity":"required",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"parameters",
"dataTypeName":"map<string,string>",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"ownerName",
"dataTypeName":"string",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
},
{
"name":"ownerType",
"dataTypeName":"HIVE_PRINCIPALTYPE",
"multiplicity":"optional",
"isComposite":false,
"isUnique":false,
"isIndexable":true,
"reverseAttributeName":null
}
]
}
]
}
\ No newline at end of file
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
......@@ -27,7 +27,6 @@ import java.lang.reflect.Constructor;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
......
typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/utils/TypesUtil.java
......@@ -21,6 +21,8 @@ package org.apache.hadoop.metadata.typesystem.types.utils;
import com.google.common.collect.ImmutableList;
import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.EnumValue;
import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.typesystem.types.IDataType;
import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
......@@ -57,15 +59,17 @@ public class TypesUtil {
Multiplicity.REQUIRED, false, null);
}
@SuppressWarnings("unchecked")
public static EnumTypeDefinition createEnumTypeDef(String name, EnumValue... enumValues) {
return new EnumTypeDefinition(name, enumValues);
}
public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(
String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition(TraitType.class, name, superTypes, attrDefs);
return new HierarchicalTypeDefinition<>(TraitType.class, name, superTypes, attrDefs);
}
@SuppressWarnings("unchecked")
public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(
String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs);
return new HierarchicalTypeDefinition<>(ClassType.class, name, superTypes, attrDefs);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment