Skip to content

A
atlas
Commit 0adfeb18 by Venkat Ranganathan
Options

Fixed hive model: StorageDescription is now a class, removed creating a…

Fixed hive model: StorageDescription is now a class, removed creating a duplicate storagedesc per partition (need to fix handling of partitions that can diverge). Made mem repository test work again. Validated changes with TPCDS. Fixed empty array handling in Graph repo
parent b10f93a2
Showing with 28 additions and 18 deletions
hivetypes/src/main/java/org/apache/hadoop/metadata/hivetypes/HiveTypeSystem.java
...@@ -61,7 +61,6 @@ public class HiveTypeSystem { ...@@ -61,7 +61,6 @@ public class HiveTypeSystem {
// Structs // Structs
HIVE_SERDE, HIVE_SERDE,
HIVE_STORAGEDESC,
HIVE_SKEWEDINFO, HIVE_SKEWEDINFO,
HIVE_ORDER, HIVE_ORDER,
HIVE_RESOURCEURI, HIVE_RESOURCEURI,
...@@ -69,6 +68,7 @@ public class HiveTypeSystem { ...@@ -69,6 +68,7 @@ public class HiveTypeSystem {
// Classes // Classes
HIVE_DB, HIVE_DB,
HIVE_STORAGEDESC,
HIVE_TABLE, HIVE_TABLE,
HIVE_COLUMN, HIVE_COLUMN,
HIVE_PARTITION, HIVE_PARTITION,
...@@ -122,7 +122,7 @@ public class HiveTypeSystem { ...@@ -122,7 +122,7 @@ public class HiveTypeSystem {
//createSkewedInfoStruct(); //createSkewedInfoStruct();
createOrderStruct(); createOrderStruct();
createResourceUriStruct(); createResourceUriStruct();
createStorageDescStruct(); createStorageDescClass();
createDBClass(); createDBClass();
createTypeClass(); createTypeClass();
...@@ -168,6 +168,7 @@ public class HiveTypeSystem { ...@@ -168,6 +168,7 @@ public class HiveTypeSystem {
if (valid) { if (valid) {
return ImmutableList.of( return ImmutableList.of(
(HierarchicalType) typeMap.get(DefinedTypes.HIVE_DB.name()), (HierarchicalType) typeMap.get(DefinedTypes.HIVE_DB.name()),
(HierarchicalType) typeMap.get(DefinedTypes.HIVE_STORAGEDESC.name()),
(HierarchicalType) typeMap.get(DefinedTypes.HIVE_TABLE.name()), (HierarchicalType) typeMap.get(DefinedTypes.HIVE_TABLE.name()),
(HierarchicalType) typeMap.get(DefinedTypes.HIVE_COLUMN.name()), (HierarchicalType) typeMap.get(DefinedTypes.HIVE_COLUMN.name()),
(HierarchicalType) typeMap.get(DefinedTypes.HIVE_PARTITION.name()), (HierarchicalType) typeMap.get(DefinedTypes.HIVE_PARTITION.name()),
...@@ -300,7 +301,7 @@ public class HiveTypeSystem { ...@@ -300,7 +301,7 @@ public class HiveTypeSystem {
private void createStorageDescStruct() throws MetadataException { private void createStorageDescClass() throws MetadataException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("cols", String.format("array<%s>", DefinedTypes.HIVE_COLUMN.name()), Multiplicity.COLLECTION, false, null), new AttributeDefinition("cols", String.format("array<%s>", DefinedTypes.HIVE_COLUMN.name()), Multiplicity.COLLECTION, false, null),
new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
...@@ -316,11 +317,10 @@ public class HiveTypeSystem { ...@@ -316,11 +317,10 @@ public class HiveTypeSystem {
new AttributeDefinition("storedAsSubDirectories", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("storedAsSubDirectories", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
}; };
HierarchicalTypeDefinition<ClassType> definition =
StructTypeDefinition definition = new HierarchicalTypeDefinition<>(ClassType.class, DefinedTypes.HIVE_STORAGEDESC.name(),
new StructTypeDefinition(DefinedTypes.HIVE_STORAGEDESC.name(), attributeDefinitions); null, attributeDefinitions);
classTypeDefinitions.put(DefinedTypes.HIVE_STORAGEDESC.name(), definition);
structTypeDefinitionMap.put(DefinedTypes.HIVE_STORAGEDESC.name(), definition);
LOG.debug("Created definition for " + DefinedTypes.HIVE_STORAGEDESC.name()); LOG.debug("Created definition for " + DefinedTypes.HIVE_STORAGEDESC.name());
} }
...@@ -401,8 +401,8 @@ public class HiveTypeSystem { ...@@ -401,8 +401,8 @@ public class HiveTypeSystem {
new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("createTime", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("lastAccessTime", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("lastAccessTime", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("sd", DefinedTypes.HIVE_STORAGEDESC.name(), Multiplicity.REQUIRED, false, null), new AttributeDefinition("sd", DefinedTypes.HIVE_STORAGEDESC.name(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("columns", String.format("array<%s>", DefinedTypes.HIVE_COLUMN.name()), //new AttributeDefinition("columns", String.format("array<%s>", DefinedTypes.HIVE_COLUMN.name()),
Multiplicity.COLLECTION, true, null), // Multiplicity.COLLECTION, true, null),
new AttributeDefinition("parameters", mapStrToStrMap.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("parameters", mapStrToStrMap.getName(), Multiplicity.OPTIONAL, false, null),
}; };
...@@ -426,8 +426,8 @@ public class HiveTypeSystem { ...@@ -426,8 +426,8 @@ public class HiveTypeSystem {
new AttributeDefinition("sd", DefinedTypes.HIVE_STORAGEDESC.name(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("sd", DefinedTypes.HIVE_STORAGEDESC.name(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("partitionKeys", String.format("array<%s>", DefinedTypes.HIVE_COLUMN.name()), new AttributeDefinition("partitionKeys", String.format("array<%s>", DefinedTypes.HIVE_COLUMN.name()),
Multiplicity.OPTIONAL, false, null), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("columns", String.format("array<%s>", DefinedTypes.HIVE_COLUMN.name()), //new AttributeDefinition("columns", String.format("array<%s>", DefinedTypes.HIVE_COLUMN.name()),
Multiplicity.COLLECTION, true, null), // Multiplicity.COLLECTION, true, null),
new AttributeDefinition("parameters", mapStrToStrMap.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("parameters", mapStrToStrMap.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
......
hivetypes/src/main/resources/hive-site.xml
...@@ -126,7 +126,8 @@ ...@@ -126,7 +126,8 @@
<property> <property>
<name>hive.metastore.uris</name> <name>hive.metastore.uris</name>
<value>thrift://10.10.11.207:9083</value> <!-- <value>thrift://10.10.11.207:9083</value> -->
<value>thrift://localhost:9083</value>
</property> </property>
<property> <property>
......
hivetypes/src/test/java/org/apache/hadoop/metadata/hivetypes/HiveGraphRepositoryTest.java
...@@ -39,6 +39,7 @@ import org.testng.annotations.BeforeClass; ...@@ -39,6 +39,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
import java.util.List; import java.util.List;
...@@ -82,7 +83,9 @@ public class HiveGraphRepositoryTest { ...@@ -82,7 +83,9 @@ public class HiveGraphRepositoryTest {
HiveImporter hImporter = new HiveImporter(repository, hts, new HiveMetaStoreClient(new HiveConf())); HiveImporter hImporter = new HiveImporter(repository, hts, new HiveMetaStoreClient(new HiveConf()));
hImporter.importHiveMetadata(); hImporter.importHiveMetadata();
LOG.info("Defined DB instances"); LOG.info("Defined DB instances");
FileWriter fw = new FileWriter("hiveobjs.txt"); File f = new File("./target/logs/hiveobjs.txt");
f.getParentFile().mkdirs();
FileWriter fw = new FileWriter(f);
BufferedWriter bw = new BufferedWriter(fw); BufferedWriter bw = new BufferedWriter(fw);
List<String> idList = List<String> idList =
repository.getEntityList(HiveTypeSystem.DefinedTypes.HIVE_DB.name()); repository.getEntityList(HiveTypeSystem.DefinedTypes.HIVE_DB.name());
......
hivetypes/src/test/java/org/apache/hadoop/metadata/hivetypes/HiveTypeSystemTest.java
...@@ -32,6 +32,7 @@ import org.testng.annotations.BeforeClass; ...@@ -32,6 +32,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
...@@ -59,8 +60,9 @@ public class HiveTypeSystemTest { ...@@ -59,8 +60,9 @@ public class HiveTypeSystemTest {
HiveImporter hImporter = new HiveImporter(mr, hts, new HiveMetaStoreClient(new HiveConf())); HiveImporter hImporter = new HiveImporter(mr, hts, new HiveMetaStoreClient(new HiveConf()));
hImporter.importHiveMetadata(); hImporter.importHiveMetadata();
LOG.info("Defined DB instances"); LOG.info("Defined DB instances");
FileWriter fw = new FileWriter("hiveobjs.txt"); File f = new File("./target/logs/hiveobjs.txt");
BufferedWriter bw = new BufferedWriter(fw); f.getParentFile().mkdirs();
FileWriter fw = new FileWriter(f); BufferedWriter bw = new BufferedWriter(fw);
for (Id id : hImporter.getDBInstances()) { for (Id id : hImporter.getDBInstances()) {
ITypedReferenceableInstance instance = mr.get(id); ITypedReferenceableInstance instance = mr.get(id);
bw.write(instance.toString()); bw.write(instance.toString());
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
...@@ -932,7 +932,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -932,7 +932,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name); LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name; String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
String keys = instanceVertex.getProperty(propertyName); String keys = instanceVertex.getProperty(propertyName);
if (keys == null || keys.length() == 0) {
return;
}
DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType(); DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
final IDataType elementType = arrayType.getElemType(); final IDataType elementType = arrayType.getElemType();
...@@ -983,7 +985,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -983,7 +985,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name); LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name; String propertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
String keys = instanceVertex.getProperty(propertyName); String keys = instanceVertex.getProperty(propertyName);
if (keys == null || keys.length() == 0) {
return;
}
DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType(); DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
final IDataType elementType = mapType.getValueType(); final IDataType elementType = mapType.getValueType();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment