Skip to content

A
atlas
Commit d8d9da29 by Venkatesh Seetharam
Options

Fix Array type mapping with tests. Contributed by Venkatesh Seetharam

parent 22e8a750
Showing with 112 additions and 9 deletions
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
......@@ -18,6 +18,9 @@
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
......@@ -88,6 +91,23 @@ public final class GraphHelper {
return results.hasNext() ? results.next() : null;
}
public static PropertyKey createPropertyKey(TitanManagement management, String propertyName) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
.makePropertyKey(propertyName)
.dataType(String.class)
.cardinality(Cardinality.SET)
.make();
management.buildIndex("by" + propertyName, Vertex.class)
.addKey(propertyKey)
.buildCompositeIndex();
}
return propertyKey;
}
public static String vertexString(final Vertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
......
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperTest.java
......@@ -20,7 +20,10 @@ package org.apache.hadoop.metadata.repository.graph;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.Referenceable;
......@@ -37,11 +40,15 @@ import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.types.TraitType;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Iterator;
@Test
@Guice(modules = RepositoryMetadataModule.class)
......@@ -53,9 +60,9 @@ public class GraphRepoMapperTest {
private static final String TABLE_NAME = "bar";
@Inject
TitanGraphService titanGraphService;
private TitanGraphService titanGraphService;
@Inject
GraphBackedMetadataRepository repositoryService;
private GraphBackedMetadataRepository repositoryService;
private TypeSystem typeSystem;
......@@ -71,6 +78,11 @@ public class GraphRepoMapperTest {
createHiveTypes();
}
@AfterMethod
public void tearDown() {
dumpGraph();
}
@Test
public void testSubmitEntity() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
......@@ -91,8 +103,28 @@ public class GraphRepoMapperTest {
ITypedReferenceableInstance table = createHiveTableInstance(dbInstance);
String tableGUID = repositoryService.createEntity(table, TABLE_TYPE);
System.out.println("added table = " + tableGUID);
}
dumpGraph();
@Test (dependsOnMethods = "testSubmitEntity")
public void testGetEntityDefinition() throws Exception {
TitanGraph graph = titanGraphService.getTitanGraph();
GraphQuery query = graph.query()
.has("type", Compare.EQUAL, TABLE_TYPE)
.has("type", Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since guid should be unique
Vertex tableVertex = results.hasNext() ? results.next() : null;
if (tableVertex == null) {
Assert.fail();
}
String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
if (guid == null) {
Assert.fail();
}
ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid);
System.out.println("*** table = " + table);
}
private void dumpGraph() {
......@@ -100,7 +132,12 @@ public class GraphRepoMapperTest {
System.out.println("*******************Graph Dump****************************");
System.out.println("Vertices of " + graph);
for (Vertex vertex : graph.getVertices()) {
System.out.println(GraphHelper.vertexString(vertex));
// System.out.println(GraphHelper.vertexString(vertex));
System.out.print("vertex = " + vertex + " [");
TitanVertex titanVertex = (TitanVertex) graph.getVertex(vertex.getId());
System.out.print("guid= " + titanVertex.getProperty(Constants.GUID_PROPERTY_KEY));
System.out.print("type= " + titanVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
System.out.println("]");
}
System.out.println("Edges of " + graph);
......@@ -132,23 +169,45 @@ public class GraphRepoMapperTest {
EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
typeSystem.defineEnumType(enumTypeDefinition);
HierarchicalTypeDefinition<ClassType> columnsDefinition =
createClassTypeDef("column_type",
ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE));
StructTypeDefinition partitionDefinition =
new StructTypeDefinition("partition_type",
new AttributeDefinition[] {
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
});
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
createClassTypeDef(TABLE_TYPE,
ImmutableList.<String>of(),
createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE),
// enum
new AttributeDefinition("tableType", "tableType",
Multiplicity.REQUIRED, false, null),
/*
new AttributeDefinition("columns",
// array of strings
new AttributeDefinition("columnNames",
String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
Multiplicity.COLLECTION, false, null),
*/
// array of classes
new AttributeDefinition("columns",
String.format("array<%s>", "column_type"),
Multiplicity.COLLECTION, true, null),
// array of structs
new AttributeDefinition("partitions",
String.format("array<%s>", "partition_type"),
Multiplicity.COLLECTION, true, null),
// struct reference
new AttributeDefinition("serde1",
"serdeType", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("serde2",
"serdeType", Multiplicity.REQUIRED, false, null),
// class reference
new AttributeDefinition("database",
DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
......@@ -158,9 +217,9 @@ public class GraphRepoMapperTest {
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
typeSystem.defineTypes(
ImmutableList.of(structTypeDefinition),
ImmutableList.of(structTypeDefinition, partitionDefinition),
ImmutableList.of(classificationTypeDefinition),
ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
ImmutableList.of(databaseTypeDefinition, columnsDefinition, tableTypeDefinition));
}
private ITypedReferenceableInstance createHiveTableInstance(
......@@ -176,8 +235,15 @@ public class GraphRepoMapperTest {
tableInstance.set("description", "bar table");
tableInstance.set("type", "managed");
tableInstance.set("tableType", 1); // enum
// refer to an existing class
tableInstance.set("database", databaseInstance);
ArrayList<String> columnNames = new ArrayList<>();
columnNames.add("first_name");
columnNames.add("last_name");
tableInstance.set("columnNames", columnNames);
Struct traitInstance = (Struct) tableInstance.getTrait("classification");
traitInstance.set("tag", "foundation_etl");
......@@ -191,6 +257,23 @@ public class GraphRepoMapperTest {
serde2Instance.set("serde", "serde2");
tableInstance.set("serde2", serde2Instance);
ArrayList<Referenceable> columns = new ArrayList<>();
for (int index = 0; index < 5; index++) {
Referenceable columnInstance = new Referenceable("column_type");
columnInstance.set("name", "column_" + index);
columnInstance.set("type", "string");
columns.add(columnInstance);
}
tableInstance.set("columns", columns);
ArrayList<Struct> partitions = new ArrayList<>();
for (int index = 0; index < 5; index++) {
Struct partitionInstance = new Struct("partition_type");
partitionInstance.set("name", "partition_" + index);
partitions.add(partitionInstance);
}
tableInstance.set("partitions", partitions);
ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment