Skip to content

A
atlas
Commit be7c4cb6 by Venkatesh Seetharam
Options

ISSUE-38 Map type to graph with type prefixes to enable search. Contributed by Venkatesh Seetharam

parent 1d1531ef
Showing with 89 additions and 72 deletions
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphUtils.java
......@@ -23,10 +23,14 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.ITypedInstance;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.storage.Id;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Iterator;
import java.util.UUID;
/**
* Utility class for graph operations.
......@@ -38,40 +42,26 @@ public final class GraphUtils {
private GraphUtils() {
}
public static Edge addEdge(Vertex fromVertex, Vertex toVertex,
String vertexPropertyKey, String edgeLabel) {
return addEdge(fromVertex, toVertex, vertexPropertyKey, edgeLabel, null);
public static Vertex createVertex(Graph graph,
ITypedReferenceableInstance typedInstance) {
return createVertex(graph, typedInstance, typedInstance.getId());
}
public static Edge addEdge(Vertex fromVertex, Vertex toVertex,
String vertexPropertyKey, String edgeLabel, String timestamp) {
Edge edge = findEdge(fromVertex, toVertex, vertexPropertyKey, edgeLabel);
public static Vertex createVertex(Graph graph,
ITypedInstance typedInstance,
Id typedInstanceId) {
final Vertex instanceVertex = graph.addVertex(null);
// type
instanceVertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typedInstance.getTypeName());
Edge edgeToVertex = edge != null ? edge : fromVertex.addEdge(edgeLabel, toVertex);
if (timestamp != null) {
edgeToVertex.setProperty(Constants.TIMESTAMP_PROPERTY_KEY, timestamp);
}
return edgeToVertex;
}
// id
final String guid = UUID.randomUUID().toString();
instanceVertex.setProperty(Constants.GUID_PROPERTY_KEY, guid);
public static Edge findEdge(Vertex fromVertex, Vertex toVertex,
String vertexPropertyKey, String edgeLabel) {
return findEdge(fromVertex, toVertex.getProperty(vertexPropertyKey),
vertexPropertyKey, edgeLabel);
}
public static Edge findEdge(Vertex fromVertex, Object toVertexName,
String vertexPropertyKey, String edgeLabel) {
Edge edgeToFind = null;
for (Edge edge : fromVertex.getEdges(Direction.OUT, edgeLabel)) {
if (edge.getVertex(Direction.IN).getProperty(vertexPropertyKey).equals(toVertexName)) {
edgeToFind = edge;
break;
}
}
// version
instanceVertex.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
return edgeToFind;
return instanceVertex;
}
public static Vertex findVertex(Graph blueprintsGraph,
......@@ -97,23 +87,23 @@ public final class GraphUtils {
public static String edgeString(final Edge edge) {
return "e[" + edge.getLabel() + "], ["
+ edge.getVertex(Direction.OUT).getProperty("name")
+ edge.getVertex(Direction.OUT)
+ " -> " + edge.getLabel() + " -> "
+ edge.getVertex(Direction.IN).getProperty("name")
+ edge.getVertex(Direction.IN)
+ "]";
}
public static void dumpToLog(final Graph graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph);
for (Vertex vertex : graph.getVertices()) {
LOG.debug(vertexString(vertex));
System.out.println(vertexString(vertex));
}
LOG.debug("Edges of {}", graph);
for (Edge edge : graph.getEdges()) {
LOG.debug(edgeString(edge));
System.out.println(edgeString(edge));
}
LOG.debug("*******************Graph Dump****************************");
}
}
\ No newline at end of file
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
......@@ -34,15 +34,16 @@ import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.types.TraitType;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import scala.actors.threadpool.Arrays;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
/**
......@@ -50,28 +51,29 @@ import java.util.UUID;
*/
public class EntityJerseyResourceIT extends BaseResourceIT {
private static final Logger LOG = LoggerFactory.getLogger(EntityJerseyResourceIT.class);
private static final String DATABASE_TYPE = "hive_database";
private static final String DATABASE_NAME = "foo";
private static final String TABLE_TYPE = "hive_table";
private static final String TABLE_NAME = "bar";
private static final String TRAIT_TYPE = "hive_fetl";
private String tableInstanceAsJSON;
private ITypedReferenceableInstance tableInstance;
private String guid;
@BeforeClass
public void setUp() throws Exception {
super.setUp();
List<HierarchicalTypeDefinition> typeDefinitions = createHiveTypes();
submitTypes(typeDefinitions);
createHiveTypes();
submitTypes();
}
@Test
public void testSubmitEntity() throws Exception {
ITypedReferenceableInstance tableInstance = createHiveTableInstance();
tableInstanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
tableInstance = createHiveTableInstance();
String tableInstanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
LOG.debug("tableInstance = " + tableInstanceAsJSON);
WebResource resource = service
.path("api/metadata/entities/submit")
......@@ -119,12 +121,24 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
final String definition = response.getString("definition");
Assert.assertNotNull(definition);
LOG.debug("tableInstanceAfterGet = " + definition);
// todo - this fails with type error, strange
// ITypedReferenceableInstance tableInstanceAfterGet = Serialization$.MODULE$.fromJson(definition);
// Assert.assertTrue(areEqual(tableInstance, tableInstanceAfterGet));
}
System.out.println("definition = " + definition);
System.out.println("tableInstanceAsJSON = " + tableInstanceAsJSON);
// Assert.assertEquals(definition, tableInstanceAsJSON);
/*
private boolean areEqual(ITypedReferenceableInstance actual,
ITypedReferenceableInstance expected) throws Exception {
for (AttributeInfo attributeInfo : actual.fieldMapping().fields.values()) {
Assert.assertEquals(actual.get(attributeInfo.name), expected.get(attributeInfo.name));
}
return true;
}
*/
@Test
public void testGetInvalidEntityDefinition() {
WebResource resource = service
......@@ -165,15 +179,19 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
System.out.println("response = " + response);
}
private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception {
ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
private void createHiveTypes() throws Exception {
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
createClassTypeDef(DATABASE_TYPE,
ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE));
typeDefinitions.add(databaseTypeDefinition);
StructTypeDefinition structTypeDefinition =
new StructTypeDefinition("serdeType",
new AttributeDefinition[] {
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
});
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
createClassTypeDef(TABLE_TYPE,
......@@ -181,32 +199,31 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE),
new AttributeDefinition(DATABASE_TYPE,
DATABASE_TYPE, Multiplicity.REQUIRED, true, DATABASE_TYPE));
typeDefinitions.add(tableTypeDefinition);
HierarchicalTypeDefinition<TraitType> fetlTypeDefinition =
createTraitTypeDef(TRAIT_TYPE,
new AttributeDefinition("serde1",
"serdeType", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("serde2",
"serdeType", Multiplicity.REQUIRED, false, null),
new AttributeDefinition("database",
DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
createTraitTypeDef("classification",
ImmutableList.<String>of(),
createRequiredAttrDef("level", DataTypes.INT_TYPE));
typeDefinitions.add(fetlTypeDefinition);
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
typeSystem.defineTypes(
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(fetlTypeDefinition),
ImmutableList.of(structTypeDefinition),
ImmutableList.of(classificationTypeDefinition),
ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
return typeDefinitions;
}
private void submitTypes(List<HierarchicalTypeDefinition> typeDefinitions) throws Exception {
for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
String typesAsJSON = TypesSerialization.toJson(
typeSystem, typeDefinition.typeName);
private void submitTypes() throws Exception {
String typesAsJSON = TypesSerialization.toJson(typeSystem,
Arrays.asList(new String[]{DATABASE_TYPE, TABLE_TYPE, "serdeType", "classification"}));
WebResource resource = service
.path("api/metadata/types/submit")
.path(typeDefinition.typeName);
.path(TABLE_TYPE);
ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON)
......@@ -218,25 +235,34 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString);
Assert.assertEquals(response.get("typeName"), typeDefinition.typeName);
Assert.assertEquals(response.get("typeName"), TABLE_TYPE);
Assert.assertNotNull(response.get("types"));
Assert.assertNotNull(response.get("requestId"));
}
}
protected ITypedReferenceableInstance createHiveTableInstance() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", DATABASE_NAME);
databaseInstance.set("description", "foo database");
Referenceable tableInstance = new Referenceable(TABLE_TYPE, TRAIT_TYPE);
Referenceable tableInstance = new Referenceable(TABLE_TYPE, "classification");
tableInstance.set("name", TABLE_NAME);
tableInstance.set("description", "bar table");
tableInstance.set("type", "managed");
tableInstance.set(DATABASE_TYPE, databaseInstance);
tableInstance.set("database", databaseInstance);
Struct traitInstance = (Struct) tableInstance.getTrait("classification");
traitInstance.set("tag", "foundation_etl");
Struct serde1Instance = new Struct("serdeType");
serde1Instance.set("name", "serde1");
serde1Instance.set("serde", "serde1");
tableInstance.set("serde1", serde1Instance);
Struct traitInstance = (Struct) tableInstance.getTrait(TRAIT_TYPE);
traitInstance.set("level", 1);
Struct serde2Instance = new Struct("serdeType");
serde2Instance.set("name", "serde2");
serde2Instance.set("serde", "serde2");
tableInstance.set("serde2", serde2Instance);
ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
return tableType.convert(tableInstance, Multiplicity.REQUIRED);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment