Skip to content

A
atlas
Commit be7c4cb6 by Venkatesh Seetharam
Options

ISSUE-38 Map type to graph with type prefixes to enable search. Contributed by Venkatesh Seetharam

parent 1d1531ef
Showing with 254 additions and 150 deletions
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
...@@ -60,7 +60,6 @@ import java.util.Collections; ...@@ -60,7 +60,6 @@ import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
/** /**
...@@ -135,6 +134,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -135,6 +134,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throw new RepositoryException(e); throw new RepositoryException(e);
} finally { } finally {
transactionalGraph.commit(); transactionalGraph.commit();
GraphUtils.dumpToLog(transactionalGraph);
} }
} }
...@@ -146,9 +146,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -146,9 +146,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
try { try {
Vertex instanceVertex = GraphUtils.findVertex(graph, Constants.GUID_PROPERTY_KEY, guid); Vertex instanceVertex = GraphUtils.findVertex(graph, Constants.GUID_PROPERTY_KEY, guid);
if (instanceVertex == null) { if (instanceVertex == null) {
LOG.debug("Could not find a vertex for guid {}", guid);
return null; return null;
} }
LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid);
return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex); return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
} catch (Exception e) { } catch (Exception e) {
...@@ -260,37 +262,30 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -260,37 +262,30 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public void createVerticesForClassTypes(TransactionalGraph transactionalGraph, public void createVerticesForClassTypes(TransactionalGraph transactionalGraph,
List<ITypedReferenceableInstance> newInstances) { List<ITypedReferenceableInstance> newInstances) {
for (ITypedReferenceableInstance typedInstance : newInstances) { for (ITypedReferenceableInstance typedInstance : newInstances) {
final Vertex instanceVertex = transactionalGraph.addVertex(null); final Vertex instanceVertex =
instanceVertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typedInstance.getTypeName()); GraphUtils.createVertex(transactionalGraph, typedInstance);
// entityVertex.setProperty("entityName", instance.getString("name")); idToVertexMap.put(typedInstance.getId(), instanceVertex);
final String guid = UUID.randomUUID().toString();
instanceVertex.setProperty(Constants.GUID_PROPERTY_KEY, guid);
final Id typedInstanceId = typedInstance.getId();
instanceVertex.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
idToVertexMap.put(typedInstanceId, instanceVertex);
} }
} }
} }
private final class TypedInstanceToGraphMapper { private final class TypedInstanceToGraphMapper {
private String mapTypedInstanceToGraph(IReferenceableInstance entity, private String mapTypedInstanceToGraph(IReferenceableInstance typedInstance,
TransactionalGraph transactionalGraph) TransactionalGraph transactionalGraph)
throws MetadataException { throws MetadataException {
EntityProcessor entityProcessor = new EntityProcessor(); EntityProcessor entityProcessor = new EntityProcessor();
try { try {
new ObjectGraphWalker(typeSystem, entityProcessor, entity).walk(); LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName());
new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk();
} catch (MetadataException me) { } catch (MetadataException me) {
throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me); throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
} }
List<ITypedReferenceableInstance> newTypedInstances = discoverInstances(entityProcessor); List<ITypedReferenceableInstance> newTypedInstances = discoverInstances(entityProcessor);
entityProcessor.createVerticesForClassTypes(transactionalGraph, newTypedInstances); entityProcessor.createVerticesForClassTypes(transactionalGraph, newTypedInstances);
return addDiscoveredInstances(entity, entityProcessor, newTypedInstances); return addDiscoveredInstances(typedInstance, entityProcessor, newTypedInstances);
} }
/* /*
...@@ -303,7 +298,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -303,7 +298,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throws RepositoryException { throws RepositoryException {
List<ITypedReferenceableInstance> newTypedInstances = new ArrayList<>(); List<ITypedReferenceableInstance> newTypedInstances = new ArrayList<>();
for (IReferenceableInstance transientInstance : entityProcessor.idToInstanceMap.values()) { for (IReferenceableInstance transientInstance : entityProcessor.idToInstanceMap.values()) {
LOG.debug("instance {}", transientInstance); LOG.debug("Discovered instance {}", transientInstance.getTypeName());
try { try {
ClassType cT = typeSystem.getDataType( ClassType cT = typeSystem.getDataType(
ClassType.class, transientInstance.getTypeName()); ClassType.class, transientInstance.getTypeName());
...@@ -331,13 +326,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -331,13 +326,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
EntityProcessor entityProcessor, EntityProcessor entityProcessor,
List<ITypedReferenceableInstance> newTypedInstances) List<ITypedReferenceableInstance> newTypedInstances)
throws MetadataException { throws MetadataException {
String typedInstanceGUID = null;
String guid = null;
for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse over newInstances for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse over newInstances
LOG.debug("Adding typed instance {}", typedInstance);
Id id = typedInstance.getId(); Id id = typedInstance.getId();
if (id == null) { if (id == null) { // oops
// oops
throw new RepositoryException("id cannot be null"); throw new RepositoryException("id cannot be null");
} }
...@@ -346,43 +340,40 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -346,43 +340,40 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
// add the attributes for the instance // add the attributes for the instance
final Map<String, AttributeInfo> fields = typedInstance.fieldMapping().fields; final Map<String, AttributeInfo> fields = typedInstance.fieldMapping().fields;
addInstanceToVertex(typedInstance, instanceVertex, fields, mapInstanceToVertex(typedInstance, instanceVertex, fields, entityProcessor.idToVertexMap);
entityProcessor.idToVertexMap);
for (String traitName : typedInstance.getTraits()) { for (String traitName : typedInstance.getTraits()) {
LOG.debug("mapping trait {}", traitName);
((TitanVertex) instanceVertex).addProperty("traits", traitName); ((TitanVertex) instanceVertex).addProperty("traits", traitName);
ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName); ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
// add the attributes for the trait instance
final String vertexPropertyName = typedInstance.getTypeName() + "." + traitName;
instanceVertex.setProperty(vertexPropertyName, traitName);
addInstanceToVertex(traitInstance, instanceVertex, // add the attributes for the trait instance
traitInstance.fieldMapping().fields, mapTraitInstanceToVertex(traitName, traitInstance, typedInstance,
entityProcessor.idToVertexMap); instanceVertex, entityProcessor.idToVertexMap);
} }
if (typedInstance.getId() == entity.getId()) { if (typedInstance.getId() == entity.getId()) { // save the guid for return
guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY); typedInstanceGUID = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
} }
} }
return guid; return typedInstanceGUID;
} }
private void addInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex, private void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
Map<String, AttributeInfo> fields, Map<String, AttributeInfo> fields,
Map<Id, Vertex> idToVertexMap) throws MetadataException { Map<Id, Vertex> idToVertexMap) throws MetadataException {
LOG.debug("Mapping instance {} to vertex {} for fields {}",
typedInstance, instanceVertex, fields);
for (AttributeInfo attributeInfo : fields.values()) { for (AttributeInfo attributeInfo : fields.values()) {
System.out.println("*** attributeInfo = " + attributeInfo); LOG.debug("mapping attributeInfo {}", attributeInfo);
final IDataType dataType = attributeInfo.dataType(); final IDataType dataType = attributeInfo.dataType();
final Object attributeValue = typedInstance.get(attributeInfo.name);
final String vertexPropertyName = final String vertexPropertyName =
typedInstance.getTypeName() + "." + attributeInfo.name; typedInstance.getTypeName() + "." + attributeInfo.name;
switch (dataType.getTypeCategory()) { switch (dataType.getTypeCategory()) {
case PRIMITIVE: case PRIMITIVE:
addPrimitiveToVertex(typedInstance, instanceVertex, attributeInfo); mapPrimitiveToVertex(typedInstance, instanceVertex, attributeInfo);
break; break;
case ENUM: case ENUM:
...@@ -399,23 +390,23 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -399,23 +390,23 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break; break;
case STRUCT: case STRUCT:
ITypedStruct structInstance = (ITypedStruct) attributeValue; mapStructInstanceToVertex(typedInstance, instanceVertex,
addInstanceToVertex(structInstance, instanceVertex, attributeInfo, idToVertexMap);
structInstance.fieldMapping().fields, idToVertexMap);
break; break;
case TRAIT: case TRAIT:
ITypedStruct traitInstance = (ITypedStruct) attributeValue; // do NOTHING - this is taken care of earlier
addInstanceToVertex(traitInstance, instanceVertex,
traitInstance.fieldMapping().fields, idToVertexMap);
break; break;
case CLASS: case CLASS:
Id id = (Id) typedInstance.get(attributeInfo.name); Id id = (Id) typedInstance.get(attributeInfo.name);
if (id != null) { if (id != null) {
Vertex referenceVertex = idToVertexMap.get(id); Vertex referenceVertex = idToVertexMap.get(id);
GraphUtils.addEdge(instanceVertex, referenceVertex, String relationshipLabel =
Constants.GUID_PROPERTY_KEY, id.className); typedInstance.getTypeName() + "." + attributeInfo.name;
LOG.debug("Adding edge {} -> label {} -> v{}",
instanceVertex, relationshipLabel, referenceVertex);
instanceVertex.addEdge(relationshipLabel, referenceVertex);
} }
break; break;
...@@ -425,9 +416,53 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -425,9 +416,53 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} }
} }
private void addPrimitiveToVertex(ITypedInstance typedInstance, private void mapStructInstanceToVertex(ITypedInstance typedInstance,
Vertex parentInstanceVertex,
AttributeInfo attributeInfo,
Map<Id, Vertex> idToVertexMap) throws MetadataException {
ITypedStruct structInstance = (ITypedStruct) typedInstance.get(attributeInfo.name);
// add a new vertex for the struct or trait instance
Vertex structInstanceVertex = GraphUtils.createVertex(
graphService.getBlueprintsGraph(), structInstance,
((ITypedReferenceableInstance) typedInstance).getId());
LOG.debug("created vertex {} for struct {}", structInstanceVertex, attributeInfo.name);
// map all the attributes to this newly created vertex
mapInstanceToVertex(structInstance, structInstanceVertex,
structInstance.fieldMapping().fields, idToVertexMap);
// add an edge to the newly created vertex from the parent
String relationshipLabel = attributeInfo.dataType().getName() + "." + attributeInfo.name;
LOG.debug("Adding edge for {} -> label {} -> v{}",
parentInstanceVertex, relationshipLabel, structInstanceVertex);
parentInstanceVertex.addEdge(relationshipLabel, structInstanceVertex);
}
private void mapTraitInstanceToVertex(String traitName, ITypedStruct traitInstance,
ITypedReferenceableInstance typedInstance,
Vertex parentInstanceVertex,
Map<Id, Vertex> idToVertexMap) throws MetadataException {
// add a new vertex for the struct or trait instance
Vertex traitInstanceVertex = GraphUtils.createVertex(
graphService.getBlueprintsGraph(), traitInstance, typedInstance.getId());
LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
// map all the attributes to this newly created vertex
mapInstanceToVertex(traitInstance, traitInstanceVertex,
traitInstance.fieldMapping().fields, idToVertexMap);
// add an edge to the newly created vertex from the parent
String relationshipLabel = typedInstance.getTypeName() + "." + traitName;
LOG.debug("Adding edge for {} -> label {} -> v{}",
parentInstanceVertex, relationshipLabel, traitInstanceVertex);
parentInstanceVertex.addEdge(relationshipLabel, traitInstanceVertex);
}
private void mapPrimitiveToVertex(ITypedInstance typedInstance,
Vertex instanceVertex, Vertex instanceVertex,
AttributeInfo attributeInfo) throws MetadataException { AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("Adding primitive {} to v {}", attributeInfo, instanceVertex);
if (typedInstance.get(attributeInfo.name) == null) { // add only if instance has this attribute if (typedInstance.get(attributeInfo.name) == null) { // add only if instance has this attribute
return; return;
} }
...@@ -473,28 +508,44 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -473,28 +508,44 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private ITypedReferenceableInstance mapGraphToTypedInstance(String guid, private ITypedReferenceableInstance mapGraphToTypedInstance(String guid,
Vertex instanceVertex) Vertex instanceVertex)
throws MetadataException { throws MetadataException {
LOG.debug("Mapping graph root vertex {} to typed instance for guid {}",
instanceVertex, guid);
String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY); String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
List<String> traits = new ArrayList<>(); List<String> traits = new ArrayList<>();
for (TitanProperty property : ((TitanVertex) instanceVertex).getProperties( "traits")) { for (TitanProperty property : ((TitanVertex) instanceVertex).getProperties("traits")) {
traits.add((String) property.getValue()); traits.add((String) property.getValue());
} }
Id id = new Id(guid, instanceVertex.<Integer>getProperty("version"), typeName); Id id = new Id(guid, instanceVertex.<Integer>getProperty("version"), typeName);
LOG.debug("Created id {} for instance type {}", id, typeName);
ClassType classType = typeSystem.getDataType(ClassType.class, typeName); ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
ITypedReferenceableInstance typedInstance = classType.createInstance( ITypedReferenceableInstance typedInstance = classType.createInstance(
id, traits.toArray(new String[traits.size()])); id, traits.toArray(new String[traits.size()]));
graphToInstanceMapper.mapVertexToInstance( mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
instanceVertex, typedInstance, classType.fieldMapping().fields); mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
return typedInstance; return typedInstance;
} }
private void mapVertexToInstanceTraits(Vertex instanceVertex,
ITypedReferenceableInstance typedInstance,
List<String> traits) throws MetadataException {
for (String traitName : traits) {
LOG.debug("mapping trait {} to instance", traitName);
TraitType traitType = typeSystem.getDataType(TraitType.class, traitName);
mapVertexToTraitInstance(
instanceVertex, typedInstance, traitName, traitType);
}
}
private void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance, private void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
Map<String, AttributeInfo> fields) throws MetadataException { Map<String, AttributeInfo> fields) throws MetadataException {
LOG.debug("Mapping vertex {} to instance {} for fields",
instanceVertex, typedInstance.getTypeName(), fields);
for (AttributeInfo attributeInfo : fields.values()) { for (AttributeInfo attributeInfo : fields.values()) {
System.out.println("*** attributeInfo = " + attributeInfo); LOG.debug("mapping attributeInfo = " + attributeInfo);
final IDataType dataType = attributeInfo.dataType(); final IDataType dataType = attributeInfo.dataType();
final String vertexPropertyName = final String vertexPropertyName =
typedInstance.getTypeName() + "." + attributeInfo.name; typedInstance.getTypeName() + "." + attributeInfo.name;
...@@ -505,9 +556,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -505,9 +556,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break; // add only if vertex has this attribute break; // add only if vertex has this attribute
case ENUM: case ENUM:
// EnumType enumType = typeSystem.getDataType(
// EnumType.class, attributeInfo.name);
// todo - is this enough
typedInstance.setInt(attributeInfo.name, typedInstance.setInt(attributeInfo.name,
instanceVertex.<Integer>getProperty(vertexPropertyName)); instanceVertex.<Integer>getProperty(vertexPropertyName));
break; break;
...@@ -521,45 +569,83 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -521,45 +569,83 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break; break;
case STRUCT: case STRUCT:
StructType structType = typeSystem.getDataType( mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo);
StructType.class, attributeInfo.name);
ITypedStruct structInstance = structType.createInstance();
typedInstance.set(attributeInfo.name, structInstance);
mapVertexToInstance(instanceVertex, structInstance,
structInstance.fieldMapping().fields);
break; break;
case TRAIT: case TRAIT:
TraitType traitType = typeSystem.getDataType( // do NOTHING - handled in class
TraitType.class, attributeInfo.name); // mapVertexToCompositeInstance(instanceVertex, typedInstance, attributeInfo);
ITypedStruct traitInstance = (ITypedStruct)
((ITypedReferenceableInstance) typedInstance).getTrait(attributeInfo.name);
typedInstance.set(attributeInfo.name, traitInstance);
mapVertexToInstance(instanceVertex, traitInstance,
traitType.fieldMapping().fields);
break; break;
case CLASS: case CLASS:
Id referenceId = null; String relationshipLabel = typedInstance.getTypeName() + "." + attributeInfo.name;
for (Edge edge : instanceVertex.getEdges(Direction.IN)) { LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
final Vertex vertex = edge.getVertex(Direction.OUT); for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
if (vertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY).equals(attributeInfo.name)) { final Vertex referenceVertex = edge.getVertex(Direction.IN);
referenceId = new Id( if (referenceVertex != null) {
vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY), final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), LOG.debug("Found vertex {} for label {} with guid {}",
referenceVertex, relationshipLabel, guid);
if (attributeInfo.isComposite) {
LOG.debug("Found composite, mapping vertex to instance");
typedInstance.set(attributeInfo.name,
mapGraphToTypedInstance(guid, referenceVertex));
} else {
Id referenceId = new Id(guid,
referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
attributeInfo.name); attributeInfo.name);
LOG.debug("Found non-composite, adding id {} ", referenceId);
typedInstance.set(attributeInfo.name, referenceId);
}
break; break;
} }
} }
break;
if (referenceId != null) { default:
typedInstance.set(attributeInfo.name, referenceId); break;
}
}
} }
private void mapVertexToStructInstance(Vertex instanceVertex,
ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("mapping vertex {} to struct {}", typedInstance, attributeInfo.name);
StructType structType = typeSystem.getDataType(
StructType.class, attributeInfo.dataType().getName());
ITypedStruct structInstance = structType.createInstance();
typedInstance.set(attributeInfo.name, structInstance);
String relationshipLabel = typedInstance.getTypeName() + "." + attributeInfo.name;
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
if (structInstanceVertex != null) {
LOG.debug("Found struct instance vertex {}, mapping to instance {} ",
structInstanceVertex, structInstance.getTypeName());
mapVertexToInstance(structInstanceVertex, structInstance,
structType.fieldMapping().fields);
break; break;
}
}
}
default: private void mapVertexToTraitInstance(Vertex instanceVertex,
ITypedReferenceableInstance typedInstance,
String traitName,
TraitType traitType) throws MetadataException {
ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
String relationshipLabel = typedInstance.getTypeName() + "." + traitName;
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
final Vertex traitInstanceVertex = edge.getVertex(Direction.IN);
if (traitInstanceVertex != null) {
LOG.debug("Found trait instance vertex {}, mapping to instance {} ",
traitInstanceVertex, traitInstance.getTypeName());
mapVertexToInstance(traitInstanceVertex, traitInstance,
traitType.fieldMapping().fields);
break; break;
} }
} }
...@@ -568,6 +654,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -568,6 +654,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private void mapVertexToInstance(Vertex instanceVertex, private void mapVertexToInstance(Vertex instanceVertex,
ITypedInstance typedInstance, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException { AttributeInfo attributeInfo) throws MetadataException {
LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
final String vertexPropertyName = typedInstance.getTypeName() + "." + attributeInfo.name; final String vertexPropertyName = typedInstance.getTypeName() + "." + attributeInfo.name;
if (instanceVertex.getProperty(vertexPropertyName) == null) { if (instanceVertex.getProperty(vertexPropertyName) == null) {
return; return;
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphUtils.java
...@@ -23,10 +23,14 @@ import com.tinkerpop.blueprints.Edge; ...@@ -23,10 +23,14 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph; import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery; import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.ITypedInstance;
import org.apache.hadoop.metadata.ITypedReferenceableInstance;
import org.apache.hadoop.metadata.storage.Id;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.Iterator; import java.util.Iterator;
import java.util.UUID;
/** /**
* Utility class for graph operations. * Utility class for graph operations.
...@@ -38,40 +42,26 @@ public final class GraphUtils { ...@@ -38,40 +42,26 @@ public final class GraphUtils {
private GraphUtils() { private GraphUtils() {
} }
public static Edge addEdge(Vertex fromVertex, Vertex toVertex, public static Vertex createVertex(Graph graph,
String vertexPropertyKey, String edgeLabel) { ITypedReferenceableInstance typedInstance) {
return addEdge(fromVertex, toVertex, vertexPropertyKey, edgeLabel, null); return createVertex(graph, typedInstance, typedInstance.getId());
} }
public static Edge addEdge(Vertex fromVertex, Vertex toVertex, public static Vertex createVertex(Graph graph,
String vertexPropertyKey, String edgeLabel, String timestamp) { ITypedInstance typedInstance,
Edge edge = findEdge(fromVertex, toVertex, vertexPropertyKey, edgeLabel); Id typedInstanceId) {
final Vertex instanceVertex = graph.addVertex(null);
// type
instanceVertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typedInstance.getTypeName());
Edge edgeToVertex = edge != null ? edge : fromVertex.addEdge(edgeLabel, toVertex); // id
if (timestamp != null) { final String guid = UUID.randomUUID().toString();
edgeToVertex.setProperty(Constants.TIMESTAMP_PROPERTY_KEY, timestamp); instanceVertex.setProperty(Constants.GUID_PROPERTY_KEY, guid);
}
return edgeToVertex;
}
public static Edge findEdge(Vertex fromVertex, Vertex toVertex, // version
String vertexPropertyKey, String edgeLabel) { instanceVertex.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
return findEdge(fromVertex, toVertex.getProperty(vertexPropertyKey),
vertexPropertyKey, edgeLabel);
}
public static Edge findEdge(Vertex fromVertex, Object toVertexName,
String vertexPropertyKey, String edgeLabel) {
Edge edgeToFind = null;
for (Edge edge : fromVertex.getEdges(Direction.OUT, edgeLabel)) {
if (edge.getVertex(Direction.IN).getProperty(vertexPropertyKey).equals(toVertexName)) {
edgeToFind = edge;
break;
}
}
return edgeToFind; return instanceVertex;
} }
public static Vertex findVertex(Graph blueprintsGraph, public static Vertex findVertex(Graph blueprintsGraph,
...@@ -97,23 +87,23 @@ public final class GraphUtils { ...@@ -97,23 +87,23 @@ public final class GraphUtils {
public static String edgeString(final Edge edge) { public static String edgeString(final Edge edge) {
return "e[" + edge.getLabel() + "], [" return "e[" + edge.getLabel() + "], ["
+ edge.getVertex(Direction.OUT).getProperty("name") + edge.getVertex(Direction.OUT)
+ " -> " + edge.getLabel() + " -> " + " -> " + edge.getLabel() + " -> "
+ edge.getVertex(Direction.IN).getProperty("name") + edge.getVertex(Direction.IN)
+ "]"; + "]";
} }
public static void dumpToLog(final Graph graph) { public static void dumpToLog(final Graph graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph); LOG.debug("Vertices of {}", graph);
for (Vertex vertex : graph.getVertices()) { for (Vertex vertex : graph.getVertices()) {
LOG.debug(vertexString(vertex)); LOG.debug(vertexString(vertex));
System.out.println(vertexString(vertex));
} }
LOG.debug("Edges of {}", graph); LOG.debug("Edges of {}", graph);
for (Edge edge : graph.getEdges()) { for (Edge edge : graph.getEdges()) {
LOG.debug(edgeString(edge)); LOG.debug(edgeString(edge));
System.out.println(edgeString(edge));
} }
LOG.debug("*******************Graph Dump****************************");
} }
} }
\ No newline at end of file
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
...@@ -34,15 +34,16 @@ import org.apache.hadoop.metadata.types.Multiplicity; ...@@ -34,15 +34,16 @@ import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition; import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.types.TraitType; import org.apache.hadoop.metadata.types.TraitType;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import scala.actors.threadpool.Arrays;
import javax.ws.rs.HttpMethod; import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID; import java.util.UUID;
/** /**
...@@ -50,28 +51,29 @@ import java.util.UUID; ...@@ -50,28 +51,29 @@ import java.util.UUID;
*/ */
public class EntityJerseyResourceIT extends BaseResourceIT { public class EntityJerseyResourceIT extends BaseResourceIT {
private static final Logger LOG = LoggerFactory.getLogger(EntityJerseyResourceIT.class);
private static final String DATABASE_TYPE = "hive_database"; private static final String DATABASE_TYPE = "hive_database";
private static final String DATABASE_NAME = "foo"; private static final String DATABASE_NAME = "foo";
private static final String TABLE_TYPE = "hive_table"; private static final String TABLE_TYPE = "hive_table";
private static final String TABLE_NAME = "bar"; private static final String TABLE_NAME = "bar";
private static final String TRAIT_TYPE = "hive_fetl";
private String tableInstanceAsJSON; private ITypedReferenceableInstance tableInstance;
private String guid; private String guid;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
super.setUp(); super.setUp();
List<HierarchicalTypeDefinition> typeDefinitions = createHiveTypes(); createHiveTypes();
submitTypes(typeDefinitions); submitTypes();
} }
@Test @Test
public void testSubmitEntity() throws Exception { public void testSubmitEntity() throws Exception {
ITypedReferenceableInstance tableInstance = createHiveTableInstance(); tableInstance = createHiveTableInstance();
String tableInstanceAsJSON = Serialization$.MODULE$.toJson(tableInstance);
tableInstanceAsJSON = Serialization$.MODULE$.toJson(tableInstance); LOG.debug("tableInstance = " + tableInstanceAsJSON);
WebResource resource = service WebResource resource = service
.path("api/metadata/entities/submit") .path("api/metadata/entities/submit")
...@@ -119,12 +121,24 @@ public class EntityJerseyResourceIT extends BaseResourceIT { ...@@ -119,12 +121,24 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
final String definition = response.getString("definition"); final String definition = response.getString("definition");
Assert.assertNotNull(definition); Assert.assertNotNull(definition);
LOG.debug("tableInstanceAfterGet = " + definition);
// todo - this fails with type error, strange
// ITypedReferenceableInstance tableInstanceAfterGet = Serialization$.MODULE$.fromJson(definition);
// Assert.assertTrue(areEqual(tableInstance, tableInstanceAfterGet));
}
System.out.println("definition = " + definition); /*
System.out.println("tableInstanceAsJSON = " + tableInstanceAsJSON); private boolean areEqual(ITypedReferenceableInstance actual,
// Assert.assertEquals(definition, tableInstanceAsJSON); ITypedReferenceableInstance expected) throws Exception {
for (AttributeInfo attributeInfo : actual.fieldMapping().fields.values()) {
Assert.assertEquals(actual.get(attributeInfo.name), expected.get(attributeInfo.name));
} }
return true;
}
*/
@Test @Test
public void testGetInvalidEntityDefinition() { public void testGetInvalidEntityDefinition() {
WebResource resource = service WebResource resource = service
...@@ -165,15 +179,19 @@ public class EntityJerseyResourceIT extends BaseResourceIT { ...@@ -165,15 +179,19 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
System.out.println("response = " + response); System.out.println("response = " + response);
} }
private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception { private void createHiveTypes() throws Exception {
ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
createClassTypeDef(DATABASE_TYPE, createClassTypeDef(DATABASE_TYPE,
ImmutableList.<String>of(), ImmutableList.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE)); createRequiredAttrDef("description", DataTypes.STRING_TYPE));
typeDefinitions.add(databaseTypeDefinition);
StructTypeDefinition structTypeDefinition =
new StructTypeDefinition("serdeType",
new AttributeDefinition[] {
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
});
HierarchicalTypeDefinition<ClassType> tableTypeDefinition = HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
createClassTypeDef(TABLE_TYPE, createClassTypeDef(TABLE_TYPE,
...@@ -181,32 +199,31 @@ public class EntityJerseyResourceIT extends BaseResourceIT { ...@@ -181,32 +199,31 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
createRequiredAttrDef("name", DataTypes.STRING_TYPE), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE), createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE),
new AttributeDefinition(DATABASE_TYPE, new AttributeDefinition("serde1",
DATABASE_TYPE, Multiplicity.REQUIRED, true, DATABASE_TYPE)); "serdeType", Multiplicity.REQUIRED, false, null),
typeDefinitions.add(tableTypeDefinition); new AttributeDefinition("serde2",
"serdeType", Multiplicity.REQUIRED, false, null),
HierarchicalTypeDefinition<TraitType> fetlTypeDefinition = new AttributeDefinition("database",
createTraitTypeDef(TRAIT_TYPE, DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
createTraitTypeDef("classification",
ImmutableList.<String>of(), ImmutableList.<String>of(),
createRequiredAttrDef("level", DataTypes.INT_TYPE)); createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
typeDefinitions.add(fetlTypeDefinition);
typeSystem.defineTypes( typeSystem.defineTypes(
ImmutableList.<StructTypeDefinition>of(), ImmutableList.of(structTypeDefinition),
ImmutableList.of(fetlTypeDefinition), ImmutableList.of(classificationTypeDefinition),
ImmutableList.of(databaseTypeDefinition, tableTypeDefinition)); ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
return typeDefinitions;
} }
private void submitTypes(List<HierarchicalTypeDefinition> typeDefinitions) throws Exception { private void submitTypes() throws Exception {
for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) { String typesAsJSON = TypesSerialization.toJson(typeSystem,
String typesAsJSON = TypesSerialization.toJson( Arrays.asList(new String[]{DATABASE_TYPE, TABLE_TYPE, "serdeType", "classification"}));
typeSystem, typeDefinition.typeName);
WebResource resource = service WebResource resource = service
.path("api/metadata/types/submit") .path("api/metadata/types/submit")
.path(typeDefinition.typeName); .path(TABLE_TYPE);
ClientResponse clientResponse = resource ClientResponse clientResponse = resource
.accept(MediaType.APPLICATION_JSON) .accept(MediaType.APPLICATION_JSON)
...@@ -218,25 +235,34 @@ public class EntityJerseyResourceIT extends BaseResourceIT { ...@@ -218,25 +235,34 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.assertNotNull(responseAsString); Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString); JSONObject response = new JSONObject(responseAsString);
Assert.assertEquals(response.get("typeName"), typeDefinition.typeName); Assert.assertEquals(response.get("typeName"), TABLE_TYPE);
Assert.assertNotNull(response.get("types")); Assert.assertNotNull(response.get("types"));
Assert.assertNotNull(response.get("requestId")); Assert.assertNotNull(response.get("requestId"));
} }
}
protected ITypedReferenceableInstance createHiveTableInstance() throws Exception { protected ITypedReferenceableInstance createHiveTableInstance() throws Exception {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set("name", DATABASE_NAME); databaseInstance.set("name", DATABASE_NAME);
databaseInstance.set("description", "foo database"); databaseInstance.set("description", "foo database");
Referenceable tableInstance = new Referenceable(TABLE_TYPE, TRAIT_TYPE); Referenceable tableInstance = new Referenceable(TABLE_TYPE, "classification");
tableInstance.set("name", TABLE_NAME); tableInstance.set("name", TABLE_NAME);
tableInstance.set("description", "bar table"); tableInstance.set("description", "bar table");
tableInstance.set("type", "managed"); tableInstance.set("type", "managed");
tableInstance.set(DATABASE_TYPE, databaseInstance); tableInstance.set("database", databaseInstance);
Struct traitInstance = (Struct) tableInstance.getTrait("classification");
traitInstance.set("tag", "foundation_etl");
Struct serde1Instance = new Struct("serdeType");
serde1Instance.set("name", "serde1");
serde1Instance.set("serde", "serde1");
tableInstance.set("serde1", serde1Instance);
Struct traitInstance = (Struct) tableInstance.getTrait(TRAIT_TYPE); Struct serde2Instance = new Struct("serdeType");
traitInstance.set("level", 1); serde2Instance.set("name", "serde2");
serde2Instance.set("serde", "serde2");
tableInstance.set("serde2", serde2Instance);
ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE); ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
return tableType.convert(tableInstance, Multiplicity.REQUIRED); return tableType.convert(tableInstance, Multiplicity.REQUIRED);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment