Skip to content

A
atlas
Commit 5273b41f by Venkatesh Seetharam
Options

Fixed an issue with persisting to graph db. guava lib confilt. Contributed by Venkatesh Seetharam

parent c8aabc11
Showing with 151 additions and 56 deletions
pom.xml
......@@ -438,12 +438,6 @@
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>18.0</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.5</version>
......
repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryService.java
......@@ -19,16 +19,18 @@
package org.apache.hadoop.metadata.services;
import com.google.common.base.Preconditions;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.service.Services;
import org.apache.hadoop.metadata.util.GraphUtils;
import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashMap;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
......@@ -74,6 +76,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
public void stop() {
// do nothing
graphService = null;
}
/**
......@@ -89,8 +92,12 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
stop();
}
private TitanGraph getGraph() {
return ((TitanGraphService) graphService).getTitanGraph();
private Graph getBlueprintsGraph() {
return graphService.getBlueprintsGraph();
}
private TransactionalGraph getTransactionalGraph() {
return graphService.getTransactionalGraph();
}
@Override
......@@ -103,18 +110,21 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
// todo check if this is a duplicate
final String guid = UUID.randomUUID().toString();
final TransactionalGraph transactionalGraph = getTransactionalGraph();
try {
getGraph().newTransaction();
transactionalGraph.rollback();
Vertex entityVertex = getGraph().addVertex(null);
Vertex entityVertex = transactionalGraph.addVertex(null);
entityVertex.setProperty("guid", guid);
entityVertex.setProperty("entityName", entityName);
entityVertex.setProperty("entityType", entityType);
for (Map.Entry<String, String> entry : properties.entrySet()) {
entityVertex.setProperty(entry.getKey(), entry.getValue());
}
} catch (Exception e) {
transactionalGraph.rollback();
} finally {
getGraph().commit();
transactionalGraph.commit();
}
return guid;
......@@ -122,36 +132,63 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
public String getEntityDefinition(String entityName, String entityType) {
Vertex entityVertex = findVertex(entityName, entityType);
Vertex entityVertex = GraphUtils.findVertex(getBlueprintsGraph(), entityName, entityType);
if (entityVertex == null) {
return null;
}
Map<String, String> properties = extractProperties(entityVertex);
Map<String, String> properties = GraphUtils.extractProperties(entityVertex);
return JSONValue.toJSONString(properties);
}
protected Vertex findVertex(String entityName, String entityType) {
LOG.debug("Finding vertex for: name={}, type={}", entityName, entityType);
GraphQuery query = getGraph().query()
.has("entityName", entityName)
.has("entityType", entityType);
Iterator<Vertex> results = query.vertices().iterator();
return results.hasNext() ? results.next() : null; // returning one since name/type is unique
@Override
public List<String> getEntityList(String entityType) {
return Collections.emptyList();
}
private Map<String, String> extractProperties(Vertex entityVertex) {
Map<String, String> properties = new HashMap<>();
for (String key : entityVertex.getPropertyKeys()) {
properties.put(key, String.valueOf(entityVertex.getProperty(key)));
}
public static void main(String[] args) throws Exception {
TitanGraphService titanGraphService = new TitanGraphService();
titanGraphService.start();
Services.get().register(titanGraphService);
GraphBackedMetadataRepositoryService service = new GraphBackedMetadataRepositoryService();
try {
service.start();
String guid = UUID.randomUUID().toString();
return properties;
final TransactionalGraph graph = service.getTransactionalGraph();
System.out.println("graph = " + graph);
System.out.println("graph.getVertices() = " + graph.getVertices());
Vertex entityVertex = null;
try {
graph.rollback();
entityVertex = graph.addVertex(null);
entityVertex.setProperty("guid", guid);
entityVertex.setProperty("entityName", "entityName");
entityVertex.setProperty("entityType", "entityType");
} catch (Exception e) {
graph.rollback();
e.printStackTrace();
} finally {
graph.commit();
}
@Override
public List<String> getEntityList(String entityType) {
return null;
System.out.println("vertex = " + GraphUtils.vertexString(entityVertex));
GraphQuery query = graph.query()
.has("entityName", "entityName")
.has("entityType", "entityType");
Iterator<Vertex> results = query.vertices().iterator();
if (results.hasNext()) {
Vertex vertexFromQuery = results.next();
System.out.println("vertex = " + GraphUtils.vertexString(vertexFromQuery));
}
} finally {
service.stop();
titanGraphService.stop();
}
}
}
repository/src/main/java/org/apache/hadoop/metadata/services/GraphService.java
......@@ -35,7 +35,7 @@ public interface GraphService extends Service {
*
* @return an handle to the graph db
*/
Graph getGraph();
Graph getBlueprintsGraph();
KeyIndexableGraph getIndexableGraph();
......
repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphService.java
......@@ -23,7 +23,6 @@ import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.graphdb.blueprints.TitanBlueprintsGraph;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Graph;
......@@ -51,11 +50,11 @@ public class TitanGraphService implements GraphService {
/**
* Constant for the configuration property that indicates the prefix.
*/
private static final String METADATA_PREFIX = "metadata.graph.";
private static final String METADATA_PREFIX = "metadata.titanGraph.";
private static final String METADATA_INDEX_KEY = "index.name";
private Configuration graphConfig;
private Graph graph;
private TitanGraph titanGraph;
private Set<String> vertexIndexedKeys;
private Set<String> edgeIndexedKeys;
......@@ -76,13 +75,13 @@ public class TitanGraphService implements GraphService {
*/
@Override
public void start() throws Exception {
graphConfig = getConfiguration();
// graphConfig = getConfiguration();
graph = initializeGraphDB();
titanGraph = initializeGraphDB();
// createIndicesForVertexKeys();
// todo - create Edge Cardinality Constraints
LOG.info("Initialized graph db: {}", graph);
LOG.info("Initialized titanGraph db: {}", titanGraph);
vertexIndexedKeys = getIndexableGraph().getIndexedKeys(Vertex.class);
LOG.info("Init vertex property keys: {}", vertexIndexedKeys);
......@@ -91,12 +90,14 @@ public class TitanGraphService implements GraphService {
LOG.info("Init edge property keys: {}", edgeIndexedKeys);
}
protected Graph initializeGraphDB() {
LOG.info("Initializing graph db");
// return GraphFactory.open(graphConfig);
protected TitanGraph initializeGraphDB() {
LOG.info("Initializing titanGraph db");
// todo: externalize this
Configuration graphConfig = new PropertiesConfiguration();
graphConfig.setProperty("storage.backend", "berkeleyje");
graphConfig.setProperty("storage.directory", "target/data/graphdb");
return TitanFactory.open(graphConfig);
}
......@@ -124,12 +125,12 @@ public class TitanGraphService implements GraphService {
* com.tinkerpop.blueprints.KeyIndexableGraph#createKeyIndex does not create an index.
*/
protected void createIndicesForVertexKeys() {
if (!((KeyIndexableGraph) graph).getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for graph");
if (!((KeyIndexableGraph) titanGraph).getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for titanGraph");
return;
}
LOG.info("Indexes does not exist, Creating indexes for graph");
LOG.info("Indexes does not exist, Creating indexes for titanGraph");
// todo - externalize this
String indexName = graphConfig.getString(METADATA_INDEX_KEY);
PropertyKey guid = createPropertyKey("guid", String.class, Cardinality.SINGLE);
......@@ -169,7 +170,9 @@ public class TitanGraphService implements GraphService {
*/
@Override
public void stop() {
if (titanGraph != null) {
titanGraph.shutdown();
}
}
/**
......@@ -186,26 +189,22 @@ public class TitanGraphService implements GraphService {
}
@Override
public Graph getGraph() {
return graph;
public Graph getBlueprintsGraph() {
return titanGraph;
}
@Override
public KeyIndexableGraph getIndexableGraph() {
return (KeyIndexableGraph) graph;
return titanGraph;
}
@Override
public TransactionalGraph getTransactionalGraph() {
return (TransactionalGraph) graph;
}
protected TitanBlueprintsGraph getTitanBlueprintsGraph() {
return (TitanBlueprintsGraph) graph;
return titanGraph;
}
public TitanGraph getTitanGraph() {
return (TitanGraph) graph;
return titanGraph;
}
@Override
......
repository/src/main/java/org/apache/hadoop/metadata/util/GraphUtils.java 0 → 100644
package org.apache.hadoop.metadata.util;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
/**
* Utility class for graph operations.
*/
public final class GraphUtils {
private static final Logger LOG = LoggerFactory.getLogger(GraphUtils.class);
private GraphUtils() {
}
public static Vertex findVertex(Graph blueprintsGraph,
String entityName, String entityType) {
LOG.debug("Finding vertex for: name={}, type={}", entityName, entityType);
GraphQuery query = blueprintsGraph.query()
.has("entityName", entityName)
.has("entityType", entityType);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since name/type is unique
return results.hasNext() ? results.next() : null;
}
public static Map<String, String> extractProperties(Vertex entityVertex) {
Map<String, String> properties = new HashMap<>();
for (String key : entityVertex.getPropertyKeys()) {
properties.put(key, String.valueOf(entityVertex.getProperty(key)));
}
return properties;
}
public static String vertexString(final Vertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
properties.append(propertyKey)
.append("=").append(vertex.getProperty(propertyKey))
.append(", ");
}
return "v[" + vertex.getId() + "], Properties[" + properties + "]";
}
public static String edgeString(final Edge edge) {
return "e[" + edge.getLabel() + "], ["
+ edge.getVertex(Direction.OUT).getProperty("name")
+ " -> " + edge.getLabel() + " -> "
+ edge.getVertex(Direction.IN).getProperty("name")
+ "]";
}
}
\ No newline at end of file
webapp/src/main/java/org/apache/hadoop/metadata/web/resources/GraphResource.java
......@@ -75,7 +75,7 @@ public class GraphResource {
}
protected Graph getGraph() {
return graphService.getGraph();
return graphService.getBlueprintsGraph();
}
protected Set<String> getVertexIndexedKeys() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment