Commit 8936008d by Venkatesh Seetharam

ISSUE-3 Externalize graph configuration. Contributed by Venkatesh Seetharam

parent 5273b41f
...@@ -36,15 +36,13 @@ public class ServiceInitializer { ...@@ -36,15 +36,13 @@ public class ServiceInitializer {
public void initialize() throws MetadataException { public void initialize() throws MetadataException {
String serviceClassNames; String serviceClassNames;
try { try {
PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties"); PropertiesConfiguration configuration =
new PropertiesConfiguration("application.properties");
serviceClassNames = configuration.getString("application.services"); serviceClassNames = configuration.getString("application.services");
} catch (ConfigurationException e) { } catch (ConfigurationException e) {
throw new MetadataException("unable to get server properties"); throw new RuntimeException("unable to get server properties");
} }
serviceClassNames
= "org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService";
for (String serviceClassName : serviceClassNames.split(",")) { for (String serviceClassName : serviceClassNames.split(",")) {
serviceClassName = serviceClassName.trim(); serviceClassName = serviceClassName.trim();
if (serviceClassName.isEmpty()) { if (serviceClassName.isEmpty()) {
......
...@@ -59,7 +59,8 @@ public final class Services implements Iterable<Service> { ...@@ -59,7 +59,8 @@ public final class Services implements Iterable<Service> {
if (services.containsKey(serviceName)) { if (services.containsKey(serviceName)) {
return (T) services.get(serviceName); return (T) services.get(serviceName);
} else { } else {
throw new NoSuchElementException("Service " + serviceName + " not registered with registry"); throw new NoSuchElementException(
"Service " + serviceName + " not registered with registry");
} }
} }
...@@ -79,7 +80,8 @@ public final class Services implements Iterable<Service> { ...@@ -79,7 +80,8 @@ public final class Services implements Iterable<Service> {
String serviceClassName; String serviceClassName;
try { try {
PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties"); PropertiesConfiguration configuration =
new PropertiesConfiguration("application.properties");
serviceClassName = configuration.getString(serviceName + ".impl"); serviceClassName = configuration.getString(serviceName + ".impl");
} catch (ConfigurationException e) { } catch (ConfigurationException e) {
throw new MetadataException("unable to get server properties"); throw new MetadataException("unable to get server properties");
......
...@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.services; ...@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.services;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.tinkerpop.blueprints.Graph; import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.TransactionalGraph; import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.service.Services; import org.apache.hadoop.metadata.service.Services;
...@@ -31,7 +30,6 @@ import org.slf4j.LoggerFactory; ...@@ -31,7 +30,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
...@@ -102,6 +100,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS ...@@ -102,6 +100,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override @Override
public String submitEntity(String entity, String entityType) { public String submitEntity(String entity, String entityType) {
LOG.info("adding entity={} type={}", entity, entityType);
Map<String, String> properties = (Map<String, String>) JSONValue.parse(entity); Map<String, String> properties = (Map<String, String>) JSONValue.parse(entity);
final String entityName = properties.get("entityName"); final String entityName = properties.get("entityName");
...@@ -132,6 +131,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS ...@@ -132,6 +131,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override @Override
public String getEntityDefinition(String entityName, String entityType) { public String getEntityDefinition(String entityName, String entityType) {
LOG.info("Retrieving entity name={} type={}", entityName, entityType);
Vertex entityVertex = GraphUtils.findVertex(getBlueprintsGraph(), entityName, entityType); Vertex entityVertex = GraphUtils.findVertex(getBlueprintsGraph(), entityName, entityType);
if (entityVertex == null) { if (entityVertex == null) {
return null; return null;
...@@ -143,52 +143,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS ...@@ -143,52 +143,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override @Override
public List<String> getEntityList(String entityType) { public List<String> getEntityList(String entityType) {
LOG.info("Retrieving entity list for type={}", entityType);
return Collections.emptyList(); return Collections.emptyList();
} }
public static void main(String[] args) throws Exception {
TitanGraphService titanGraphService = new TitanGraphService();
titanGraphService.start();
Services.get().register(titanGraphService);
GraphBackedMetadataRepositoryService service = new GraphBackedMetadataRepositoryService();
try {
service.start();
String guid = UUID.randomUUID().toString();
final TransactionalGraph graph = service.getTransactionalGraph();
System.out.println("graph = " + graph);
System.out.println("graph.getVertices() = " + graph.getVertices());
Vertex entityVertex = null;
try {
graph.rollback();
entityVertex = graph.addVertex(null);
entityVertex.setProperty("guid", guid);
entityVertex.setProperty("entityName", "entityName");
entityVertex.setProperty("entityType", "entityType");
} catch (Exception e) {
graph.rollback();
e.printStackTrace();
} finally {
graph.commit();
}
System.out.println("vertex = " + GraphUtils.vertexString(entityVertex));
GraphQuery query = graph.query()
.has("entityName", "entityName")
.has("entityType", "entityType");
Iterator<Vertex> results = query.vertices().iterator();
if (results.hasNext()) {
Vertex vertexFromQuery = results.next();
System.out.println("vertex = " + GraphUtils.vertexString(vertexFromQuery));
}
} finally {
service.stop();
titanGraphService.stop();
}
}
} }
...@@ -18,13 +18,9 @@ ...@@ -18,13 +18,9 @@
package org.apache.hadoop.metadata.services; package org.apache.hadoop.metadata.services;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Graph; import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.KeyIndexableGraph; import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.TransactionalGraph; import com.tinkerpop.blueprints.TransactionalGraph;
...@@ -50,10 +46,8 @@ public class TitanGraphService implements GraphService { ...@@ -50,10 +46,8 @@ public class TitanGraphService implements GraphService {
/** /**
* Constant for the configuration property that indicates the prefix. * Constant for the configuration property that indicates the prefix.
*/ */
private static final String METADATA_PREFIX = "metadata.titanGraph."; private static final String METADATA_PREFIX = "metadata.graph.";
private static final String METADATA_INDEX_KEY = "index.name";
private Configuration graphConfig;
private TitanGraph titanGraph; private TitanGraph titanGraph;
private Set<String> vertexIndexedKeys; private Set<String> vertexIndexedKeys;
private Set<String> edgeIndexedKeys; private Set<String> edgeIndexedKeys;
...@@ -75,11 +69,10 @@ public class TitanGraphService implements GraphService { ...@@ -75,11 +69,10 @@ public class TitanGraphService implements GraphService {
*/ */
@Override @Override
public void start() throws Exception { public void start() throws Exception {
// graphConfig = getConfiguration(); Configuration graphConfig = getConfiguration();
titanGraph = initializeGraphDB(graphConfig);
titanGraph = initializeGraphDB(); createIndicesForVertexKeys();
// createIndicesForVertexKeys();
// todo - create Edge Cardinality Constraints // todo - create Edge Cardinality Constraints
LOG.info("Initialized titanGraph db: {}", titanGraph); LOG.info("Initialized titanGraph db: {}", titanGraph);
...@@ -90,29 +83,17 @@ public class TitanGraphService implements GraphService { ...@@ -90,29 +83,17 @@ public class TitanGraphService implements GraphService {
LOG.info("Init edge property keys: {}", edgeIndexedKeys); LOG.info("Init edge property keys: {}", edgeIndexedKeys);
} }
protected TitanGraph initializeGraphDB() {
LOG.info("Initializing titanGraph db");
// todo: externalize this
Configuration graphConfig = new PropertiesConfiguration();
graphConfig.setProperty("storage.backend", "berkeleyje");
graphConfig.setProperty("storage.directory", "target/data/graphdb");
return TitanFactory.open(graphConfig);
}
private static Configuration getConfiguration() throws ConfigurationException { private static Configuration getConfiguration() throws ConfigurationException {
PropertiesConfiguration configProperties = new PropertiesConfiguration("application.properties"); PropertiesConfiguration configProperties =
new PropertiesConfiguration("application.properties");
Configuration graphConfig = new PropertiesConfiguration(); Configuration graphConfig = new PropertiesConfiguration();
final Iterator<String> iterator = configProperties.getKeys(); final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) { while (iterator.hasNext()) {
String key = iterator.next(); String key = iterator.next();
System.out.println("key = " + key);
if (key.startsWith(METADATA_PREFIX)) { if (key.startsWith(METADATA_PREFIX)) {
String value = (String) configProperties.getProperty(key); String value = (String) configProperties.getProperty(key);
key = key.substring(METADATA_PREFIX.length()); key = key.substring(METADATA_PREFIX.length());
System.out.println("**** key = " + key + ", value = " + value);
graphConfig.setProperty(key, value); graphConfig.setProperty(key, value);
} }
} }
...@@ -120,49 +101,19 @@ public class TitanGraphService implements GraphService { ...@@ -120,49 +101,19 @@ public class TitanGraphService implements GraphService {
return graphConfig; return graphConfig;
} }
/** protected TitanGraph initializeGraphDB(Configuration graphConfig) {
* This unfortunately requires a handle to Titan implementation since LOG.info("Initializing titanGraph db");
* com.tinkerpop.blueprints.KeyIndexableGraph#createKeyIndex does not create an index. return TitanFactory.open(graphConfig);
*/ }
protected void createIndicesForVertexKeys() { protected void createIndicesForVertexKeys() {
if (!((KeyIndexableGraph) titanGraph).getIndexedKeys(Vertex.class).isEmpty()) { if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for titanGraph"); LOG.info("Indexes already exist for titanGraph");
return; return;
} }
LOG.info("Indexes does not exist, Creating indexes for titanGraph"); LOG.info("Indexes does not exist, Creating indexes for titanGraph");
// todo - externalize this // todo - add index for vertex and edge property keys
String indexName = graphConfig.getString(METADATA_INDEX_KEY);
PropertyKey guid = createPropertyKey("guid", String.class, Cardinality.SINGLE);
createIndex(indexName, guid, Vertex.class, true);
getTitanGraph().commit();
}
private PropertyKey createPropertyKey(String propertyKeyName, Class<String> dataType,
Cardinality cardinality) {
PropertyKey propertyKey = getTitanGraph().getManagementSystem()
.makePropertyKey(propertyKeyName)
.dataType(dataType)
.cardinality(cardinality)
.make();
LOG.info("Created property key {}", propertyKey);
return propertyKey;
}
private void createIndex(String indexName, PropertyKey propertyKey,
Class<? extends Element> clazz, boolean isUnique) {
TitanManagement managementSystem = getTitanGraph().getManagementSystem();
managementSystem.buildPropertyIndex(propertyKey, indexName);
TitanManagement.IndexBuilder indexBuilder = managementSystem
.buildIndex(indexName, clazz)
.addKey(propertyKey);
if (isUnique) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
} }
/** /**
......
...@@ -16,23 +16,21 @@ ...@@ -16,23 +16,21 @@
# limitations under the License. # limitations under the License.
# #
application.services=org.apache.hadoop.metadata.services.TitanGraphService,\
org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
application.services=org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
#metadata.graph.schema.default=none
# Graph implementation # Graph implementation
metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory #metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
# Graph Storage # Graph Storage
metadata.graph.storage.backend=berkeleyje metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=${user.dir}/target/data/graphdb metadata.graph.storage.directory=target/data/berkeley
# Graph Search Index # Graph Search Index
#metadata.graph.index.name=search metadata.graph.index.search.backend=elasticsearch
#metadata.graph.index.search.backend=elasticsearch metadata.graph.index.search.directory=target/data/es
#metadata.graph.index.search.directory=${user.dir}/target/data/searchindex metadata.graph.index.search.elasticsearch.client-only=false
#metadata.graph.index.search.elasticsearch.client-only=false metadata.graph.index.search.elasticsearch.local-mode=true
#metadata.graph.index.search.elasticsearch.local-mode=true
metadata.enableTLS=false metadata.enableTLS=false
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment