Skip to content

A
atlas
Commit 8936008d by Venkatesh Seetharam
Options

ISSUE-3 Externalize graph configuration. Contributed by Venkatesh Seetharam

parent 5273b41f
Showing with 31 additions and 127 deletions
common/src/main/java/org/apache/hadoop/metadata/service/ServiceInitializer.java
......@@ -36,15 +36,13 @@ public class ServiceInitializer {
public void initialize() throws MetadataException {
String serviceClassNames;
try {
PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties");
PropertiesConfiguration configuration =
new PropertiesConfiguration("application.properties");
serviceClassNames = configuration.getString("application.services");
} catch (ConfigurationException e) {
throw new MetadataException("unable to get server properties");
throw new RuntimeException("unable to get server properties");
}
serviceClassNames
= "org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService";
for (String serviceClassName : serviceClassNames.split(",")) {
serviceClassName = serviceClassName.trim();
if (serviceClassName.isEmpty()) {
......
common/src/main/java/org/apache/hadoop/metadata/service/Services.java
......@@ -59,7 +59,8 @@ public final class Services implements Iterable<Service> {
if (services.containsKey(serviceName)) {
return (T) services.get(serviceName);
} else {
throw new NoSuchElementException("Service " + serviceName + " not registered with registry");
throw new NoSuchElementException(
"Service " + serviceName + " not registered with registry");
}
}
......@@ -79,7 +80,8 @@ public final class Services implements Iterable<Service> {
String serviceClassName;
try {
PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties");
PropertiesConfiguration configuration =
new PropertiesConfiguration("application.properties");
serviceClassName = configuration.getString(serviceName + ".impl");
} catch (ConfigurationException e) {
throw new MetadataException("unable to get server properties");
......
repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryService.java
......@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.services;
import com.google.common.base.Preconditions;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.service.Services;
......@@ -31,7 +30,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
......@@ -102,6 +100,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
public String submitEntity(String entity, String entityType) {
LOG.info("adding entity={} type={}", entity, entityType);
Map<String, String> properties = (Map<String, String>) JSONValue.parse(entity);
final String entityName = properties.get("entityName");
......@@ -132,6 +131,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
public String getEntityDefinition(String entityName, String entityType) {
LOG.info("Retrieving entity name={} type={}", entityName, entityType);
Vertex entityVertex = GraphUtils.findVertex(getBlueprintsGraph(), entityName, entityType);
if (entityVertex == null) {
return null;
......@@ -143,52 +143,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
@Override
public List<String> getEntityList(String entityType) {
LOG.info("Retrieving entity list for type={}", entityType);
return Collections.emptyList();
}
public static void main(String[] args) throws Exception {
TitanGraphService titanGraphService = new TitanGraphService();
titanGraphService.start();
Services.get().register(titanGraphService);
GraphBackedMetadataRepositoryService service = new GraphBackedMetadataRepositoryService();
try {
service.start();
String guid = UUID.randomUUID().toString();
final TransactionalGraph graph = service.getTransactionalGraph();
System.out.println("graph = " + graph);
System.out.println("graph.getVertices() = " + graph.getVertices());
Vertex entityVertex = null;
try {
graph.rollback();
entityVertex = graph.addVertex(null);
entityVertex.setProperty("guid", guid);
entityVertex.setProperty("entityName", "entityName");
entityVertex.setProperty("entityType", "entityType");
} catch (Exception e) {
graph.rollback();
e.printStackTrace();
} finally {
graph.commit();
}
System.out.println("vertex = " + GraphUtils.vertexString(entityVertex));
GraphQuery query = graph.query()
.has("entityName", "entityName")
.has("entityType", "entityType");
Iterator<Vertex> results = query.vertices().iterator();
if (results.hasNext()) {
Vertex vertexFromQuery = results.next();
System.out.println("vertex = " + GraphUtils.vertexString(vertexFromQuery));
}
} finally {
service.stop();
titanGraphService.stop();
}
}
}
repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphService.java
......@@ -18,13 +18,9 @@
package org.apache.hadoop.metadata.services;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.TransactionalGraph;
......@@ -50,10 +46,8 @@ public class TitanGraphService implements GraphService {
/**
* Constant for the configuration property that indicates the prefix.
*/
private static final String METADATA_PREFIX = "metadata.titanGraph.";
private static final String METADATA_INDEX_KEY = "index.name";
private static final String METADATA_PREFIX = "metadata.graph.";
private Configuration graphConfig;
private TitanGraph titanGraph;
private Set<String> vertexIndexedKeys;
private Set<String> edgeIndexedKeys;
......@@ -75,11 +69,10 @@ public class TitanGraphService implements GraphService {
*/
@Override
public void start() throws Exception {
// graphConfig = getConfiguration();
Configuration graphConfig = getConfiguration();
titanGraph = initializeGraphDB(graphConfig);
titanGraph = initializeGraphDB();
// createIndicesForVertexKeys();
createIndicesForVertexKeys();
// todo - create Edge Cardinality Constraints
LOG.info("Initialized titanGraph db: {}", titanGraph);
......@@ -90,29 +83,17 @@ public class TitanGraphService implements GraphService {
LOG.info("Init edge property keys: {}", edgeIndexedKeys);
}
protected TitanGraph initializeGraphDB() {
LOG.info("Initializing titanGraph db");
// todo: externalize this
Configuration graphConfig = new PropertiesConfiguration();
graphConfig.setProperty("storage.backend", "berkeleyje");
graphConfig.setProperty("storage.directory", "target/data/graphdb");
return TitanFactory.open(graphConfig);
}
private static Configuration getConfiguration() throws ConfigurationException {
PropertiesConfiguration configProperties = new PropertiesConfiguration("application.properties");
PropertiesConfiguration configProperties =
new PropertiesConfiguration("application.properties");
Configuration graphConfig = new PropertiesConfiguration();
final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) {
String key = iterator.next();
System.out.println("key = " + key);
if (key.startsWith(METADATA_PREFIX)) {
String value = (String) configProperties.getProperty(key);
key = key.substring(METADATA_PREFIX.length());
System.out.println("**** key = " + key + ", value = " + value);
graphConfig.setProperty(key, value);
}
}
......@@ -120,49 +101,19 @@ public class TitanGraphService implements GraphService {
return graphConfig;
}
/**
* This unfortunately requires a handle to Titan implementation since
* com.tinkerpop.blueprints.KeyIndexableGraph#createKeyIndex does not create an index.
*/
protected TitanGraph initializeGraphDB(Configuration graphConfig) {
LOG.info("Initializing titanGraph db");
return TitanFactory.open(graphConfig);
}
protected void createIndicesForVertexKeys() {
if (!((KeyIndexableGraph) titanGraph).getIndexedKeys(Vertex.class).isEmpty()) {
if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for titanGraph");
return;
}
LOG.info("Indexes does not exist, Creating indexes for titanGraph");
// todo - externalize this
String indexName = graphConfig.getString(METADATA_INDEX_KEY);
PropertyKey guid = createPropertyKey("guid", String.class, Cardinality.SINGLE);
createIndex(indexName, guid, Vertex.class, true);
getTitanGraph().commit();
}
private PropertyKey createPropertyKey(String propertyKeyName, Class<String> dataType,
Cardinality cardinality) {
PropertyKey propertyKey = getTitanGraph().getManagementSystem()
.makePropertyKey(propertyKeyName)
.dataType(dataType)
.cardinality(cardinality)
.make();
LOG.info("Created property key {}", propertyKey);
return propertyKey;
}
private void createIndex(String indexName, PropertyKey propertyKey,
Class<? extends Element> clazz, boolean isUnique) {
TitanManagement managementSystem = getTitanGraph().getManagementSystem();
managementSystem.buildPropertyIndex(propertyKey, indexName);
TitanManagement.IndexBuilder indexBuilder = managementSystem
.buildIndex(indexName, clazz)
.addKey(propertyKey);
if (isUnique) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
// todo - add index for vertex and edge property keys
}
/**
......
webapp/src/main/resources/application.properties
......@@ -16,23 +16,21 @@
# limitations under the License.
#
application.services=org.apache.hadoop.metadata.services.TitanGraphService,\
org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
application.services=org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
#metadata.graph.schema.default=none
# Graph implementation
metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
#metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
# Graph Storage
metadata.graph.storage.backend=berkeleyje
metadata.graph.storage.directory=${user.dir}/target/data/graphdb
metadata.graph.storage.directory=target/data/berkeley
# Graph Search Index
#metadata.graph.index.name=search
#metadata.graph.index.search.backend=elasticsearch
#metadata.graph.index.search.directory=${user.dir}/target/data/searchindex
#metadata.graph.index.search.elasticsearch.client-only=false
#metadata.graph.index.search.elasticsearch.local-mode=true
metadata.graph.index.search.backend=elasticsearch
metadata.graph.index.search.directory=target/data/es
metadata.graph.index.search.elasticsearch.client-only=false
metadata.graph.index.search.elasticsearch.local-mode=true
metadata.enableTLS=false
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment