ISSUE-3 Externalize graph configuration. Contributed by Venkatesh Seetharam

common/src/main/java/org/apache/hadoop/metadata/service/ServiceInitializer.java

@@ -36,15 +36,13 @@ public class ServiceInitializer {
     public void initialize() throws MetadataException {
         String serviceClassNames;
         try {
-            PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties");
+            PropertiesConfiguration configuration =
+                    new PropertiesConfiguration("application.properties");
             serviceClassNames = configuration.getString("application.services");
         } catch (ConfigurationException e) {
-            throw new MetadataException("unable to get server properties");
+            throw new RuntimeException("unable to get server properties");
         }
 
-        serviceClassNames
-                = "org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService";
-
         for (String serviceClassName : serviceClassNames.split(",")) {
             serviceClassName = serviceClassName.trim();
             if (serviceClassName.isEmpty()) {

common/src/main/java/org/apache/hadoop/metadata/service/Services.java

@@ -59,7 +59,8 @@ public final class Services implements Iterable<Service> {
         if (services.containsKey(serviceName)) {
             return (T) services.get(serviceName);
         } else {
-            throw new NoSuchElementException("Service " + serviceName + " not registered with registry");
+            throw new NoSuchElementException(
+                    "Service " + serviceName + " not registered with registry");
         }
     }
 
@@ -79,7 +80,8 @@ public final class Services implements Iterable<Service> {
 
         String serviceClassName;
         try {
-            PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties");
+            PropertiesConfiguration configuration =
+                    new PropertiesConfiguration("application.properties");
             serviceClassName = configuration.getString(serviceName + ".impl");
         } catch (ConfigurationException e) {
             throw new MetadataException("unable to get server properties");

repository/src/main/java/org/apache/hadoop/metadata/services/GraphBackedMetadataRepositoryService.java

@@ -20,7 +20,6 @@ package org.apache.hadoop.metadata.services;
 
 import com.google.common.base.Preconditions;
 import com.tinkerpop.blueprints.Graph;
-import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.TransactionalGraph;
 import com.tinkerpop.blueprints.Vertex;
 import org.apache.hadoop.metadata.service.Services;
@@ -31,7 +30,6 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -102,6 +100,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
 
     @Override
     public String submitEntity(String entity, String entityType) {
+        LOG.info("adding entity={} type={}", entity, entityType);
         Map<String, String> properties = (Map<String, String>) JSONValue.parse(entity);
 
         final String entityName = properties.get("entityName");
@@ -132,6 +131,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
 
     @Override
     public String getEntityDefinition(String entityName, String entityType) {
+        LOG.info("Retrieving entity name={} type={}", entityName, entityType);
         Vertex entityVertex = GraphUtils.findVertex(getBlueprintsGraph(), entityName, entityType);
         if (entityVertex == null) {
             return null;
@@ -143,52 +143,7 @@ public class GraphBackedMetadataRepositoryService implements MetadataRepositoryS
 
     @Override
     public List<String> getEntityList(String entityType) {
+        LOG.info("Retrieving entity list for type={}", entityType);
         return Collections.emptyList();
     }
-
-    public static void main(String[] args) throws Exception {
-        TitanGraphService titanGraphService = new TitanGraphService();
-        titanGraphService.start();
-        Services.get().register(titanGraphService);
-
-        GraphBackedMetadataRepositoryService service = new GraphBackedMetadataRepositoryService();
-        try {
-            service.start();
-            String guid = UUID.randomUUID().toString();
-
-            final TransactionalGraph graph = service.getTransactionalGraph();
-            System.out.println("graph = " + graph);
-            System.out.println("graph.getVertices() = " + graph.getVertices());
-
-
-            Vertex entityVertex = null;
-            try {
-                graph.rollback();
-                entityVertex = graph.addVertex(null);
-                entityVertex.setProperty("guid", guid);
-                entityVertex.setProperty("entityName", "entityName");
-                entityVertex.setProperty("entityType", "entityType");
-            } catch (Exception e) {
-                graph.rollback();
-                e.printStackTrace();
-            } finally {
-                graph.commit();
-            }
-
-            System.out.println("vertex = " + GraphUtils.vertexString(entityVertex));
-
-            GraphQuery query = graph.query()
-                    .has("entityName", "entityName")
-                    .has("entityType", "entityType");
-
-            Iterator<Vertex> results = query.vertices().iterator();
-            if (results.hasNext()) {
-                Vertex vertexFromQuery = results.next();
-                System.out.println("vertex = " + GraphUtils.vertexString(vertexFromQuery));
-            }
-        } finally {
-            service.stop();
-            titanGraphService.stop();
-        }
-    }
 }

repository/src/main/java/org/apache/hadoop/metadata/services/TitanGraphService.java

@@ -18,13 +18,9 @@
 
 package org.apache.hadoop.metadata.services;
 
-import com.thinkaurelius.titan.core.Cardinality;
-import com.thinkaurelius.titan.core.PropertyKey;
 import com.thinkaurelius.titan.core.TitanFactory;
 import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.schema.TitanManagement;
 import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Element;
 import com.tinkerpop.blueprints.Graph;
 import com.tinkerpop.blueprints.KeyIndexableGraph;
 import com.tinkerpop.blueprints.TransactionalGraph;
@@ -50,10 +46,8 @@ public class TitanGraphService implements GraphService {
     /**
      * Constant for the configuration property that indicates the prefix.
      */
-    private static final String METADATA_PREFIX = "metadata.titanGraph.";
-    private static final String METADATA_INDEX_KEY = "index.name";
+    private static final String METADATA_PREFIX = "metadata.graph.";
 
-    private Configuration graphConfig;
     private TitanGraph titanGraph;
     private Set<String> vertexIndexedKeys;
     private Set<String> edgeIndexedKeys;
@@ -75,11 +69,10 @@ public class TitanGraphService implements GraphService {
      */
     @Override
     public void start() throws Exception {
-//        graphConfig = getConfiguration();
+        Configuration graphConfig = getConfiguration();
+        titanGraph = initializeGraphDB(graphConfig);
 
-        titanGraph = initializeGraphDB();
-
-//        createIndicesForVertexKeys();
+        createIndicesForVertexKeys();
         // todo - create Edge Cardinality Constraints
         LOG.info("Initialized titanGraph db: {}", titanGraph);
 
@@ -90,29 +83,17 @@ public class TitanGraphService implements GraphService {
         LOG.info("Init edge property keys: {}", edgeIndexedKeys);
     }
 
-    protected TitanGraph initializeGraphDB() {
-        LOG.info("Initializing titanGraph db");
-
-        // todo: externalize this
-        Configuration graphConfig = new PropertiesConfiguration();
-        graphConfig.setProperty("storage.backend", "berkeleyje");
-        graphConfig.setProperty("storage.directory", "target/data/graphdb");
-
-        return TitanFactory.open(graphConfig);
-    }
-
     private static Configuration getConfiguration() throws ConfigurationException {
-        PropertiesConfiguration configProperties = new PropertiesConfiguration("application.properties");
+        PropertiesConfiguration configProperties =
+                new PropertiesConfiguration("application.properties");
 
         Configuration graphConfig = new PropertiesConfiguration();
         final Iterator<String> iterator = configProperties.getKeys();
         while (iterator.hasNext()) {
             String key = iterator.next();
-            System.out.println("key = " + key);
             if (key.startsWith(METADATA_PREFIX)) {
                 String value = (String) configProperties.getProperty(key);
                 key = key.substring(METADATA_PREFIX.length());
-                System.out.println("**** key = " + key + ", value = " + value);
                 graphConfig.setProperty(key, value);
             }
         }
@@ -120,49 +101,19 @@ public class TitanGraphService implements GraphService {
         return graphConfig;
     }
 
-    /**
-     * This unfortunately requires a handle to Titan implementation since
-     * com.tinkerpop.blueprints.KeyIndexableGraph#createKeyIndex does not create an index.
-     */
+    protected TitanGraph initializeGraphDB(Configuration graphConfig) {
+        LOG.info("Initializing titanGraph db");
+        return TitanFactory.open(graphConfig);
+    }
+
     protected void createIndicesForVertexKeys() {
-        if (!((KeyIndexableGraph) titanGraph).getIndexedKeys(Vertex.class).isEmpty()) {
+        if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
             LOG.info("Indexes already exist for titanGraph");
             return;
         }
 
         LOG.info("Indexes does not exist, Creating indexes for titanGraph");
-        // todo - externalize this
-        String indexName = graphConfig.getString(METADATA_INDEX_KEY);
-        PropertyKey guid = createPropertyKey("guid", String.class, Cardinality.SINGLE);
-        createIndex(indexName, guid, Vertex.class, true);
-
-        getTitanGraph().commit();
-    }
-
-    private PropertyKey createPropertyKey(String propertyKeyName, Class<String> dataType,
-                                          Cardinality cardinality) {
-        PropertyKey propertyKey = getTitanGraph().getManagementSystem()
-                .makePropertyKey(propertyKeyName)
-                .dataType(dataType)
-                .cardinality(cardinality)
-                .make();
-        LOG.info("Created property key {}", propertyKey);
-        return propertyKey;
-    }
-
-    private void createIndex(String indexName, PropertyKey propertyKey,
-                             Class<? extends Element> clazz, boolean isUnique) {
-        TitanManagement managementSystem = getTitanGraph().getManagementSystem();
-        managementSystem.buildPropertyIndex(propertyKey, indexName);
-
-        TitanManagement.IndexBuilder indexBuilder = managementSystem
-                .buildIndex(indexName, clazz)
-                .addKey(propertyKey);
-        if (isUnique) {
-            indexBuilder.unique();
-        }
-
-        indexBuilder.buildCompositeIndex();
+        // todo - add index for vertex and edge property keys
     }
 
     /**

webapp/src/main/resources/application.properties

@@ -16,23 +16,21 @@
 # limitations under the License.
 #
 
+application.services=org.apache.hadoop.metadata.services.TitanGraphService,\
+                     org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
 
-application.services=org.apache.hadoop.metadata.services.TitanGraphService,org.apache.hadoop.metadata.services.GraphBackedMetadataRepositoryService
-
-#metadata.graph.schema.default=none
 
 # Graph implementation
-metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
+#metadata.graph.blueprints.graph=com.thinkaurelius.titan.core.TitanFactory
 
 # Graph Storage
 metadata.graph.storage.backend=berkeleyje
-metadata.graph.storage.directory=${user.dir}/target/data/graphdb
+metadata.graph.storage.directory=target/data/berkeley
 
 # Graph Search Index
-#metadata.graph.index.name=search
-#metadata.graph.index.search.backend=elasticsearch
-#metadata.graph.index.search.directory=${user.dir}/target/data/searchindex
-#metadata.graph.index.search.elasticsearch.client-only=false
-#metadata.graph.index.search.elasticsearch.local-mode=true
+metadata.graph.index.search.backend=elasticsearch
+metadata.graph.index.search.directory=target/data/es
+metadata.graph.index.search.elasticsearch.client-only=false
+metadata.graph.index.search.elasticsearch.local-mode=true
 
 metadata.enableTLS=false