Add type specific indexes. Contributed by Venkatesh Seetharam

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java

@@ -24,10 +24,27 @@ final class Constants {
     }
 
     static final String GUID_PROPERTY_KEY = "guid";
+    static final String GUID_INDEX = "guid_index";
+
     static final String ENTITY_TYPE_PROPERTY_KEY = "type";
+    static final String ENTITY_TYPE_INDEX = "type_index";
+
     static final String VERSION_PROPERTY_KEY = "version";
     static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
 
+    /**
+     * search backing index name.
+     */
     static final String BACKING_INDEX = "search";
     static final String INDEX_NAME = "metadata";
+
+    /**
+     * search backing index name for vertex keys.
+     */
+    static final String VERTEX_INDEX = "vertex_index";
+
+    /**
+     * search backing index name for edge labels.
+     */
+    static final String EDGE_INDEX = "edge_index";
 }

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java

@@ -18,12 +18,14 @@
 
 package org.apache.hadoop.metadata.repository.graph;
 
-import com.thinkaurelius.titan.core.Cardinality;
+import com.thinkaurelius.titan.core.EdgeLabel;
+import com.thinkaurelius.titan.core.Order;
 import com.thinkaurelius.titan.core.PropertyKey;
 import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
 import com.thinkaurelius.titan.core.schema.TitanManagement;
+import com.tinkerpop.blueprints.Direction;
 import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Element;
 import com.tinkerpop.blueprints.Vertex;
 import org.apache.hadoop.metadata.MetadataException;
 import org.apache.hadoop.metadata.repository.SearchIndexer;
@@ -69,39 +71,26 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
 
         LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
         try {
-            createIndex(management, Constants.GUID_PROPERTY_KEY, String.class, true);
-            createIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
-            createIndex(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
+            management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
+                    .buildMixedIndex(Constants.BACKING_INDEX);
+            management.buildIndex(Constants.EDGE_INDEX, Edge.class)
+                    .buildMixedIndex(Constants.BACKING_INDEX);
+
+            // create a composite index for guid as its unique
+            createCompositeIndex(management, Constants.GUID_INDEX,
+                    Constants.GUID_PROPERTY_KEY, String.class, true);
+
+            // create a composite and mixed index for type since it can be combined with other keys
+            createCompositeIndex(management, Constants.ENTITY_TYPE_INDEX,
+                    Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false);
+            createVertexMixedIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
         } finally {
             management.commit();
         }
 
         LOG.info("Index creation for global keys complete.");
-        // dumpIndexKeys();
     }
 
-/*
-    private void dumpIndexKeys() {
-        for (TitanGraphIndex index : management.getGraphIndexes(Vertex.class)) {
-            System.out.println("index.getName() = " + index.getName());
-
-            for (PropertyKey key : index.getFieldKeys()) {
-                System.out.println("key.getName() = " + key.getName());
-                System.out.println("key = " + key);
-            }
-        }
-
-        for (TitanGraphIndex index : management.getGraphIndexes(Edge.class)) {
-            System.out.println("index.getName() = " + index.getName());
-
-            for (PropertyKey key : index.getFieldKeys()) {
-                System.out.println("key.getName() = " + key.getName());
-                System.out.println("key = " + key);
-            }
-        }
-    }
-*/
-
     /**
      * This is upon adding a new type to Store.
      *
@@ -113,18 +102,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
     public void onAdd(String typeName, IDataType dataType) throws MetadataException {
         LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType);
 
-        try {
         TitanManagement management = titanGraph.getManagementSystem();
+        try {
             addIndexForType(management, dataType);
             management.commit();
             LOG.info("Index creation for type {} complete", typeName);
 
         } catch (Exception e) {
             LOG.error("Error creating index for type {}", dataType, e);
-            // management.rollback();
+            management.rollback();
         }
-
-        // dumpIndexKeys();
     }
 
     private void addIndexForType(TitanManagement management, IDataType dataType) {
@@ -170,18 +157,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
         final String propertyName = typeName + "." + field.name;
         switch (field.dataType().getTypeCategory()) {
             case PRIMITIVE:
-                createIndex(management, propertyName,
-                        getPrimitiveClass(field.dataType()), field.isUnique);
+                createVertexMixedIndex(
+                        management, propertyName, getPrimitiveClass(field.dataType()));
                 break;
 
             case ENUM:
-                createIndex(management, propertyName, Integer.class, field.isUnique);
+                createVertexMixedIndex(management, propertyName, Integer.class);
                 break;
 
             case ARRAY:
             case MAP:
                 // index the property holder for element names
-                createIndex(management, propertyName, String.class, field.isUnique);
+                createVertexMixedIndex(management, propertyName, String.class);
                 break;
 
             case STRUCT:
@@ -195,7 +182,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
 
             case CLASS:
                 // this is only A reference, index the attribute for edge
-                createEdgeIndex(management, propertyName, String.class);
+                createEdgeMixedIndex(management, propertyName);
                 break;
 
             default:
@@ -229,39 +216,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
         throw new IllegalArgumentException("unknown data type " + dataType);
     }
 
-    private static PropertyKey createIndex(TitanManagement management,
-                                           String propertyName, Class propertyClass) {
-        return createIndex(management,
-                propertyName, propertyClass, Cardinality.SINGLE, false, Vertex.class);
-    }
-
-    private static PropertyKey createIndex(TitanManagement management,
-                                           String propertyName,
-                                           Class propertyClass, boolean isUnique) {
-        return createIndex(management,
-                propertyName, propertyClass, Cardinality.SINGLE, isUnique, Vertex.class);
-    }
-
-    private static PropertyKey createEdgeIndex(TitanManagement management,
-                                               String propertyName, Class propertyClass) {
-        return createIndex(management,
-                propertyName, propertyClass, Cardinality.SINGLE, false, Edge.class);
-    }
-
-    private static PropertyKey createIndex(TitanManagement management,
+    private static PropertyKey createCompositeIndex(TitanManagement management, String indexName,
                                                     String propertyName, Class propertyClass,
-                                           Cardinality cardinality, boolean isUnique,
-                                           Class<? extends Element> elementClass) {
+                                                    boolean isUnique) {
         PropertyKey propertyKey = management.getPropertyKey(propertyName);
         if (propertyKey == null) {
             propertyKey = management
                     .makePropertyKey(propertyName)
                     .dataType(propertyClass)
-                    .cardinality(cardinality)
                     .make();
 
             TitanManagement.IndexBuilder indexBuilder = management
-                    .buildIndex("index_" + propertyName, elementClass)
+                    .buildIndex(indexName, Vertex.class)
                     .addKey(propertyKey);
 
             if (isUnique) {
@@ -273,4 +239,26 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
 
         return propertyKey;
     }
+
+    private static PropertyKey createVertexMixedIndex(TitanManagement management,
+                                                      String propertyName, Class propertyClass) {
+        PropertyKey propertyKey = management.getPropertyKey(propertyName);
+        if (propertyKey == null) {
+            propertyKey = management
+                    .makePropertyKey(propertyName)
+                    .dataType(propertyClass)
+                    .make();
+        }
+
+        TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
+        management.addIndexKey(vertexIndex, propertyKey);
+
+        return propertyKey;
+    }
+
+    private static void createEdgeMixedIndex(TitanManagement management,
+                                             String propertyName) {
+        EdgeLabel edgeLabel = management.makeEdgeLabel(propertyName).make();
+        management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
+    }
 }

repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java

@@ -19,8 +19,6 @@
 package org.apache.hadoop.metadata.repository.graph;
 
 import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanIndexQuery;
-import com.tinkerpop.blueprints.Compare;
 import com.tinkerpop.blueprints.Direction;
 import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.Graph;
@@ -73,34 +71,25 @@ public final class GraphHelper {
         return instanceVertex;
     }
 
-    public static Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
+    public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex,
+                               String edgeLabel) {
         LOG.debug("Adding edge for {} -> struct label {} -> v{}",
                 fromVertex, edgeLabel, toVertex);
-        return fromVertex.addEdge(edgeLabel, toVertex);
+
+        return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
     }
 
-    public static Vertex findVertexByGUID(Graph blueprintsGraph,
+    public static Vertex findVertexByGUID(TitanGraph titanGraph,
                                           String value) {
         LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
 
-        GraphQuery query = blueprintsGraph.query()
-                .has(Constants.GUID_PROPERTY_KEY, Compare.EQUAL, value);
+        GraphQuery query = titanGraph.query()
+                .has(Constants.GUID_PROPERTY_KEY, value);
         Iterator<Vertex> results = query.vertices().iterator();
         // returning one since guid should be unique
         return results.hasNext() ? results.next() : null;
     }
 
-    public static Vertex findVertexByGUIDUsingIndex(TitanGraph titanGraph,
-                                                    String value) {
-        LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
-
-        TitanIndexQuery query = titanGraph.indexQuery("index_" + Constants.GUID_PROPERTY_KEY,
-                Constants.GUID_PROPERTY_KEY + " = " +  value);
-        Iterator<TitanIndexQuery.Result<Vertex>> results = query.vertices().iterator();
-        // returning one since guid should be unique
-        return results.hasNext() ? results.next().getElement() : null;
-    }
-
     public static String vertexString(final Vertex vertex) {
         StringBuilder properties = new StringBuilder();
         for (String propertyKey : vertex.getPropertyKeys()) {
@@ -120,6 +109,7 @@ public final class GraphHelper {
                 + "]";
     }
 
+/*
     public static void dumpToLog(final Graph graph) {
         LOG.debug("*******************Graph Dump****************************");
         LOG.debug("Vertices of {}", graph);
@@ -133,4 +123,5 @@ public final class GraphHelper {
         }
         LOG.debug("*******************Graph Dump****************************");
     }
+*/
 }
\ No newline at end of file

repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java

@@ -70,12 +70,14 @@ public class GraphBackedMetadataRepositoryTest {
         // start the injected repository service
         repositoryService.start();
 
+        new GraphBackedSearchIndexer(titanGraphService);
+
         ts = TypeSystem.getInstance();
 
         defineDeptEmployeeTypes(ts);
     }
 
-    @Test(enabled = false)
+    @Test
     public void testSubmitEntity() throws Exception {
         Referenceable hrDept = createDeptEg1(ts);
         ClassType deptType = ts.getDataType(ClassType.class, "Department");
@@ -97,7 +99,7 @@ public class GraphBackedMetadataRepositoryTest {
         }
     }
 
-    @Test(dependsOnMethods = "testSubmitEntity", enabled = false)
+    @Test(dependsOnMethods = "testSubmitEntity")
     public void testGetEntityDefinition() throws Exception {
         ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid);
         Assert.assertNotNull(entity);
@@ -112,11 +114,12 @@ public class GraphBackedMetadataRepositoryTest {
     @Test(enabled = false)
     public void testGetEntityList() throws Exception {
         List<String> entityList = repositoryService.getEntityList(ENTITY_TYPE);
+        System.out.println("entityList = " + entityList);
         Assert.assertNotNull(entityList);
         Assert.assertEquals(entityList.size(), 1); // one department
     }
 
-    @Test(enabled = false)
+    @Test
     public void testRawSearch1() throws Exception {
         Referenceable hrDept = createDeptEg1(ts);
         ClassType deptType = ts.getDataType(ClassType.class, "Department");

repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperTest.java

@@ -73,6 +73,8 @@ public class GraphRepoMapperTest {
         // start the injected repository service
         repositoryService.start();
 
+        new GraphBackedSearchIndexer(titanGraphService);
+
         typeSystem = TypeSystem.getInstance();
 
         createHiveTypes();
@@ -109,7 +111,6 @@ public class GraphRepoMapperTest {
     public void testGetEntityDefinition() throws Exception {
         TitanGraph graph = titanGraphService.getTitanGraph();
         GraphQuery query = graph.query()
-                .has("type", Compare.EQUAL, TABLE_TYPE)
                 .has("type", Compare.EQUAL, TABLE_TYPE);
         Iterator<Vertex> results = query.vertices().iterator();
         // returning one since guid should be unique