Skip to content

A
atlas
Commit 25ff452b by Venkatesh Seetharam
Options

Add type specific indexes. Contributed by Venkatesh Seetharam

parent 51d00a85
Showing with 138 additions and 130 deletions
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java
......@@ -24,10 +24,27 @@ final class Constants {
}
static final String GUID_PROPERTY_KEY = "guid";
static final String GUID_INDEX = "guid_index";
static final String ENTITY_TYPE_PROPERTY_KEY = "type";
static final String ENTITY_TYPE_INDEX = "type_index";
static final String VERSION_PROPERTY_KEY = "version";
static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
/**
* search backing index name.
*/
static final String BACKING_INDEX = "search";
static final String INDEX_NAME = "metadata";
/**
* search backing index name for vertex keys.
*/
static final String VERTEX_INDEX = "vertex_index";
/**
* search backing index name for edge labels.
*/
static final String EDGE_INDEX = "edge_index";
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
......@@ -18,12 +18,14 @@
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.Order;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.repository.SearchIndexer;
......@@ -69,39 +71,26 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
try {
createIndex(management, Constants.GUID_PROPERTY_KEY, String.class, true);
createIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
createIndex(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
.buildMixedIndex(Constants.BACKING_INDEX);
management.buildIndex(Constants.EDGE_INDEX, Edge.class)
.buildMixedIndex(Constants.BACKING_INDEX);
// create a composite index for guid as its unique
createCompositeIndex(management, Constants.GUID_INDEX,
Constants.GUID_PROPERTY_KEY, String.class, true);
// create a composite and mixed index for type since it can be combined with other keys
createCompositeIndex(management, Constants.ENTITY_TYPE_INDEX,
Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false);
createVertexMixedIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
} finally {
management.commit();
}
LOG.info("Index creation for global keys complete.");
// dumpIndexKeys();
}
/*
private void dumpIndexKeys() {
for (TitanGraphIndex index : management.getGraphIndexes(Vertex.class)) {
System.out.println("index.getName() = " + index.getName());
for (PropertyKey key : index.getFieldKeys()) {
System.out.println("key.getName() = " + key.getName());
System.out.println("key = " + key);
}
}
for (TitanGraphIndex index : management.getGraphIndexes(Edge.class)) {
System.out.println("index.getName() = " + index.getName());
for (PropertyKey key : index.getFieldKeys()) {
System.out.println("key.getName() = " + key.getName());
System.out.println("key = " + key);
}
}
}
*/
/**
* This is upon adding a new type to Store.
*
......@@ -113,18 +102,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
public void onAdd(String typeName, IDataType dataType) throws MetadataException {
LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType);
TitanManagement management = titanGraph.getManagementSystem();
try {
TitanManagement management = titanGraph.getManagementSystem();
addIndexForType(management, dataType);
management.commit();
LOG.info("Index creation for type {} complete", typeName);
} catch (Exception e) {
LOG.error("Error creating index for type {}", dataType, e);
// management.rollback();
management.rollback();
}
// dumpIndexKeys();
}
private void addIndexForType(TitanManagement management, IDataType dataType) {
......@@ -170,18 +157,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
final String propertyName = typeName + "." + field.name;
switch (field.dataType().getTypeCategory()) {
case PRIMITIVE:
createIndex(management, propertyName,
getPrimitiveClass(field.dataType()), field.isUnique);
createVertexMixedIndex(
management, propertyName, getPrimitiveClass(field.dataType()));
break;
case ENUM:
createIndex(management, propertyName, Integer.class, field.isUnique);
createVertexMixedIndex(management, propertyName, Integer.class);
break;
case ARRAY:
case MAP:
// index the property holder for element names
createIndex(management, propertyName, String.class, field.isUnique);
createVertexMixedIndex(management, propertyName, String.class);
break;
case STRUCT:
......@@ -195,7 +182,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
case CLASS:
// this is only A reference, index the attribute for edge
createEdgeIndex(management, propertyName, String.class);
createEdgeMixedIndex(management, propertyName);
break;
default:
......@@ -229,39 +216,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
throw new IllegalArgumentException("unknown data type " + dataType);
}
private static PropertyKey createIndex(TitanManagement management,
String propertyName, Class propertyClass) {
return createIndex(management,
propertyName, propertyClass, Cardinality.SINGLE, false, Vertex.class);
}
private static PropertyKey createIndex(TitanManagement management,
String propertyName,
Class propertyClass, boolean isUnique) {
return createIndex(management,
propertyName, propertyClass, Cardinality.SINGLE, isUnique, Vertex.class);
}
private static PropertyKey createEdgeIndex(TitanManagement management,
String propertyName, Class propertyClass) {
return createIndex(management,
propertyName, propertyClass, Cardinality.SINGLE, false, Edge.class);
}
private static PropertyKey createIndex(TitanManagement management,
String propertyName, Class propertyClass,
Cardinality cardinality, boolean isUnique,
Class<? extends Element> elementClass) {
private static PropertyKey createCompositeIndex(TitanManagement management, String indexName,
String propertyName, Class propertyClass,
boolean isUnique) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
.makePropertyKey(propertyName)
.dataType(propertyClass)
.cardinality(cardinality)
.make();
TitanManagement.IndexBuilder indexBuilder = management
.buildIndex("index_" + propertyName, elementClass)
.buildIndex(indexName, Vertex.class)
.addKey(propertyKey);
if (isUnique) {
......@@ -273,4 +239,26 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
return propertyKey;
}
private static PropertyKey createVertexMixedIndex(TitanManagement management,
String propertyName, Class propertyClass) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
.makePropertyKey(propertyName)
.dataType(propertyClass)
.make();
}
TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
management.addIndexKey(vertexIndex, propertyKey);
return propertyKey;
}
private static void createEdgeMixedIndex(TitanManagement management,
String propertyName) {
EdgeLabel edgeLabel = management.makeEdgeLabel(propertyName).make();
management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
......@@ -19,8 +19,6 @@
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
......@@ -73,34 +71,25 @@ public final class GraphHelper {
return instanceVertex;
}
public static Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex,
String edgeLabel) {
LOG.debug("Adding edge for {} -> struct label {} -> v{}",
fromVertex, edgeLabel, toVertex);
return fromVertex.addEdge(edgeLabel, toVertex);
return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
}
public static Vertex findVertexByGUID(Graph blueprintsGraph,
public static Vertex findVertexByGUID(TitanGraph titanGraph,
String value) {
LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
GraphQuery query = blueprintsGraph.query()
.has(Constants.GUID_PROPERTY_KEY, Compare.EQUAL, value);
GraphQuery query = titanGraph.query()
.has(Constants.GUID_PROPERTY_KEY, value);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since guid should be unique
return results.hasNext() ? results.next() : null;
}
public static Vertex findVertexByGUIDUsingIndex(TitanGraph titanGraph,
String value) {
LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
TitanIndexQuery query = titanGraph.indexQuery("index_" + Constants.GUID_PROPERTY_KEY,
Constants.GUID_PROPERTY_KEY + " = " + value);
Iterator<TitanIndexQuery.Result<Vertex>> results = query.vertices().iterator();
// returning one since guid should be unique
return results.hasNext() ? results.next().getElement() : null;
}
public static String vertexString(final Vertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
......@@ -120,6 +109,7 @@ public final class GraphHelper {
+ "]";
}
/*
public static void dumpToLog(final Graph graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph);
......@@ -133,4 +123,5 @@ public final class GraphHelper {
}
LOG.debug("*******************Graph Dump****************************");
}
*/
}
\ No newline at end of file
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -59,7 +59,7 @@ public class GraphBackedMetadataRepositoryTest {
private TitanGraphService titanGraphService;
@Inject
private GraphBackedMetadataRepository repositoryService;
private TypeSystem ts;
private String guid;
......@@ -70,12 +70,14 @@ public class GraphBackedMetadataRepositoryTest {
// start the injected repository service
repositoryService.start();
new GraphBackedSearchIndexer(titanGraphService);
ts = TypeSystem.getInstance();
defineDeptEmployeeTypes(ts);
}
@Test(enabled = false)
@Test
public void testSubmitEntity() throws Exception {
Referenceable hrDept = createDeptEg1(ts);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
......@@ -97,7 +99,7 @@ public class GraphBackedMetadataRepositoryTest {
}
}
@Test(dependsOnMethods = "testSubmitEntity", enabled = false)
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetEntityDefinition() throws Exception {
ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid);
Assert.assertNotNull(entity);
......@@ -112,11 +114,12 @@ public class GraphBackedMetadataRepositoryTest {
@Test(enabled = false)
public void testGetEntityList() throws Exception {
List<String> entityList = repositoryService.getEntityList(ENTITY_TYPE);
System.out.println("entityList = " + entityList);
Assert.assertNotNull(entityList);
Assert.assertEquals(entityList.size(), 1); // one department
}
@Test(enabled = false)
@Test
public void testRawSearch1() throws Exception {
Referenceable hrDept = createDeptEg1(ts);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
......
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperTest.java
......@@ -73,6 +73,8 @@ public class GraphRepoMapperTest {
// start the injected repository service
repositoryService.start();
new GraphBackedSearchIndexer(titanGraphService);
typeSystem = TypeSystem.getInstance();
createHiveTypes();
......@@ -109,7 +111,6 @@ public class GraphRepoMapperTest {
public void testGetEntityDefinition() throws Exception {
TitanGraph graph = titanGraphService.getTitanGraph();
GraphQuery query = graph.query()
.has("type", Compare.EQUAL, TABLE_TYPE)
.has("type", Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since guid should be unique
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment