Skip to content

A
atlas
Commit 25ff452b by Venkatesh Seetharam
Options

Add type specific indexes. Contributed by Venkatesh Seetharam

parent 51d00a85
Showing with 138 additions and 130 deletions
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/Constants.java
......@@ -24,10 +24,27 @@ final class Constants {
}
static final String GUID_PROPERTY_KEY = "guid";
static final String GUID_INDEX = "guid_index";
static final String ENTITY_TYPE_PROPERTY_KEY = "type";
static final String ENTITY_TYPE_INDEX = "type_index";
static final String VERSION_PROPERTY_KEY = "version";
static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
/**
* search backing index name.
*/
static final String BACKING_INDEX = "search";
static final String INDEX_NAME = "metadata";
/**
* search backing index name for vertex keys.
*/
static final String VERTEX_INDEX = "vertex_index";
/**
* search backing index name for edge labels.
*/
static final String EDGE_INDEX = "edge_index";
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
......@@ -18,16 +18,13 @@
package org.apache.hadoop.metadata.repository.graph;
import com.google.common.base.Preconditions;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.TransactionalGraph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.commons.collections.iterators.IteratorChain;
import org.apache.hadoop.metadata.IReferenceableInstance;
import org.apache.hadoop.metadata.ITypedInstance;
......@@ -56,7 +53,6 @@ import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
......@@ -131,15 +127,14 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
String typeName) throws RepositoryException {
LOG.info("adding entity={} type={}", typedInstance, typeName);
final TransactionalGraph transactionalGraph = graphService.getTransactionalGraph();
try {
transactionalGraph.rollback();
titanGraph.rollback();
final String guid = instanceToGraphMapper.mapTypedInstanceToGraph(typedInstance);
transactionalGraph.commit(); // commit if there are no errors
titanGraph.commit(); // commit if there are no errors
return guid;
} catch (MetadataException e) {
transactionalGraph.rollback();
titanGraph.rollback();
throw new RepositoryException(e);
}
}
......@@ -148,10 +143,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException {
LOG.info("Retrieving entity with guid={}", guid);
final TransactionalGraph transactionalGraph = graphService.getTransactionalGraph();
try {
transactionalGraph.rollback(); // clean up before starting a query
Vertex instanceVertex = GraphHelper.findVertexByGUID(transactionalGraph, guid);
titanGraph.rollback(); // clean up before starting a query
Vertex instanceVertex = GraphHelper.findVertexByGUID(titanGraph, guid);
if (instanceVertex == null) {
LOG.debug("Could not find a vertex for guid {}", guid);
return null;
......@@ -168,7 +162,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
public List<String> getEntityList(String entityType) throws RepositoryException {
LOG.info("Retrieving entity list for type={}", entityType);
// todo - replace this with index based query
GraphQuery query = graphService.getBlueprintsGraph().query()
.has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
Iterator<Vertex> results = query.vertices().iterator();
......@@ -183,18 +177,35 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
return entityList;
/*
TitanIndexQuery query = titanGraph.indexQuery(Constants.VERTEX_INDEX,
"v." + Constants.ENTITY_TYPE_PROPERTY_KEY + ":(" + entityType + ")");
Iterator<TitanIndexQuery.Result<Vertex>> results = query.vertices().iterator();
if (!results.hasNext()) {
return Collections.emptyList();
}
ArrayList<String> entityList = new ArrayList<>();
while (results.hasNext()) {
Vertex vertex = results.next().getElement();
entityList.add(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY));
}
return entityList;
*/
}
private static void searchWalker (Vertex vtx, final int max, int counter, HashMap<String,JSONObject> e, HashMap<String,JSONObject> v, String edgesToFollow) {
private static void searchWalker (Vertex vtx, final int max, int counter,
HashMap<String,JSONObject> e,
HashMap<String,JSONObject> v, String edgesToFollow) {
counter++;
if (counter <= max) {
Map<String,String> jsonVertexMap = new HashMap<>();
Iterator<Edge> edgeIterator;
Map<String,String> jsonVertexMap = new HashMap<String,String>();
Iterator<Edge> edgeIterator = null;
// If we're doing a lineage traversal, only follow the edges specified by the query. Otherwise
// return them all.
// If we're doing a lineage traversal, only follow the edges specified by the query.
// Otherwise return them all.
if (edgesToFollow != null) {
IteratorChain ic = new IteratorChain();
......@@ -224,7 +235,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Edge edge = edgeIterator.next();
String label = edge.getLabel();
Map<String,String> jsonEdgeMap = new HashMap<String,String>();
Map<String,String> jsonEdgeMap = new HashMap<>();
String tail = edge.getVertex(Direction.OUT).getId().toString();
String head = edge.getVertex(Direction.IN).getId().toString();
......@@ -232,8 +243,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
jsonEdgeMap.put("head", head);
jsonEdgeMap.put("label", label);
Direction d = null;
Direction d;
if (tail.equals(vtx.getId().toString())) {
d = Direction.IN;
} else {
......@@ -249,12 +259,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
e.put(edge.getId().toString(), new JSONObject(jsonEdgeMap));
searchWalker (edge.getVertex(d), max, counter, e, v, edgesToFollow);
}
}
}
}
......@@ -264,16 +270,16 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
* @param prop is the Vertex property to search.
*/
@Override
public Map<String,HashMap<String,JSONObject>> textSearch(String searchText, int depth, String prop) {
public Map<String,HashMap<String,JSONObject>> textSearch(String searchText,
int depth, String prop) {
HashMap<String,HashMap<String,JSONObject>> result = new HashMap<String,HashMap<String,JSONObject>>();
HashMap<String,HashMap<String,JSONObject>> result = new HashMap<>();
// HashMaps, which contain sub JOSN Objects to be relayed back to the parent.
HashMap<String,JSONObject> vertices = new HashMap<String,JSONObject>();
HashMap<String,JSONObject> edges = new HashMap<String,JSONObject>();
HashMap<String,JSONObject> vertices = new HashMap<>();
HashMap<String,JSONObject> edges = new HashMap<>();
/* Later - when we allow search limitation by "type".
/* todo: Later - when we allow search limitation by "type".
ArrayList<String> typesList = new ArrayList<String>();
for (String s: types.split(",")) {
......@@ -307,16 +313,17 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
* @param edgesToFollow is a comma-separated-list of edges to follow.
*/
@Override
public Map<String,HashMap<String,JSONObject>> relationshipWalk(String guid, int depth, String edgesToFollow) {
public Map<String,HashMap<String,JSONObject>> relationshipWalk(String guid, int depth,
String edgesToFollow) {
HashMap<String,HashMap<String,JSONObject>> result = new HashMap<String,HashMap<String,JSONObject>>();
HashMap<String,HashMap<String,JSONObject>> result = new HashMap<>();
// HashMaps, which contain sub JOSN Objects to be relayed back to the parent.
HashMap<String,JSONObject> vertices = new HashMap<String,JSONObject>();
HashMap<String,JSONObject> edges = new HashMap<String,JSONObject>();
HashMap<String,JSONObject> vertices = new HashMap<>();
HashMap<String,JSONObject> edges = new HashMap<>();
// Get the Vertex with the specified GUID.
Vertex v = GraphHelper.findVertexByGUID(graphService.getBlueprintsGraph(), guid);
Vertex v = GraphHelper.findVertexByGUID(titanGraph, guid);
if (v != null) {
searchWalker(v, depth, 0, edges, vertices, edgesToFollow);
......@@ -589,7 +596,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
(ITypedStruct) typedInstance.get(attributeInfo.name),
attributeInfo, idToVertexMap);
// add an edge to the newly created vertex from the parent
GraphHelper.addEdge(instanceVertex, structInstanceVertex, propertyName);
GraphHelper.addEdge(
titanGraph, instanceVertex, structInstanceVertex, propertyName);
break;
case TRAIT:
......@@ -687,8 +695,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Vertex structInstanceVertex = mapStructInstanceToVertex(id,
(ITypedStruct) value, attributeInfo, idToVertexMap);
// add an edge to the newly created vertex from the parent
GraphHelper.addEdge(instanceVertex, structInstanceVertex,
propertyName);
GraphHelper.addEdge(
titanGraph, instanceVertex, structInstanceVertex, propertyName);
break;
case CLASS:
......@@ -717,9 +725,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
if (referenceVertex != null) {
// add an edge to the class vertex from the instance
GraphHelper.addEdge(instanceVertex, referenceVertex, propertyKey);
} else { // Oops - todo - throw an exception?
System.out.println("BOOOOO = " + id);
GraphHelper.addEdge(titanGraph, instanceVertex, referenceVertex, propertyKey);
}
}
}
......@@ -754,7 +760,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
// add an edge to the newly created vertex from the parent
String relationshipLabel = typedInstance.getTypeName() + "." + traitName;
GraphHelper.addEdge(parentInstanceVertex, traitInstanceVertex, relationshipLabel);
GraphHelper.addEdge(
titanGraph, parentInstanceVertex, traitInstanceVertex, relationshipLabel);
}
private void mapPrimitiveToVertex(ITypedInstance typedInstance,
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
......@@ -18,12 +18,14 @@
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.Order;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.repository.SearchIndexer;
......@@ -69,39 +71,26 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
try {
createIndex(management, Constants.GUID_PROPERTY_KEY, String.class, true);
createIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
createIndex(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
.buildMixedIndex(Constants.BACKING_INDEX);
management.buildIndex(Constants.EDGE_INDEX, Edge.class)
.buildMixedIndex(Constants.BACKING_INDEX);
// create a composite index for guid as its unique
createCompositeIndex(management, Constants.GUID_INDEX,
Constants.GUID_PROPERTY_KEY, String.class, true);
// create a composite and mixed index for type since it can be combined with other keys
createCompositeIndex(management, Constants.ENTITY_TYPE_INDEX,
Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false);
createVertexMixedIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
} finally {
management.commit();
}
LOG.info("Index creation for global keys complete.");
// dumpIndexKeys();
}
/*
private void dumpIndexKeys() {
for (TitanGraphIndex index : management.getGraphIndexes(Vertex.class)) {
System.out.println("index.getName() = " + index.getName());
for (PropertyKey key : index.getFieldKeys()) {
System.out.println("key.getName() = " + key.getName());
System.out.println("key = " + key);
}
}
for (TitanGraphIndex index : management.getGraphIndexes(Edge.class)) {
System.out.println("index.getName() = " + index.getName());
for (PropertyKey key : index.getFieldKeys()) {
System.out.println("key.getName() = " + key.getName());
System.out.println("key = " + key);
}
}
}
*/
/**
* This is upon adding a new type to Store.
*
......@@ -113,18 +102,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
public void onAdd(String typeName, IDataType dataType) throws MetadataException {
LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType);
TitanManagement management = titanGraph.getManagementSystem();
try {
TitanManagement management = titanGraph.getManagementSystem();
addIndexForType(management, dataType);
management.commit();
LOG.info("Index creation for type {} complete", typeName);
} catch (Exception e) {
LOG.error("Error creating index for type {}", dataType, e);
// management.rollback();
management.rollback();
}
// dumpIndexKeys();
}
private void addIndexForType(TitanManagement management, IDataType dataType) {
......@@ -170,18 +157,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
final String propertyName = typeName + "." + field.name;
switch (field.dataType().getTypeCategory()) {
case PRIMITIVE:
createIndex(management, propertyName,
getPrimitiveClass(field.dataType()), field.isUnique);
createVertexMixedIndex(
management, propertyName, getPrimitiveClass(field.dataType()));
break;
case ENUM:
createIndex(management, propertyName, Integer.class, field.isUnique);
createVertexMixedIndex(management, propertyName, Integer.class);
break;
case ARRAY:
case MAP:
// index the property holder for element names
createIndex(management, propertyName, String.class, field.isUnique);
createVertexMixedIndex(management, propertyName, String.class);
break;
case STRUCT:
......@@ -195,7 +182,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
case CLASS:
// this is only A reference, index the attribute for edge
createEdgeIndex(management, propertyName, String.class);
createEdgeMixedIndex(management, propertyName);
break;
default:
......@@ -229,39 +216,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
throw new IllegalArgumentException("unknown data type " + dataType);
}
private static PropertyKey createIndex(TitanManagement management,
String propertyName, Class propertyClass) {
return createIndex(management,
propertyName, propertyClass, Cardinality.SINGLE, false, Vertex.class);
}
private static PropertyKey createIndex(TitanManagement management,
String propertyName,
Class propertyClass, boolean isUnique) {
return createIndex(management,
propertyName, propertyClass, Cardinality.SINGLE, isUnique, Vertex.class);
}
private static PropertyKey createEdgeIndex(TitanManagement management,
String propertyName, Class propertyClass) {
return createIndex(management,
propertyName, propertyClass, Cardinality.SINGLE, false, Edge.class);
}
private static PropertyKey createIndex(TitanManagement management,
String propertyName, Class propertyClass,
Cardinality cardinality, boolean isUnique,
Class<? extends Element> elementClass) {
private static PropertyKey createCompositeIndex(TitanManagement management, String indexName,
String propertyName, Class propertyClass,
boolean isUnique) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
.makePropertyKey(propertyName)
.dataType(propertyClass)
.cardinality(cardinality)
.make();
TitanManagement.IndexBuilder indexBuilder = management
.buildIndex("index_" + propertyName, elementClass)
.buildIndex(indexName, Vertex.class)
.addKey(propertyKey);
if (isUnique) {
......@@ -273,4 +239,26 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
return propertyKey;
}
private static PropertyKey createVertexMixedIndex(TitanManagement management,
String propertyName, Class propertyClass) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management
.makePropertyKey(propertyName)
.dataType(propertyClass)
.make();
}
TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
management.addIndexKey(vertexIndex, propertyKey);
return propertyKey;
}
private static void createEdgeMixedIndex(TitanManagement management,
String propertyName) {
EdgeLabel edgeLabel = management.makeEdgeLabel(propertyName).make();
management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
}
}
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
......@@ -19,8 +19,6 @@
package org.apache.hadoop.metadata.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
......@@ -73,34 +71,25 @@ public final class GraphHelper {
return instanceVertex;
}
public static Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex,
String edgeLabel) {
LOG.debug("Adding edge for {} -> struct label {} -> v{}",
fromVertex, edgeLabel, toVertex);
return fromVertex.addEdge(edgeLabel, toVertex);
return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
}
public static Vertex findVertexByGUID(Graph blueprintsGraph,
public static Vertex findVertexByGUID(TitanGraph titanGraph,
String value) {
LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
GraphQuery query = blueprintsGraph.query()
.has(Constants.GUID_PROPERTY_KEY, Compare.EQUAL, value);
GraphQuery query = titanGraph.query()
.has(Constants.GUID_PROPERTY_KEY, value);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since guid should be unique
return results.hasNext() ? results.next() : null;
}
public static Vertex findVertexByGUIDUsingIndex(TitanGraph titanGraph,
String value) {
LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
TitanIndexQuery query = titanGraph.indexQuery("index_" + Constants.GUID_PROPERTY_KEY,
Constants.GUID_PROPERTY_KEY + " = " + value);
Iterator<TitanIndexQuery.Result<Vertex>> results = query.vertices().iterator();
// returning one since guid should be unique
return results.hasNext() ? results.next().getElement() : null;
}
public static String vertexString(final Vertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
......@@ -120,6 +109,7 @@ public final class GraphHelper {
+ "]";
}
/*
public static void dumpToLog(final Graph graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph);
......@@ -133,4 +123,5 @@ public final class GraphHelper {
}
LOG.debug("*******************Graph Dump****************************");
}
*/
}
\ No newline at end of file
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -59,7 +59,7 @@ public class GraphBackedMetadataRepositoryTest {
private TitanGraphService titanGraphService;
@Inject
private GraphBackedMetadataRepository repositoryService;
private TypeSystem ts;
private String guid;
......@@ -70,12 +70,14 @@ public class GraphBackedMetadataRepositoryTest {
// start the injected repository service
repositoryService.start();
new GraphBackedSearchIndexer(titanGraphService);
ts = TypeSystem.getInstance();
defineDeptEmployeeTypes(ts);
}
@Test(enabled = false)
@Test
public void testSubmitEntity() throws Exception {
Referenceable hrDept = createDeptEg1(ts);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
......@@ -97,7 +99,7 @@ public class GraphBackedMetadataRepositoryTest {
}
}
@Test(dependsOnMethods = "testSubmitEntity", enabled = false)
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetEntityDefinition() throws Exception {
ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid);
Assert.assertNotNull(entity);
......@@ -112,11 +114,12 @@ public class GraphBackedMetadataRepositoryTest {
@Test(enabled = false)
public void testGetEntityList() throws Exception {
List<String> entityList = repositoryService.getEntityList(ENTITY_TYPE);
System.out.println("entityList = " + entityList);
Assert.assertNotNull(entityList);
Assert.assertEquals(entityList.size(), 1); // one department
}
@Test(enabled = false)
@Test
public void testRawSearch1() throws Exception {
Referenceable hrDept = createDeptEg1(ts);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
......
repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperTest.java
......@@ -73,6 +73,8 @@ public class GraphRepoMapperTest {
// start the injected repository service
repositoryService.start();
new GraphBackedSearchIndexer(titanGraphService);
typeSystem = TypeSystem.getInstance();
createHiveTypes();
......@@ -109,7 +111,6 @@ public class GraphRepoMapperTest {
public void testGetEntityDefinition() throws Exception {
TitanGraph graph = titanGraphService.getTitanGraph();
GraphQuery query = graph.query()
.has("type", Compare.EQUAL, TABLE_TYPE)
.has("type", Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
// returning one since guid should be unique
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment