Skip to content

A
atlas
Commit f0f22765 by Venkatesh Seetharam
Options

Add tests for search indexing. Contributed by Venkatesh Seetharam

parent 4301264e
Showing with 58 additions and 67 deletions
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
...@@ -156,7 +156,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -156,7 +156,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override @Override
public List<String> getEntityList(String entityType) throws RepositoryException { public List<String> getEntityList(String entityType) throws RepositoryException {
LOG.info("Retrieving entity list for type={}", entityType); LOG.info("Retrieving entity list for type={}", entityType);
// todo - replace this with index based query
GraphQuery query = graphService.getBlueprintsGraph().query() GraphQuery query = graphService.getBlueprintsGraph().query()
.has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType); .has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
Iterator<Vertex> results = query.vertices().iterator(); Iterator<Vertex> results = query.vertices().iterator();
...@@ -171,23 +170,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository { ...@@ -171,23 +170,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
} }
return entityList; return entityList;
/*
TitanIndexQuery query = titanGraph.indexQuery(Constants.VERTEX_INDEX,
"v." + Constants.ENTITY_TYPE_PROPERTY_KEY + ":(" + entityType + ")");
Iterator<TitanIndexQuery.Result<Vertex>> results = query.vertices().iterator();
if (!results.hasNext()) {
return Collections.emptyList();
}
ArrayList<String> entityList = new ArrayList<>();
while (results.hasNext()) {
Vertex vertex = results.next().getElement();
entityList.add(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY));
}
return entityList;
*/
} }
private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor { private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
......
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
...@@ -70,23 +70,20 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -70,23 +70,20 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
} }
LOG.info("Indexes do not exist, Creating indexes for titanGraph."); LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
try { management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
management.buildIndex(Constants.VERTEX_INDEX, Vertex.class) .buildMixedIndex(Constants.BACKING_INDEX);
.buildMixedIndex(Constants.BACKING_INDEX); management.buildIndex(Constants.EDGE_INDEX, Edge.class)
management.buildIndex(Constants.EDGE_INDEX, Edge.class) .buildMixedIndex(Constants.BACKING_INDEX);
.buildMixedIndex(Constants.BACKING_INDEX); management.commit();
// create a composite index for guid as its unique // create a composite index for guid as its unique
createCompositeIndex(management, Constants.GUID_INDEX, createCompositeIndex(Constants.GUID_INDEX,
Constants.GUID_PROPERTY_KEY, String.class, true); Constants.GUID_PROPERTY_KEY, String.class, true);
// create a composite and mixed index for type since it can be combined with other keys // create a composite and mixed index for type since it can be combined with other keys
createCompositeIndex(management, Constants.ENTITY_TYPE_INDEX, createCompositeIndex(Constants.ENTITY_TYPE_INDEX,
Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false); Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false);
createVertexMixedIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class); createVertexMixedIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
} finally {
management.commit();
}
LOG.info("Index creation for global keys complete."); LOG.info("Index creation for global keys complete.");
} }
...@@ -102,19 +99,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -102,19 +99,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
public void onAdd(String typeName, IDataType dataType) throws MetadataException { public void onAdd(String typeName, IDataType dataType) throws MetadataException {
LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType); LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType);
TitanManagement management = titanGraph.getManagementSystem();
try { try {
addIndexForType(management, dataType); addIndexForType(dataType);
management.commit();
LOG.info("Index creation for type {} complete", typeName); LOG.info("Index creation for type {} complete", typeName);
} catch (Exception e) { } catch (Throwable throwable) {
LOG.error("Error creating index for type {}", dataType, e); // gets handle to currently open transaction
management.rollback(); titanGraph.getManagementSystem().rollback();
LOG.error("Error creating index for type {}", dataType, throwable);
} }
} }
private void addIndexForType(TitanManagement management, IDataType dataType) { private void addIndexForType(IDataType dataType) {
switch (dataType.getTypeCategory()) { switch (dataType.getTypeCategory()) {
case PRIMITIVE: case PRIMITIVE:
case ENUM: case ENUM:
...@@ -125,17 +121,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -125,17 +121,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
case STRUCT: case STRUCT:
StructType structType = (StructType) dataType; StructType structType = (StructType) dataType;
createIndexForFields(management, structType, structType.fieldMapping().fields); createIndexForFields(structType, structType.fieldMapping().fields);
break; break;
case TRAIT: case TRAIT:
TraitType traitType = (TraitType) dataType; TraitType traitType = (TraitType) dataType;
createIndexForFields(management, traitType, traitType.fieldMapping().fields); createIndexForFields(traitType, traitType.fieldMapping().fields);
break; break;
case CLASS: case CLASS:
ClassType classType = (ClassType) dataType; ClassType classType = (ClassType) dataType;
createIndexForFields(management, classType, classType.fieldMapping().fields); createIndexForFields(classType, classType.fieldMapping().fields);
break; break;
default: default:
...@@ -143,37 +139,35 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -143,37 +139,35 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
} }
} }
private void createIndexForFields(TitanManagement management, private void createIndexForFields(IDataType dataType, Map<String, AttributeInfo> fields) {
IDataType dataType, Map<String, AttributeInfo> fields) {
for (AttributeInfo field : fields.values()) { for (AttributeInfo field : fields.values()) {
if (field.isIndexable) { if (field.isIndexable) {
createIndexForAttribute(management, dataType.getName(), field); createIndexForAttribute(dataType.getName(), field);
} }
} }
} }
private void createIndexForAttribute(TitanManagement management, private void createIndexForAttribute(String typeName, AttributeInfo field) {
String typeName, AttributeInfo field) {
final String propertyName = typeName + "." + field.name; final String propertyName = typeName + "." + field.name;
switch (field.dataType().getTypeCategory()) { switch (field.dataType().getTypeCategory()) {
case PRIMITIVE: case PRIMITIVE:
createVertexMixedIndex( createVertexMixedIndex(
management, propertyName, getPrimitiveClass(field.dataType())); propertyName, getPrimitiveClass(field.dataType()));
break; break;
case ENUM: case ENUM:
createVertexMixedIndex(management, propertyName, Integer.class); createVertexMixedIndex(propertyName, Integer.class);
break; break;
case ARRAY: case ARRAY:
case MAP: case MAP:
// index the property holder for element names // index the property holder for element names
createVertexMixedIndex(management, propertyName, String.class); createVertexMixedIndex(propertyName, String.class);
break; break;
case STRUCT: case STRUCT:
StructType structType = (StructType) field.dataType(); StructType structType = (StructType) field.dataType();
createIndexForFields(management, structType, structType.fieldMapping().fields); createIndexForFields(structType, structType.fieldMapping().fields);
break; break;
case TRAIT: case TRAIT:
...@@ -182,7 +176,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -182,7 +176,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
case CLASS: case CLASS:
// this is only A reference, index the attribute for edge // this is only A reference, index the attribute for edge
createEdgeMixedIndex(management, propertyName); createEdgeMixedIndex(propertyName);
break; break;
default: default:
...@@ -216,9 +210,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -216,9 +210,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
throw new IllegalArgumentException("unknown data type " + dataType); throw new IllegalArgumentException("unknown data type " + dataType);
} }
private static PropertyKey createCompositeIndex(TitanManagement management, String indexName, private PropertyKey createCompositeIndex(String indexName, String propertyName,
String propertyName, Class propertyClass, Class propertyClass, boolean isUnique) {
boolean isUnique) { TitanManagement management = titanGraph.getManagementSystem();
PropertyKey propertyKey = management.getPropertyKey(propertyName); PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) { if (propertyKey == null) {
propertyKey = management propertyKey = management
...@@ -235,30 +229,39 @@ public class GraphBackedSearchIndexer implements SearchIndexer { ...@@ -235,30 +229,39 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
} }
indexBuilder.buildCompositeIndex(); indexBuilder.buildCompositeIndex();
management.commit();
} }
LOG.info("Created index for property {} in composite index {}", propertyName, indexName);
return propertyKey; return propertyKey;
} }
private static PropertyKey createVertexMixedIndex(TitanManagement management, private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass) {
String propertyName, Class propertyClass) { TitanManagement management = titanGraph.getManagementSystem();
PropertyKey propertyKey = management.getPropertyKey(propertyName); PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) { if (propertyKey == null) {
propertyKey = management propertyKey = management
.makePropertyKey(propertyName) .makePropertyKey(propertyName)
.dataType(propertyClass) .dataType(propertyClass)
.make(); .make();
}
TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX); TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
management.addIndexKey(vertexIndex, propertyKey); management.addIndexKey(vertexIndex, propertyKey);
management.commit();
LOG.info("Created mixed vertex index for property {}", propertyName);
}
return propertyKey; return propertyKey;
} }
private static void createEdgeMixedIndex(TitanManagement management, private void createEdgeMixedIndex(String propertyName) {
String propertyName) { TitanManagement management = titanGraph.getManagementSystem();
EdgeLabel edgeLabel = management.makeEdgeLabel(propertyName).make(); EdgeLabel edgeLabel = management.getEdgeLabel(propertyName);
management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT); if (edgeLabel == null) {
edgeLabel = management.makeEdgeLabel(propertyName).make();
management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
management.commit();
LOG.info("Created index for edge label {}", propertyName);
}
} }
} }
repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphService.java
...@@ -115,6 +115,7 @@ public class TitanGraphService implements GraphService { ...@@ -115,6 +115,7 @@ public class TitanGraphService implements GraphService {
* @throws ConfigurationException * @throws ConfigurationException
*/ */
// TODO move this functionality to the SearchIndexer? // TODO move this functionality to the SearchIndexer?
@Deprecated
protected void createIndicesForVertexKeys() throws ConfigurationException { protected void createIndicesForVertexKeys() throws ConfigurationException {
if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) { if (!titanGraph.getIndexedKeys(Vertex.class).isEmpty()) {
LOG.info("Indexes already exist for titanGraph"); LOG.info("Indexes already exist for titanGraph");
......
repository/src/main/resources/log4j.xml
...@@ -74,6 +74,11 @@ ...@@ -74,6 +74,11 @@
<appender-ref ref="FILE"/> <appender-ref ref="FILE"/>
</logger> </logger>
<logger name="org.elasticsearch" additivity="false">
<level value="warn"/>
<appender-ref ref="FILE"/>
</logger>
<root> <root>
<priority value="debug"/> <priority value="debug"/>
<appender-ref ref="console"/> <appender-ref ref="console"/>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment