Skip to content

A
atlas
Commit a10444d3 by Shwetha GS
Options

ATLAS-1042 Performance improvement changes for propertykey+typeName based…

ATLAS-1042 Performance improvement changes for propertykey+typeName based queries (sumasai via shwethags)
parent b73e62ba
Showing with 39 additions and 10 deletions
release-log.txt
......@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
ATLAS-1042 Performance improvement changes for propertykey+typeName based queries (sumasai via shwethags)
ATLAS-1036 Multiple instances of AtlasPluginClassloader getting initialized (sumasai, mneethiraj)
ATLAS-1033 fix for issues flagged by Coverity scan (mneethiraj via sumasai)
ATLAS-1036 Compilation error on java 1.8 - GraphBackedDiscoveryService (shwethags via sumasai)
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -106,6 +106,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
// create a composite index for entity state
createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, Cardinality.SINGLE, true);
// create a mixed index for entity state. Set systemProperty flag deliberately to false
// so that it doesnt create a composite index which has issues with
// titan 0.5.4 - Refer https://groups.google.com/forum/#!searchin/aureliusgraphs/hemanth/aureliusgraphs/bx7T843mzXU/fjAsclx7GAAJ
createIndexes(management, Constants.STATE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, false);
// create a composite index for entity state
createIndexes(management, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false,
Cardinality.SINGLE, true);
......@@ -170,7 +175,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.info("Creating indexes for type name={}, definition={}", dataType.getName(), dataType.getClass());
try {
addIndexForType(management, dataType);
LOG.debug("Index creation for type {} complete", dataType.getName());
LOG.info("Index creation for type {} complete", dataType.getName());
} catch (Throwable throwable) {
LOG.error("Error creating index for type {}", dataType, throwable);
//Rollback indexes if any failure
......@@ -325,7 +330,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
} else if (isUnique) {
// send uniqueness as false because there can be many vertexes with the same property value
// but state can be active / deleted.
createCompositeIndex(management, propertyName, propertyClass, propertyKey, false);
createCompositeIndexWithTypeName(management, propertyName, propertyClass, propertyKey);
}
}
return propertyKey;
......@@ -333,7 +338,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private void createCompositeIndex(TitanManagement management, String propertyName, Class propertyClass,
PropertyKey propertyKey, boolean enforceUniqueness) {
LOG.debug("Creating composite index for property {} of type {} ", propertyName,
LOG.info("Creating composite index for property {} of type {} ", propertyName,
propertyClass.getName());
TitanManagement.IndexBuilder indexBuilder =
management.buildIndex(propertyName, Vertex.class).addKey(propertyKey);
......@@ -341,17 +346,34 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName());
LOG.info("Created composite index for property {} of type {} ", propertyName, propertyClass.getName());
}
private void createCompositeIndexWithTypeName(TitanManagement management, String propertyName, Class propertyClass,
PropertyKey propertyKey) {
LOG.info("Creating composite index for property {} of type {} and {}", propertyName,
propertyClass.getName(), Constants.ENTITY_TYPE_PROPERTY_KEY);
PropertyKey typePropertyKey = management.getPropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY);
if (typePropertyKey == null) {
typePropertyKey = management.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY).
dataType(String.class).cardinality(Cardinality.SINGLE)
.make();
}
TitanManagement.IndexBuilder indexBuilder =
management.buildIndex(propertyName + Constants.ENTITY_TYPE_PROPERTY_KEY, Vertex.class).
addKey(propertyKey).addKey(typePropertyKey);
indexBuilder.buildCompositeIndex();
LOG.info("Created composite index for property {} of type {} and {}", propertyName, propertyClass.getName(), Constants.ENTITY_TYPE_PROPERTY_KEY);
}
private void enhanceMixedIndex(TitanManagement management, String propertyName, Class propertyClass,
Cardinality cardinality, PropertyKey propertyKey) {
if (checkIfMixedIndexApplicable(propertyClass, cardinality)) {
//Use backing index
LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
LOG.info("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
management.addIndexKey(vertexIndex, propertyKey);
LOG.debug("Created backing index for property {} of type {} ", propertyName, propertyClass.getName());
LOG.info("Created backing index for property {} of type {} ", propertyName, propertyClass.getName());
}
}
......
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -23,6 +23,8 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
......@@ -69,6 +69,8 @@ public class GraphBackedSearchIndexerTest {
assertNotNull(edgeIndex);
assertTrue(edgeIndex.isMixedIndex());
assertTrue(Edge.class.isAssignableFrom(edgeIndex.getIndexedElement()));
verifyVertexIndexContains(managementSystem, Constants.STATE_PROPERTY_KEY);
}
@Test
......@@ -126,15 +128,14 @@ public class GraphBackedSearchIndexerTest {
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
createClassTypeDef("Database", "Database type description", null,
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createUniqueRequiredAttrDef("managedType", managedType));
TypesUtil.createRequiredAttrDef("managedType", managedType));
ClassType databaseType = typeSystem.defineClassType(databaseTypeDefinition);
graphBackedSearchIndexer.onAdd(Arrays.asList(databaseType));
verifySystemCompositeIndex(managementSystem, "Database.name", false);
verifyVertexIndexContains(managementSystem, "Database.name");
verifySystemCompositeIndex(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY, false);
verifyVertexIndexContains(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY);
verifySystemCompositeIndex(managementSystem, "Database.managedType", false);
verifyVertexIndexContains(managementSystem, "Database.managedType");
}
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
......@@ -33,6 +33,7 @@ import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
......@@ -134,6 +135,8 @@ public class GraphRepoMapperScaleTest {
for (int index = 500; index < 600; index++) {
searchWithIndex("hive_table.name", "bar-" + index);
}
searchWithIndex(Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
}
private void searchWithOutIndex(String key, String value) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment