Skip to content

A
atlas
Commit 37b27107 by Sarath Subramanian
Options

ATLAS-2420: Create indexes for relationship attributes

parent d7ac76c4
Showing with 317 additions and 239 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -29,7 +29,9 @@ public final class Constants {
*/
public static final String INTERNAL_PROPERTY_KEY_PREFIX = "__";
public static final String RELATIONSHIP_PROPERTY_KEY_PREFIX = "_r";
public static final String GUID_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "guid";
public static final String RELATIONSHIP_GUID_PROPERTY_KEY = RELATIONSHIP_PROPERTY_KEY_PREFIX + GUID_PROPERTY_KEY;
/**
* Entity type name property key.
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
......@@ -35,15 +35,6 @@ public interface AtlasGraphManagement {
boolean containsPropertyKey(String key);
/**
* Creates a full text index for the given property.
*
* @param indexName the name of the index to create
* @param propertyKey full text property to index
* @param backingIndex the name of the backing index to use
*/
void createFullTextIndex(String indexName, AtlasPropertyKey propertyKey, String backingIndex);
/**
* Rolls back the changes that have been made to the management system.
*/
void rollback();
......@@ -87,13 +78,22 @@ public interface AtlasGraphManagement {
AtlasEdgeLabel getEdgeLabel(String label);
/**
* Creates a composite index for the graph.
* Creates a composite vertex index for the graph.
*
* @param propertyName
* @param isUnique
* @param propertyKeys
*/
void createVertexCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys);
/**
* Creates a composite edge index for the graph.
*
* @param propertyName
* @param isUnique
* @param propertyKeys
*/
void createExactMatchIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys);
void createEdgeCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys);
/**
* Looks up the index with the specified name in the graph. Returns null if
......@@ -109,23 +109,33 @@ public interface AtlasGraphManagement {
*
* @param name the name of the index to create
* @param backingIndex the name of the backing index to use
* @param propertyKeys list of propertyKeys to be added to the index
*/
void createVertexIndex(String name, String backingIndex, List<AtlasPropertyKey> propertyKeys);
void createVertexMixedIndex(String name, String backingIndex, List<AtlasPropertyKey> propertyKeys);
/**
* Adds a property key to the given index in the graph.
* Creates a mixed Edge index for the graph.
*
* @param vertexIndex
* @param propertyKey
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
* @param propertyKeys list of propertyKeys to be added to the index
*/
void addVertexIndexKey(String vertexIndex, AtlasPropertyKey propertyKey);
void createEdgeMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys);
/**
* Creates a mixed Edge index for the graph.
* Creates a full text index for the given property.
*
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
* @param propertyKeys list of propertyKeys to be added to the index
*/
void createEdgeIndex(String index, String backingIndex);
void createFullTextMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys);
/**
* Adds a property key to the given index in the graph.
*
* @param vertexIndex
* @param propertyKey
*/
void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
}
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
......@@ -25,6 +25,7 @@ import org.janusgraph.core.schema.Mapping;
import org.janusgraph.core.schema.PropertyKeyMaker;
import org.janusgraph.core.schema.JanusGraphIndex;
import org.janusgraph.core.schema.JanusGraphManagement;
import org.janusgraph.core.schema.JanusGraphManagement.IndexBuilder;
import org.janusgraph.graphdb.internal.Token;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
......@@ -33,7 +34,6 @@ import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.commons.lang.StringUtils;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Element;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -46,50 +46,52 @@ import java.util.Set;
* Janus implementation of AtlasGraphManagement.
*/
public class AtlasJanusGraphManagement implements AtlasGraphManagement {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphManagement.class);
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphManagement.class);
private static final char[] RESERVED_CHARS = { '{', '}', '"', '$', Token.SEPARATOR_CHAR };
private AtlasJanusGraph graph;
private AtlasJanusGraph graph;
private JanusGraphManagement management;
private Set<String> newMultProperties = new HashSet<>();
private Set<String> newMultProperties = new HashSet<>();
public AtlasJanusGraphManagement(AtlasJanusGraph graph, JanusGraphManagement managementSystem) {
this.management = managementSystem;
this.graph = graph;
this.graph = graph;
}
@Override
public void createVertexIndex(String propertyName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
public void createVertexMixedIndex(String indexName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
IndexBuilder indexBuilder = management.buildIndex(indexName, Vertex.class);
JanusGraphManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for (AtlasPropertyKey key : propertyKeys) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
indexBuilder.addKey(janusKey);
}
indexBuilder.buildMixedIndex(backingIndex);
}
@Override
public void createEdgeIndex(String index, String backingIndex) {
buildMixedIndex(index, Edge.class, backingIndex);
}
public void createEdgeMixedIndex(String indexName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
IndexBuilder indexBuilder = management.buildIndex(indexName, Edge.class);
private void buildMixedIndex(String index, Class<? extends Element> janusClass, String backingIndex) {
for (AtlasPropertyKey key : propertyKeys) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
indexBuilder.addKey(janusKey);
}
management.buildIndex(index, janusClass).buildMixedIndex(backingIndex);
indexBuilder.buildMixedIndex(backingIndex);
}
@Override
public void createFullTextIndex(String indexName, AtlasPropertyKey propertyKey, String backingIndex) {
public void createFullTextMixedIndex(String indexName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
IndexBuilder indexBuilder = management.buildIndex(indexName, Vertex.class);
PropertyKey fullText = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
for (AtlasPropertyKey key : propertyKeys) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
indexBuilder.addKey(janusKey, org.janusgraph.core.schema.Parameter.of("mapping", Mapping.TEXT));
}
management.buildIndex(indexName, Vertex.class)
.addKey(fullText, org.janusgraph.core.schema.Parameter.of("mapping", Mapping.TEXT))
.buildMixedIndex(backingIndex);
indexBuilder.buildMixedIndex(backingIndex);
}
@Override
......@@ -114,25 +116,28 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
//for some reason, name checking was removed from StandardPropertyKeyMaker.make()
//in Janus. For consistency, do the check here.
Preconditions.checkArgument(StringUtils.isNotBlank(name), "Need to specify name");
for (char c : RESERVED_CHARS) {
Preconditions.checkArgument(name.indexOf(c) < 0, "Name can not contains reserved character %s: %s", c,
name);
Preconditions.checkArgument(name.indexOf(c) < 0, "Name can not contains reserved character %s: %s", c, name);
}
}
@Override
public AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, AtlasCardinality cardinality) {
if (cardinality.isMany()) {
newMultProperties.add(propertyName);
}
PropertyKeyMaker propertyKeyBuilder = management.makePropertyKey(propertyName).dataType(propertyClass);
if (cardinality != null) {
Cardinality janusCardinality = AtlasJanusObjectFactory.createCardinality(cardinality);
propertyKeyBuilder.cardinality(janusCardinality);
}
PropertyKey propertyKey = propertyKeyBuilder.make();
return GraphDbObjectFactory.createPropertyKey(propertyKey);
}
......@@ -151,6 +156,7 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
for (int i = 0;; i++) {
String deletedKeyName = janusPropertyKey + "_deleted_" + i;
if (null == management.getPropertyKey(deletedKeyName)) {
management.changeName(janusPropertyKey, deletedKeyName);
break;
......@@ -161,6 +167,7 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
@Override
public AtlasPropertyKey getPropertyKey(String propertyName) {
checkName(propertyName);
return GraphDbObjectFactory.createPropertyKey(management.getPropertyKey(propertyName));
}
......@@ -169,37 +176,50 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
return GraphDbObjectFactory.createEdgeLabel(management.getEdgeLabel(label));
}
public void createExactMatchVertexIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
JanusGraphManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for (AtlasPropertyKey key : propertyKeys) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
indexBuilder.addKey(janusKey);
}
if (enforceUniqueness) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
}
@Override
public void addVertexIndexKey(String indexName, AtlasPropertyKey propertyKey) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
management.addIndexKey(vertexIndex, janusKey);
}
@Override
public AtlasGraphIndex getGraphIndex(String indexName) {
JanusGraphIndex index = management.getGraphIndex(indexName);
return GraphDbObjectFactory.createGraphIndex(index);
}
@Override
public void createExactMatchIndex(String propertyName, boolean isUnique,
List<AtlasPropertyKey> propertyKeys) {
createExactMatchVertexIndex(propertyName, isUnique, propertyKeys);
public void createVertexCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys) {
IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for (AtlasPropertyKey key : propertyKeys) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
indexBuilder.addKey(janusKey);
}
if (isUnique) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
}
}
@Override
public void createEdgeCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys) {
IndexBuilder indexBuilder = management.buildIndex(propertyName, Edge.class);
for (AtlasPropertyKey key : propertyKeys) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
indexBuilder.addKey(janusKey);
}
if (isUnique) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
}
}
\ No newline at end of file
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
......@@ -56,8 +56,7 @@ public abstract class AbstractGraphDatabaseTest {
AtlasGraphManagement mgmt = db.getGraph().getManagementSystem();
if (mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
mgmt.createVertexIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX,
Collections.<AtlasPropertyKey>emptyList());
mgmt.createVertexMixedIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX, Collections.emptyList());
}
mgmt.makePropertyKey("age13", Integer.class, AtlasCardinality.SINGLE);
......@@ -100,14 +99,14 @@ public abstract class AbstractGraphDatabaseTest {
AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
try {
if (propertyClass != Integer.class) {
management.addVertexIndexKey(BACKING_INDEX_NAME, key);
management.addMixedIndex(BACKING_INDEX_NAME, key);
}
} catch(Throwable t) {
//ok
t.printStackTrace();
}
try {
management.createExactMatchIndex(propertyName, isUnique, Collections.singletonList(key));
management.createVertexCompositeIndex(propertyName, isUnique, Collections.singletonList(key));
} catch(Throwable t) {
//ok
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusDatabaseTest.java
......@@ -30,10 +30,8 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.math.BigDecimal;
......@@ -43,7 +41,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.testng.Assert.*;
/**
......@@ -67,7 +64,7 @@ public class AtlasJanusDatabaseTest {
AtlasPropertyKey propertyKey = mgmt.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = mgmt.makePropertyKey(propertyName, String.class, AtlasCardinality.SET);
mgmt.createExactMatchIndex(propertyName, false, Collections.singletonList(propertyKey));
mgmt.createVertexCompositeIndex(propertyName, false, Collections.singletonList(propertyKey));
}
}
mgmt.commit();
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java
......@@ -58,23 +58,16 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
}
@Override
public void createEdgeIndex(String index, String backingIndex) {
buildMixedIndex(index, Edge.class, backingIndex);
}
private void buildMixedIndex(String index, Class<? extends Element> titanClass, String backingIndex) {
management.buildIndex(index, titanClass).buildMixedIndex(backingIndex);
public void createEdgeMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
}
@Override
public void createFullTextIndex(String indexName, AtlasPropertyKey propertyKey, String backingIndex) {
public void createFullTextMixedIndex(String index, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
}
PropertyKey fullText = TitanObjectFactory.createPropertyKey(propertyKey);
private void buildMixedIndex(String index, Class<? extends Element> titanClass, String backingIndex) {
management.buildIndex(indexName, Vertex.class)
.addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
.buildMixedIndex(backingIndex);
management.buildIndex(index, titanClass).buildMixedIndex(backingIndex);
}
@Override
......@@ -146,8 +139,8 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
}
@Override
public void createExactMatchIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
public void createVertexCompositeIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for(AtlasPropertyKey key : propertyKeys) {
......@@ -161,7 +154,23 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
}
@Override
public void createVertexIndex(String propertyName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
public void createEdgeCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Edge.class);
for(AtlasPropertyKey key : propertyKeys) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(key);
indexBuilder.addKey(titanKey);
}
if (isUnique) {
indexBuilder.unique();
}
indexBuilder.buildCompositeIndex();
}
@Override
public void createVertexMixedIndex(String propertyName, String backingIndex, List<AtlasPropertyKey> propertyKeys) {
TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
for(AtlasPropertyKey key : propertyKeys) {
......@@ -173,7 +182,7 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
@Override
public void addVertexIndexKey(String indexName, AtlasPropertyKey propertyKey) {
public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
PropertyKey titanKey = TitanObjectFactory.createPropertyKey(propertyKey);
TitanGraphIndex vertexIndex = management.getGraphIndex(indexName);
management.addIndexKey(vertexIndex, titanKey);
......
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/AbstractGraphDatabaseTest.java
......@@ -55,8 +55,7 @@ public abstract class AbstractGraphDatabaseTest {
AtlasGraphManagement mgmt = db.getGraph().getManagementSystem();
if (mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
mgmt.createVertexIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX,
Collections.<AtlasPropertyKey>emptyList());
mgmt.createVertexMixedIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX, Collections.emptyList());
}
mgmt.makePropertyKey("age13", Integer.class, AtlasCardinality.SINGLE);
......@@ -100,14 +99,14 @@ public abstract class AbstractGraphDatabaseTest {
AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
try {
if (propertyClass != Integer.class) {
management.addVertexIndexKey(BACKING_INDEX_NAME, key);
management.addMixedIndex(BACKING_INDEX_NAME, key);
}
} catch(Throwable t) {
//ok
t.printStackTrace();
}
try {
management.createExactMatchIndex(propertyName, isUnique, Collections.singletonList(key));
management.createVertexCompositeIndex(propertyName, isUnique, Collections.singletonList(key));
} catch(Throwable t) {
//ok
......
graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseTest.java
......@@ -65,7 +65,7 @@ public class Titan0DatabaseTest {
AtlasPropertyKey propertyKey = mgmt.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = mgmt.makePropertyKey(propertyName, String.class, AtlasCardinality.SET);
mgmt.createExactMatchIndex(propertyName, false, Collections.singletonList(propertyKey));
mgmt.createVertexCompositeIndex(propertyName, false, Collections.singletonList(propertyKey));
}
}
mgmt.commit();
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -44,6 +44,7 @@ import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasEnumType;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
......@@ -64,6 +65,26 @@ import java.util.List;
import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
import static org.apache.atlas.repository.Constants.BACKING_INDEX;
import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
import static org.apache.atlas.repository.Constants.EDGE_INDEX;
import static org.apache.atlas.repository.Constants.ENTITY_TEXT_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.FULLTEXT_INDEX;
import static org.apache.atlas.repository.Constants.GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.SUPER_TYPES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERTEX_INDEX;
import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SINGLE;
/**
......@@ -73,8 +94,8 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChangeHandler, TypeDefChangeListener {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
private static final List<Class> VERTEX_INDEX_EXCLUSIONS = new ArrayList() {
private static final List<Class> INDEX_EXCLUSION_CLASSES = new ArrayList() {
{
add(Boolean.class);
add(BigDecimal.class);
......@@ -97,7 +118,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
@VisibleForTesting
GraphBackedSearchIndexer( IAtlasGraphProvider provider, Configuration configuration, AtlasTypeRegistry typeRegistry)
GraphBackedSearchIndexer(IAtlasGraphProvider provider, Configuration configuration, AtlasTypeRegistry typeRegistry)
throws IndexException, RepositoryException {
this.provider = provider;
this.typeRegistry = typeRegistry;
......@@ -178,7 +199,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
management = provider.get().getManagementSystem();
if (management != null) {
AtlasGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
AtlasGraphIndex vertexIndex = management.getGraphIndex(VERTEX_INDEX);
if (vertexIndex != null) {
recomputeIndexedKeys = false;
......@@ -214,12 +235,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
* Initializes the indices for the graph - create indices for Global AtlasVertex Keys
*/
private void initialize() throws RepositoryException, IndexException {
initialize(provider.get());
initialize(provider.get());
}
/**
* Initializes the indices for the graph - create indices for Global AtlasVertex Keys
* Initializes the indices for the graph - create indices for Global AtlasVertex and AtlasEdge Keys
*/
private void initialize(AtlasGraph graph) throws RepositoryException, IndexException {
AtlasGraphManagement management = graph.getManagementSystem();
......@@ -227,64 +247,45 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
try {
LOG.info("Creating indexes for graph.");
if (management.getGraphIndex(Constants.VERTEX_INDEX) == null) {
management.createVertexIndex(Constants.VERTEX_INDEX, Constants.BACKING_INDEX, Collections.<AtlasPropertyKey>emptyList());
if (management.getGraphIndex(VERTEX_INDEX) == null) {
management.createVertexMixedIndex(VERTEX_INDEX, BACKING_INDEX, Collections.emptyList());
LOG.info("Created index {}", Constants.VERTEX_INDEX);
LOG.info("Created index : {}", VERTEX_INDEX);
}
if (management.getGraphIndex(Constants.EDGE_INDEX) == null) {
management.createEdgeIndex(Constants.EDGE_INDEX, Constants.BACKING_INDEX);
if (management.getGraphIndex(EDGE_INDEX) == null) {
management.createEdgeMixedIndex(EDGE_INDEX, BACKING_INDEX, Collections.emptyList());
LOG.info("Created index {}", Constants.EDGE_INDEX);
LOG.info("Created index : {}", EDGE_INDEX);
}
if (management.getGraphIndex(FULLTEXT_INDEX) == null) {
management.createFullTextMixedIndex(FULLTEXT_INDEX, BACKING_INDEX, Collections.emptyList());
// create a composite index for guid as its unique
createIndexes(management, Constants.GUID_PROPERTY_KEY, String.class, true,
AtlasCardinality.SINGLE, true, true);
// Add creation_timestamp property to Vertex Index (mixed index)
createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, AtlasCardinality.SINGLE, false, false);
// Add modification_timestamp property to Vertex Index (mixed index)
createIndexes(management, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false,
AtlasCardinality.SINGLE, false, false);
// create a mixed index for entity state. Set systemProperty flag deliberately to false
// so that it doesnt create a composite index which has issues with
// titan 0.5.4 - Refer https://groups.google.com/forum/#!searchin/aureliusgraphs/hemanth/aureliusgraphs/bx7T843mzXU/fjAsclx7GAAJ
createIndexes(management, Constants.STATE_PROPERTY_KEY, String.class, false, AtlasCardinality.SINGLE, false, false);
// Create a composite and mixed index for created by property
createIndexes(management, Constants.CREATED_BY_KEY, String.class, false,
AtlasCardinality.SINGLE, true, true);
// Create a composite and mixed index for modified by property
createIndexes(management, Constants.MODIFIED_BY_KEY, String.class, false,
AtlasCardinality.SINGLE, true, true);
// create a composite and mixed index for type since it can be combined with other keys
createIndexes(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, AtlasCardinality.SINGLE,
true, true);
// create a composite and mixed index for type since it can be combined with other keys
createIndexes(management, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET,
true, true);
// create a composite and mixed index for traitNames since it can be combined with other
// keys. Traits must be a set and not a list.
createIndexes(management, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, AtlasCardinality.SET,
true, true);
// Index for full text search
createFullTextIndex(management);
LOG.info("Created index : {}", FULLTEXT_INDEX);
}
//Indexes for graph backed type system store
createTypeStoreIndexes(management);
// create vertex indexes
createVertexIndex(management, GUID_PROPERTY_KEY, String.class, true, SINGLE, true, true);
createVertexIndex(management, TIMESTAMP_PROPERTY_KEY, Long.class, false, SINGLE, false, false);
createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false, SINGLE, false, false);
createVertexIndex(management, STATE_PROPERTY_KEY, String.class, false, SINGLE, false, false);
createVertexIndex(management, CREATED_BY_KEY, String.class, false, SINGLE, true, true);
createVertexIndex(management, MODIFIED_BY_KEY, String.class, false, SINGLE, true, true);
createVertexIndex(management, ENTITY_TYPE_PROPERTY_KEY, String.class, false, SINGLE, true, true);
createVertexIndex(management, SUPER_TYPES_PROPERTY_KEY, String.class, false, SET, true, true);
createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, String.class, false, SET, true, true);
createVertexIndex(management, TYPENAME_PROPERTY_KEY, String.class, true, SINGLE, true, true);
createVertexIndex(management, VERTEX_TYPE_PROPERTY_KEY, String.class, false, SINGLE, true, true);
// create edge indexes
createEdgeIndex(management, RELATIONSHIP_GUID_PROPERTY_KEY, String.class, SINGLE, true);
// create fulltext indexes
createFullTextIndex(management, ENTITY_TEXT_PROPERTY_KEY, String.class, SINGLE);
commit(management);
LOG.info("Index creation for global keys complete.");
} catch (Throwable t) {
rollback(management);
......@@ -292,27 +293,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private void createFullTextIndex(AtlasGraphManagement management) {
if (!management.containsPropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY)) {
AtlasPropertyKey fullText =
management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY, String.class, AtlasCardinality.SINGLE);
management.createFullTextIndex(Constants.FULLTEXT_INDEX, fullText, Constants.BACKING_INDEX);
LOG.info("Created index {}", Constants.ENTITY_TEXT_PROPERTY_KEY);
}
}
private void createTypeStoreIndexes(AtlasGraphManagement management) {
//Create unique index on typeName
createIndexes(management, Constants.TYPENAME_PROPERTY_KEY, String.class, true, AtlasCardinality.SINGLE,
true, true);
//create index on vertex type
createIndexes(management, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, false, AtlasCardinality.SINGLE,
true, true);
}
private void addIndexForType(AtlasGraphManagement management, AtlasBaseTypeDef typeDef) {
if (typeDef instanceof AtlasEnumDef) {
// Only handle complex types like Struct, Classification and Entity
......@@ -331,32 +311,39 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private void createIndexForAttribute(AtlasGraphManagement management, String typeName,
AtlasAttributeDef attributeDef) {
final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName());
AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality());
boolean isUnique = attributeDef.getIsUnique();
boolean isIndexable = attributeDef.getIsIndexable();
String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName);
boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName);
private void createIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) {
final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName());
AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality());
boolean isUnique = attributeDef.getIsUnique();
boolean isIndexable = attributeDef.getIsIndexable();
String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName);
boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName);
try {
AtlasType atlasType = typeRegistry.getType(attribTypeName);
AtlasType atlasType = typeRegistry.getType(typeName);
AtlasType attributeType = typeRegistry.getType(attribTypeName);
if (isMapType || isClassificationType(atlasType)) {
if (isMapType || isClassificationType(attributeType)) {
LOG.warn("Ignoring non-indexable attribute {}", attribTypeName);
} if (isArrayType) {
createLabelIfNeeded(management, propertyName, attribTypeName);
} if (isEntityType(atlasType)) {
} if (isEntityType(attributeType)) {
createEdgeLabel(management, propertyName);
} else if (isBuiltInType) {
createIndexes(management, propertyName, getPrimitiveClass(attribTypeName), isUnique, cardinality, false, isIndexable);
} else if (isEnumType(atlasType)) {
createIndexes(management, propertyName, String.class, isUnique, cardinality, false, isIndexable);
} else if (isStructType(atlasType)) {
if (isRelationshipType(atlasType)) {
createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false);
} else {
createVertexIndex(management, propertyName, getPrimitiveClass(attribTypeName), isUnique, cardinality, false, isIndexable);
}
} else if (isEnumType(attributeType)) {
if (isRelationshipType(atlasType)) {
createEdgeIndex(management, propertyName, String.class, cardinality, false);
} else {
createVertexIndex(management, propertyName, String.class, isUnique, cardinality, false, isIndexable);
}
} else if (isStructType(attributeType)) {
AtlasStructDef structDef = typeRegistry.getStructDefByName(attribTypeName);
updateIndexForTypeDef(management, structDef);
}
......@@ -391,6 +378,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
return type instanceof AtlasStructType;
}
private boolean isRelationshipType(AtlasType type) {
return type instanceof AtlasRelationshipType;
}
private Class getPrimitiveClass(String attribTypeName) {
switch (attribTypeName.toLowerCase()) {
case ATLAS_TYPE_BOOLEAN:
......@@ -422,11 +413,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private AtlasCardinality toAtlasCardinality(AtlasAttributeDef.Cardinality cardinality) {
switch (cardinality) {
case SINGLE:
return AtlasCardinality.SINGLE;
return SINGLE;
case LIST:
return AtlasCardinality.LIST;
return LIST;
case SET:
return AtlasCardinality.SET;
return SET;
}
// Should never reach this point
throw new IllegalArgumentException(String.format("Bad cardinality %s", cardinality));
......@@ -448,25 +439,61 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private AtlasPropertyKey createIndexes(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality, boolean createCompositeForAttribute,
boolean createCompositeWithTypeandSuperTypes) {
private AtlasPropertyKey createVertexIndex(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality, boolean createCompositeIndex,
boolean createCompositeIndexWithTypeAndSuperTypes) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
if (isIndexApplicable(propertyClass, cardinality)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
management.addMixedIndex(VERTEX_INDEX, propertyKey);
LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
}
if (propertyKey != null) {
if (createCompositeIndex) {
createVertexCompositeIndex(management, propertyClass, propertyKey, isUnique);
} else if (createCompositeIndexWithTypeAndSuperTypes) {
createVertexCompositeIndexWithTypeName(management, propertyClass, propertyKey);
createVertexCompositeIndexWithSuperTypeName(management, propertyClass, propertyKey);
}
} else {
LOG.warn("Index not created for {}: propertyKey is null", propertyName);
}
return propertyKey;
}
private AtlasPropertyKey createEdgeIndex(AtlasGraphManagement management, String propertyName, Class propertyClass,
AtlasCardinality cardinality, boolean createCompositeIndex) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
updateVertexIndex(management, propertyName, propertyClass, cardinality, propertyKey);
if (isIndexApplicable(propertyClass, cardinality)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for edge property {} of type {} ", propertyName, propertyClass.getName());
}
management.addMixedIndex(EDGE_INDEX, propertyKey);
LOG.info("Created backing index for edge property {} of type {} ", propertyName, propertyClass.getName());
}
}
if (propertyKey != null) {
if (createCompositeForAttribute) {
createExactMatchIndex(management, propertyClass, propertyKey, isUnique);
} else if (createCompositeWithTypeandSuperTypes) {
// Index with typename since typename+property key queries need to
// speed up
createExactMatchIndexWithTypeName(management, propertyClass, propertyKey);
createExactMatchIndexWithSuperTypeName(management, propertyClass, propertyKey);
if (createCompositeIndex) {
createEdgeCompositeIndex(management, propertyClass, propertyKey);
}
} else {
LOG.warn("Index not created for {}: propertyKey is null", propertyName);
......@@ -474,10 +501,32 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
return propertyKey;
}
private AtlasPropertyKey createFullTextIndex(AtlasGraphManagement management, String propertyName, Class propertyClass,
AtlasCardinality cardinality) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
if (isIndexApplicable(propertyClass, cardinality)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
management.addMixedIndex(FULLTEXT_INDEX, propertyKey);
LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
LOG.info("Created index {}", FULLTEXT_INDEX);
}
return propertyKey;
}
private void createExactMatchIndex(AtlasGraphManagement management, Class propertyClass,
AtlasPropertyKey propertyKey, boolean enforceUniqueness) {
private void createVertexCompositeIndex(AtlasGraphManagement management, Class propertyClass, AtlasPropertyKey propertyKey,
boolean enforceUniqueness) {
String propertyName = propertyKey.getName();
if (LOG.isDebugEnabled()) {
......@@ -485,33 +534,42 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
AtlasGraphIndex existingIndex = management.getGraphIndex(propertyName);
if (existingIndex == null) {
management.createExactMatchIndex(propertyName, enforceUniqueness, Collections.singletonList(propertyKey));
management.createVertexCompositeIndex(propertyName, enforceUniqueness, Collections.singletonList(propertyKey));
LOG.info("Created composite index for property {} of type {}; isUnique={} ", propertyName, propertyClass.getName(), enforceUniqueness);
}
}
private void createExactMatchIndexWithTypeName(AtlasGraphManagement management,
Class propertyClass, AtlasPropertyKey propertyKey) {
createExactMatchIndexWithSystemProperty(management, propertyClass, propertyKey,
Constants.ENTITY_TYPE_PROPERTY_KEY, AtlasCardinality.SINGLE);
private void createEdgeCompositeIndex(AtlasGraphManagement management, Class propertyClass, AtlasPropertyKey propertyKey) {
String propertyName = propertyKey.getName();
if (LOG.isDebugEnabled()) {
LOG.debug("Creating composite index for property {} of type {}", propertyName, propertyClass.getName());
}
AtlasGraphIndex existingIndex = management.getGraphIndex(propertyName);
if (existingIndex == null) {
management.createEdgeCompositeIndex(propertyName, false, Collections.singletonList(propertyKey));
LOG.info("Created composite index for property {} of type {}", propertyName, propertyClass.getName());
}
}
private void createExactMatchIndexWithSuperTypeName(AtlasGraphManagement management,
Class propertyClass, AtlasPropertyKey propertyKey) {
createExactMatchIndexWithSystemProperty(management, propertyClass, propertyKey,
Constants.SUPER_TYPES_PROPERTY_KEY, AtlasCardinality.SET);
private void createVertexCompositeIndexWithTypeName(AtlasGraphManagement management, Class propertyClass, AtlasPropertyKey propertyKey) {
createVertexCompositeIndexWithSystemProperty(management, propertyClass, propertyKey, ENTITY_TYPE_PROPERTY_KEY, SINGLE);
}
private void createExactMatchIndexWithSystemProperty(AtlasGraphManagement management,
Class propertyClass, AtlasPropertyKey propertyKey, final String systemPropertyKey,
AtlasCardinality cardinality) {
private void createVertexCompositeIndexWithSuperTypeName(AtlasGraphManagement management, Class propertyClass, AtlasPropertyKey propertyKey) {
createVertexCompositeIndexWithSystemProperty(management, propertyClass, propertyKey, SUPER_TYPES_PROPERTY_KEY, SET);
}
private void createVertexCompositeIndexWithSystemProperty(AtlasGraphManagement management, Class propertyClass, AtlasPropertyKey propertyKey,
final String systemPropertyKey, AtlasCardinality cardinality) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating composite index for property {} of type {} and {}", propertyKey.getName(), propertyClass.getName(),
systemPropertyKey);
LOG.debug("Creating composite index for property {} of type {} and {}", propertyKey.getName(), propertyClass.getName(), systemPropertyKey);
}
AtlasPropertyKey typePropertyKey = management.getPropertyKey(systemPropertyKey);
......@@ -526,33 +584,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
List<AtlasPropertyKey> keys = new ArrayList<>(2);
keys.add(propertyKey);
keys.add(typePropertyKey);
management.createExactMatchIndex(indexName, false, keys);
LOG.info("Created composite index for property {} of type {} and {}", propertyKey.getName(), propertyClass.getName(),
systemPropertyKey);
}
}
private void updateVertexIndex(AtlasGraphManagement management, String propertyName, Class propertyClass,
AtlasCardinality cardinality, AtlasPropertyKey propertyKey) {
if (checkIfVertexIndexApplicable(propertyClass, cardinality)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
}
management.createVertexCompositeIndex(indexName, false, keys);
// Use backing index
management.addVertexIndexKey(Constants.VERTEX_INDEX, propertyKey);
LOG.info("Created backing index for property {} of type {} ", propertyName, propertyClass.getName());
LOG.info("Created composite index for property {} of type {} and {}", propertyKey.getName(), propertyClass.getName(), systemPropertyKey);
}
}
private boolean checkIfVertexIndexApplicable(Class propertyClass, AtlasCardinality cardinality) {
return !(VERTEX_INDEX_EXCLUSIONS.contains(propertyClass) || cardinality.isMany());
private boolean isIndexApplicable(Class propertyClass, AtlasCardinality cardinality) {
return !(INDEX_EXCLUSION_CLASSES.contains(propertyClass) || cardinality.isMany());
}
private void commit(AtlasGraphManagement management) throws IndexException {
try {
management.commit();
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -591,7 +591,7 @@ public final class GraphHelper {
AtlasEdge ret;
try {
ret = findEdge(Constants.GUID_PROPERTY_KEY, guid);
ret = findEdge(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, guid);
} catch (EntityNotFoundException e) {
throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_GUID_NOT_FOUND, guid);
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
......@@ -423,7 +423,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
final String guid = UUID.randomUUID().toString();
AtlasGraphUtilsV1.setProperty(ret, Constants.ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
AtlasGraphUtilsV1.setProperty(ret, Constants.GUID_PROPERTY_KEY, guid);
AtlasGraphUtilsV1.setProperty(ret, Constants.RELATIONSHIP_GUID_PROPERTY_KEY, guid);
AtlasGraphUtilsV1.setProperty(ret, Constants.VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
AtlasGraphUtilsV1.setProperty(ret, Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name());
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment