Skip to content

A
atlas
Commit d234de2d by Sarath Subramanian
Options

ATLAS-3112: Allow Indexing of array attributes (LIST or SET) in indexing store

parent 984445e9
Showing with 70 additions and 34 deletions
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
......@@ -155,6 +155,7 @@ public interface AtlasGraphManagement {
*
* @param vertexIndex
* @param propertyKey
* @param propertyClass
*/
void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey, Class propertyClass);
}
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
......@@ -195,11 +195,15 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
}
@Override
public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey, Class propertyClass) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
management.addIndexKey(vertexIndex, janusKey);
if (propertyClass == String.class) {
management.addIndexKey(vertexIndex, janusKey, Mapping.STRING.asParameter());
} else {
management.addIndexKey(vertexIndex, janusKey);
}
}
@Override
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
......@@ -99,7 +99,7 @@ public abstract class AbstractGraphDatabaseTest {
AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
try {
if (propertyClass != Integer.class) {
management.addMixedIndex(BACKING_INDEX_NAME, key);
management.addMixedIndex(BACKING_INDEX_NAME, key, propertyClass);
}
} catch(Throwable t) {
//ok
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -271,12 +271,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, UniqueKind.NONE, Long.class, SINGLE, true, false);
createVertexIndex(management, ENTITY_TYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
createVertexIndex(management, SUPER_TYPES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false);
createVertexIndex(management, TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
createVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, SUPER_TYPES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false);
createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
......@@ -351,10 +352,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createLabelIfNeeded(management, propertyName, attribTypeName);
AtlasArrayType arrayType = (AtlasArrayType) attributeType;
boolean isReference = isReference(arrayType.getElementType());
AtlasType elementType = arrayType.getElementType();
boolean isReference = isReference(elementType);
if (!isReference) {
createPropertyKey(management, propertyName, ArrayList.class, SINGLE);
createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(elementType.getTypeName()), cardinality, isIndexable, false);
}
}
......@@ -508,12 +510,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
if (isIndexApplicable(propertyClass, cardinality)) {
if (isIndexApplicable(propertyClass)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
management.addMixedIndex(VERTEX_INDEX, propertyKey);
management.addMixedIndex(VERTEX_INDEX, propertyKey, propertyClass);
LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
......@@ -591,12 +593,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
if (isIndexApplicable(propertyClass, cardinality)) {
if (isIndexApplicable(propertyClass)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for edge property {} of type {} ", propertyName, propertyClass.getName());
}
management.addMixedIndex(EDGE_INDEX, propertyKey);
management.addMixedIndex(EDGE_INDEX, propertyKey, propertyClass);
LOG.info("Created backing index for edge property {} of type {} ", propertyName, propertyClass.getName());
}
......@@ -619,12 +621,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
if (isIndexApplicable(propertyClass, cardinality)) {
if (isIndexApplicable(propertyClass)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
management.addMixedIndex(FULLTEXT_INDEX, propertyKey);
management.addMixedIndex(FULLTEXT_INDEX, propertyKey, propertyClass);
LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
......@@ -700,8 +702,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private boolean isIndexApplicable(Class propertyClass, AtlasCardinality cardinality) {
return !(INDEX_EXCLUSION_CLASSES.contains(propertyClass) || cardinality.isMany());
private boolean isIndexApplicable(Class propertyClass) {
return !INDEX_EXCLUSION_CLASSES.contains(propertyClass);
}
public void commit(AtlasGraphManagement management) throws IndexException {
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -199,7 +199,7 @@ public final class GraphHelper {
AtlasGraphUtilsV2.setEncodedProperty(ret, MODIFIED_BY_KEY, RequestContext.get().getUser());
for (String superTypeName : superTypeNames) {
AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
AtlasGraphUtilsV2.addToEncodedSetProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
return ret;
......@@ -1545,7 +1545,7 @@ public final class GraphHelper {
if (isReference(elementType)) {
return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute);
} else {
return (List) instanceVertex.getListProperty(propertyName);
return (List) instanceVertex.getPropertyValues(propertyName, List.class);
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
......@@ -32,6 +32,7 @@ import org.apache.atlas.model.patches.AtlasPatch;
import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches;
import org.apache.atlas.model.patches.AtlasPatch.PatchStatus;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
......@@ -64,6 +65,8 @@ import java.util.Map;
import java.util.Set;
import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.LIST;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
......@@ -187,15 +190,23 @@ public class AtlasGraphUtilsV2 {
* @param propertyName
* @param value
*/
public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Object value) {
return addProperty(vertex, propertyName, value, false);
public static AtlasVertex addToListProperty(AtlasVertex vertex, String propertyName, Object value) {
return addProperty(vertex, propertyName, LIST, false, value);
}
public static AtlasVertex addEncodedProperty(AtlasVertex vertex, String propertyName, Object value) {
return addProperty(vertex, propertyName, value, true);
public static AtlasVertex addToSetProperty(AtlasVertex vertex, String propertyName, Object value) {
return addProperty(vertex, propertyName, SET, false, value);
}
public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Object value, boolean isEncoded) {
public static AtlasVertex addToEncodedListProperty(AtlasVertex vertex, String propertyName, Object value) {
return addProperty(vertex, propertyName, LIST, true, value);
}
public static AtlasVertex addToEncodedSetProperty(AtlasVertex vertex, String propertyName, Object value) {
return addProperty(vertex, propertyName, SET, true, value);
}
public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Cardinality cardinality, boolean isEncoded, Object value) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> addProperty({}, {}, {})", toString(vertex), propertyName, value);
}
......@@ -204,7 +215,11 @@ public class AtlasGraphUtilsV2 {
propertyName = encodePropertyKey(propertyName);
}
vertex.addProperty(propertyName, value);
if (cardinality == LIST) {
vertex.addListProperty(propertyName, value);
} else {
vertex.addProperty(propertyName, value);
}
return vertex;
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
......@@ -78,6 +78,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CR
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.LIST;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
......@@ -116,6 +117,8 @@ import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled
import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.addToListProperty;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.addToSetProperty;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isReference;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
......@@ -165,7 +168,7 @@ public class EntityGraphMapper {
AtlasVertex ret = createStructVertex(entity);
for (String superTypeName : entityType.getAllSuperTypes()) {
AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
AtlasGraphUtilsV2.addToEncodedSetProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
AtlasGraphUtilsV2.setEncodedProperty(ret, GUID_PROPERTY_KEY, guid);
......@@ -310,7 +313,10 @@ public class EntityGraphMapper {
AtlasVertex ret = createStructVertex(classification);
AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, classificationType.getAllSuperTypes());
for (String superTypeName : classificationType.getAllSuperTypes()) {
AtlasGraphUtilsV2.addToEncodedSetProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
AtlasGraphUtilsV2.setEncodedProperty(ret, CLASSIFICATION_ENTITY_GUID, classification.getEntityGuid());
AtlasGraphUtilsV2.setEncodedProperty(ret, CLASSIFICATION_ENTITY_STATUS, classification.getEntityStatus().name());
......@@ -1080,9 +1086,9 @@ public class EntityGraphMapper {
}
if (isNewElementsNull) {
setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), null);
setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), null, cardinality);
} else {
setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated);
setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated, cardinality);
}
if (LOG.isDebugEnabled()) {
......@@ -1392,7 +1398,7 @@ public class EntityGraphMapper {
return (List)vertex.getListProperty(vertexPropertyName, AtlasEdge.class);
}
else {
return (List)vertex.getListProperty(vertexPropertyName);
return (List) vertex.getPropertyValues(vertexPropertyName, List.class);
}
}
......@@ -1436,9 +1442,17 @@ public class EntityGraphMapper {
return Collections.emptyList();
}
private void setArrayElementsProperty(AtlasType elementType, boolean isSoftReference, AtlasVertex vertex, String vertexPropertyName, List<Object> values) {
private void setArrayElementsProperty(AtlasType elementType, boolean isSoftReference, AtlasVertex vertex, String propertyName, List<Object> values, Cardinality cardinality) {
if (!isReference(elementType) || isSoftReference) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, vertexPropertyName, values);
//remove existing array values before setting new values
vertex.removeProperty(propertyName);
if (cardinality == LIST) {
values.forEach(value -> addToListProperty(vertex, propertyName, value));
} else {
values.forEach(value -> addToSetProperty(vertex, propertyName, value));
}
}
}
......@@ -1523,7 +1537,7 @@ public class EntityGraphMapper {
LOG.debug("Adding classification [{}] to [{}] using edge label: [{}]", classificationName, entityType.getTypeName(), getTraitLabel(classificationName));
}
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
AtlasGraphUtilsV2.addToEncodedSetProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
// add a new AtlasVertex for the struct or trait instance
AtlasVertex classificationVertex = createClassificationVertex(classification);
......@@ -1924,7 +1938,7 @@ public class EntityGraphMapper {
entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
for (String traitName : traitNames) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
AtlasGraphUtilsV2.addToEncodedSetProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java
......@@ -268,7 +268,7 @@ public final class EntityStateChecker {
entityVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
for (String classificationName : traitVertexNames) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, classificationName);
AtlasGraphUtilsV2.addToEncodedSetProperty(entityVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, classificationName);
}
}
......@@ -284,7 +284,7 @@ public final class EntityStateChecker {
entityVertex.removeProperty(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
for (String classificationName : propagatedTraitVertexNames) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
AtlasGraphUtilsV2.addToEncodedListProperty(entityVertex, Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment