Skip to content

A
atlas
Commit df85aa7a by Le Ma Committed by Sarath Subramanian
Options

ATLAS-3457: Support Multi-Classification search without attributes

Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org>
parent d1f34186
Showing with 62 additions and 16 deletions
intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
......@@ -62,6 +62,7 @@ public class SearchParameters implements Serializable {
public static final String ALL_CLASSIFICATIONS = "_CLASSIFIED";
public static final String NO_CLASSIFICATIONS = "_NOT_CLASSIFIED";
public static final String ALL_ENTITY_TYPES = "_ALL_ENTITY_TYPES";
public static final String ALL_CLASSIFICATION_TYPES = "_ALL_CLASSIFICATION_TYPES";
/**
* @return The type of query
......
intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
......@@ -93,6 +93,8 @@ public class AtlasClassificationType extends AtlasStructType {
public AtlasClassificationDef getClassificationDef() { return classificationDef; }
public static AtlasClassificationType getClassificationRoot() {return CLASSIFICATION_ROOT; }
@Override
void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
super.resolveReferences(typeRegistry);
......@@ -284,6 +286,10 @@ public class AtlasClassificationType extends AtlasStructType {
return StringUtils.isNotEmpty(classificationName) && allSuperTypes.contains(classificationName);
}
public boolean hasAttribute(String attrName) {
return allAttributes.containsKey(attrName);
}
/**
* List of all the entity type names that are valid for this classification type.
*
......@@ -506,7 +512,7 @@ public class AtlasClassificationType extends AtlasStructType {
add(new AtlasAttributeDef(STATE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
}};
AtlasClassificationDef classificationDef = new AtlasClassificationDef(CLASSIFICATION_ROOT_NAME, "", "", attributeDefs);
AtlasClassificationDef classificationDef = new AtlasClassificationDef(CLASSIFICATION_ROOT_NAME, "Root classification for system attributes", "1.0", attributeDefs);
return new AtlasClassificationType(classificationDef);
}
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -17,6 +17,7 @@
*/
package org.apache.atlas.type;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_INT;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
import static org.apache.atlas.type.Constants.*;
......@@ -638,7 +639,7 @@ public class AtlasEntityType extends AtlasStructType {
add(new AtlasAttributeDef(CLASSIFICATION_TEXT_KEY, ATLAS_TYPE_STRING, false, true));
add(new AtlasAttributeDef(CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
add(new AtlasAttributeDef(PROPAGATED_CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
add(new AtlasAttributeDef(IS_INCOMPLETE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
add(new AtlasAttributeDef(IS_INCOMPLETE_PROPERTY_KEY, ATLAS_TYPE_INT, false, true));
add(new AtlasAttributeDef(LABELS_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
add(new AtlasAttributeDef(CUSTOM_ATTRIBUTES_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
}};
......
intg/src/main/java/org/apache/atlas/type/Constants.java
......@@ -32,7 +32,6 @@ public final class Constants {
public static final String TYPE_NAME_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typeName");
public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
public static final String SUPER_TYPES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "superTypeNames");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
public static final String TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "timestamp");
public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp");
......@@ -44,8 +43,6 @@ public final class Constants {
public static final String HISTORICAL_GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids");
public static final String LABELS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "labels");
public static final String CUSTOM_ATTRIBUTES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "customAttributes");
public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames");
public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames");
......@@ -53,4 +50,3 @@ public final class Constants {
private Constants() {}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
......@@ -164,11 +164,11 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
Predicate typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
Predicate typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(typeNamePredicate, attributePredicate);
inMemoryPredicate = typeNamePredicate == null ? attributePredicate : PredicateUtils.andPredicate(typeNamePredicate, attributePredicate);
} else {
inMemoryPredicate = typeNamePredicate;
}
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
......@@ -54,7 +54,6 @@ import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOpe
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC;
import static org.apache.atlas.type.AtlasEntityType.ENTITY_ROOT;
public class EntitySearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class);
......
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
......@@ -45,8 +45,9 @@ import org.slf4j.LoggerFactory;
import java.util.*;
import static org.apache.atlas.discovery.SearchProcessor.ALL_ENTITY_TYPE_QUERY;
import static org.apache.atlas.discovery.SearchProcessor.ALL_TYPE_QUERY;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
......@@ -77,8 +78,10 @@ public class SearchContext {
public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot();
public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot();
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
this.classificationName = searchParameters.getClassification();
this.searchParameters = searchParameters;
......@@ -116,8 +119,13 @@ public class SearchContext {
validateAttributes(classificationType, searchParameters.getTagFilters());
if (classificationType != null && !isBuiltInClassificationType()) {
if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
classificationTypeAndSubTypes = Collections.singleton(ALL_TYPE_QUERY);
classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
} else {
classificationTypeAndSubTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName());
classificationTypeAndSubTypesQryStr = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypesQryStr() : classificationType.getTypeQryStr();
}
} else {
classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = "";
......@@ -125,8 +133,8 @@ public class SearchContext {
if (entityType != null) {
if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
typeAndSubTypes = Collections.singleton(ALL_ENTITY_TYPE_QUERY);
typeAndSubTypesQryStr = ALL_ENTITY_TYPE_QUERY;
typeAndSubTypes = Collections.singleton(ALL_TYPE_QUERY);
typeAndSubTypesQryStr = ALL_TYPE_QUERY;
} else {
typeAndSubTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
typeAndSubTypesQryStr = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypesQryStr() : entityType.getTypeQryStr();
......@@ -304,6 +312,8 @@ public class SearchContext {
ret = MATCH_ALL_CLASSIFIED;
} else if (StringUtils.equals(classificationName, MATCH_ALL_NOT_CLASSIFIED.getTypeName())) {
ret = MATCH_ALL_NOT_CLASSIFIED;
} else if (StringUtils.equals(classificationName, ALL_CLASSIFICATION_TYPES)){
ret = MATCH_ALL_CLASSIFICATION_TYPES;
} else {
ret = typeRegistry.getClassificationTypeByName(classificationName);
}
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
......@@ -45,6 +45,10 @@ import java.math.BigInteger;
import java.util.*;
import java.util.regex.Pattern;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.util.SearchPredicateUtil.*;
public abstract class SearchProcessor {
......@@ -62,7 +66,7 @@ public abstract class SearchProcessor {
public static final String SPACE_STRING = " ";
public static final String BRACE_OPEN_STR = "(";
public static final String BRACE_CLOSE_STR = ")";
public static final String ALL_ENTITY_TYPE_QUERY = "[* TO *]";
public static final String ALL_TYPE_QUERY = "[* TO *]";
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final Map<SearchParameters.Operator, VertexAttributePredicateGenerator> OPERATOR_PREDICATE_MAP = new HashMap<>();
......@@ -136,6 +140,21 @@ public abstract class SearchProcessor {
return context.getEntityType() == SearchContext.MATCH_ALL_ENTITY_TYPES;
}
protected boolean isClassificationRootType() {
return context.getClassificationType() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
}
protected boolean isSystemAttribute(String attrName) {
return AtlasEntityType.getEntityRoot().hasAttribute(attrName) || AtlasClassificationType.getClassificationRoot().hasAttribute(attrName);
}
protected boolean isPipeSeparatedSystemAttribute(String attrName) {
return StringUtils.equals(attrName, CLASSIFICATION_NAMES_KEY) ||
StringUtils.equals(attrName, PROPAGATED_CLASSIFICATION_NAMES_KEY) ||
StringUtils.equals(attrName, LABELS_PROPERTY_KEY) ||
StringUtils.equals(attrName, CUSTOM_ATTRIBUTES_PROPERTY_KEY);
}
protected int collectResultVertices(final List<AtlasVertex> ret, final int startIdx, final int limit, int resultIdx, final List<AtlasVertex> entityVertices) {
for (AtlasVertex entityVertex : entityVertices) {
resultIdx++;
......@@ -185,7 +204,7 @@ public abstract class SearchProcessor {
graphFiltered.add(attributeName);
}
if (structType instanceof AtlasEntityType) {
if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) {
// Capture the entity attributes
context.getEntityAttributes().add(attributeName);
}
......@@ -438,7 +457,7 @@ public abstract class SearchProcessor {
return PredicateUtils.anyPredicate(predicates);
}
}
} else if (indexAttributes.contains(criteria.getAttributeName())){
} else if (indexAttributes.contains(criteria.getAttributeName()) && !isPipeSeparatedSystemAttribute(criteria.getAttributeName())){
return toInMemoryPredicate(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
......@@ -46,6 +46,7 @@ import java.util.List;
import java.util.Set;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer.getTypesToCreate;
import static org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer.getTypesToUpdate;
......@@ -217,8 +218,12 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
AtlasClassificationDef ret = typeRegistry.getClassificationDefByName(name);
if (ret == null) {
ret = StringUtils.equalsIgnoreCase(name, ALL_CLASSIFICATION_TYPES) ? AtlasClassificationType.getClassificationRoot().getClassificationDef() : null;
if (ret == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name);
}
}
return ret;
}
......
repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
......@@ -32,6 +32,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.utils.TestResourceFileUtils;
import org.slf4j.Logger;
......@@ -701,4 +702,12 @@ public class AtlasTypeDefGraphStoreTest {
assertNotNull(entityDefByName);
assertEquals(entityDefByName, AtlasEntityType.getEntityRoot().getEntityDef());
}
@Test
public void testGetOnAllClassificationTypes() throws AtlasBaseException {
AtlasClassificationDef classificationTypeDef = typeDefStore.getClassificationDefByName("_ALL_CLASSIFICATION_TYPES");
assertNotNull(classificationTypeDef);
assertEquals(classificationTypeDef, AtlasClassificationType.getClassificationRoot().getClassificationDef());
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment