Skip to content

A
atlas
Commit 15388229 by Sarath Subramanian
Options

ATLAS-2775: Enhance basic search to search on entities with any classifications…

ATLAS-2775: Enhance basic search to search on entities with any classifications or no classifications
parent 74dc605d
Showing with 73 additions and 26 deletions
intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
......@@ -55,6 +55,10 @@ public class SearchParameters implements Serializable {
private FilterCriteria tagFilters;
private Set<String> attributes;
public static final String WILDCARD_CLASSIFICATIONS = "*";
public static final String ALL_CLASSIFICATIONS = "_CLASSIFIED";
public static final String NO_CLASSIFICATIONS = "_NOT_CLASSIFIED";
/**
* @return The type of query
*/
......
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
......@@ -50,6 +50,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
......@@ -88,7 +94,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
// for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
boolean useIndexSearch = classificationType != SearchContext.MATCH_ALL_CLASSIFICATION &&
boolean useIndexSearch = classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION &&
classificationType != MATCH_ALL_CLASSIFIED &&
classificationType != MATCH_ALL_NOT_CLASSIFIED &&
typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS &&
CollectionUtils.isEmpty(graphAttributes) &&
canApplyIndexFilter(classificationType, filterCriteria, false);
......@@ -149,25 +157,32 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
}
} else {
tagGraphQueryWithAttributes = null;
List<AtlasGraphQuery> orConditions = new LinkedList<>();
tagGraphQueryWithAttributes = null;
List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType != SearchContext.MATCH_ALL_CLASSIFICATION) {
orConditions.add(graph.query().createChildQuery().in(Constants.TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
orConditions.add(graph.query().createChildQuery().in(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
orConditions.add(graph.query().createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.orPredicate(
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class));
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null).has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.andPredicate(
SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else {
orConditions.add(graph.query().createChildQuery().has(Constants.TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
orConditions.add(graph.query().createChildQuery().has(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
orConditions.add(graph.query().createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
orConditions.add(graph.query().createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.orPredicate(
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class));
}
if (context.getSearchParameters().getExcludeDeletedEntities()) {
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
......@@ -40,6 +40,12 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
public class EntitySearchProcessor extends SearchProcessor {
......@@ -93,12 +99,15 @@ public class EntitySearchProcessor extends SearchProcessor {
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
final Predicate traitPredicate;
if (classificationType == SearchContext.MATCH_ALL_CLASSIFICATION) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class));
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class));
}
processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
......@@ -154,12 +163,15 @@ public class EntitySearchProcessor extends SearchProcessor {
if (filterClassification) {
List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType == SearchContext.MATCH_ALL_CLASSIFICATION) {
orConditions.add(query.createChildQuery().has(Constants.TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
orConditions.add(query.createChildQuery().has(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
orConditions.add(query.createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null)
.has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
} else {
orConditions.add(query.createChildQuery().in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
orConditions.add(query.createChildQuery().in(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
orConditions.add(query.createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
orConditions.add(query.createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
}
query.or(orConditions);
......
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
......@@ -31,6 +31,10 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
public class FullTextSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(FullTextSearchProcessor.class);
......@@ -61,7 +65,9 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if (context.getClassificationType() != null && context.getClassificationType() != SearchContext.MATCH_ALL_CLASSIFICATION) {
if (context.getClassificationType() != null && context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION &&
context.getClassificationType() != MATCH_ALL_CLASSIFIED &&
context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) {
String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
......
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
......@@ -45,6 +45,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
/*
* Search context captures elements required for performing a basic search
* For every search request the search context will determine the execution sequence of the search processor(s) and the
......@@ -61,7 +65,9 @@ public class SearchContext {
private SearchProcessor searchProcessor;
private boolean terminateSearch = false;
public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef("*"));
public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
String classificationName = searchParameters.getClassification();
......@@ -199,8 +205,12 @@ public class SearchContext {
private AtlasClassificationType getClassificationType(String classificationName) {
AtlasClassificationType ret;
if (StringUtils.equals(classificationName, MATCH_ALL_CLASSIFICATION.getTypeName())) {
ret = MATCH_ALL_CLASSIFICATION;
if (StringUtils.equals(classificationName, MATCH_ALL_WILDCARD_CLASSIFICATION.getTypeName())) {
ret = MATCH_ALL_WILDCARD_CLASSIFICATION;
} else if (StringUtils.equals(classificationName, MATCH_ALL_CLASSIFIED.getTypeName())) {
ret = MATCH_ALL_CLASSIFIED;
} else if (StringUtils.equals(classificationName, MATCH_ALL_NOT_CLASSIFIED.getTypeName())) {
ret = MATCH_ALL_NOT_CLASSIFIED;
} else {
ret = typeRegistry.getClassificationTypeByName(classificationName);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment