Skip to content

A
atlas
Commit 15388229 by Sarath Subramanian
Options

ATLAS-2775: Enhance basic search to search on entities with any classifications…

ATLAS-2775: Enhance basic search to search on entities with any classifications or no classifications
parent 74dc605d
Showing with 73 additions and 26 deletions
intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
...@@ -55,6 +55,10 @@ public class SearchParameters implements Serializable { ...@@ -55,6 +55,10 @@ public class SearchParameters implements Serializable {
private FilterCriteria tagFilters; private FilterCriteria tagFilters;
private Set<String> attributes; private Set<String> attributes;
public static final String WILDCARD_CLASSIFICATIONS = "*";
public static final String ALL_CLASSIFICATIONS = "_CLASSIFIED";
public static final String NO_CLASSIFICATIONS = "_NOT_CLASSIFIED";
/** /**
* @return The type of query * @return The type of query
*/ */
......
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
...@@ -50,6 +50,12 @@ import java.util.List; ...@@ -50,6 +50,12 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
...@@ -88,7 +94,9 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -88,7 +94,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes); processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
// for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query // for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
boolean useIndexSearch = classificationType != SearchContext.MATCH_ALL_CLASSIFICATION && boolean useIndexSearch = classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION &&
classificationType != MATCH_ALL_CLASSIFIED &&
classificationType != MATCH_ALL_NOT_CLASSIFIED &&
typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS &&
CollectionUtils.isEmpty(graphAttributes) && CollectionUtils.isEmpty(graphAttributes) &&
canApplyIndexFilter(classificationType, filterCriteria, false); canApplyIndexFilter(classificationType, filterCriteria, false);
...@@ -149,25 +157,32 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -149,25 +157,32 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery); LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
} }
} else { } else {
tagGraphQueryWithAttributes = null; tagGraphQueryWithAttributes = null;
List<AtlasGraphQuery> orConditions = new LinkedList<>(); List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType != SearchContext.MATCH_ALL_CLASSIFICATION) { if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
orConditions.add(graph.query().createChildQuery().in(Constants.TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes)); orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
orConditions.add(graph.query().createChildQuery().in(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes)); orConditions.add(graph.query().createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
entityGraphQueryTraitNames = graph.query().or(orConditions); entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.orPredicate( entityPredicateTraitNames = PredicateUtils.orPredicate(
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class), SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class)); SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null).has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.andPredicate(
SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else { } else {
orConditions.add(graph.query().createChildQuery().has(Constants.TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); orConditions.add(graph.query().createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
orConditions.add(graph.query().createChildQuery().has(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); orConditions.add(graph.query().createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
entityGraphQueryTraitNames = graph.query().or(orConditions); entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.orPredicate( entityPredicateTraitNames = PredicateUtils.orPredicate(
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, null, List.class), SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class));
} }
if (context.getSearchParameters().getExcludeDeletedEntities()) { if (context.getSearchParameters().getExcludeDeletedEntities()) {
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
...@@ -40,6 +40,12 @@ import java.util.LinkedList; ...@@ -40,6 +40,12 @@ import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
public class EntitySearchProcessor extends SearchProcessor { public class EntitySearchProcessor extends SearchProcessor {
...@@ -93,12 +99,15 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -93,12 +99,15 @@ public class EntitySearchProcessor extends SearchProcessor {
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class); .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
final Predicate traitPredicate; final Predicate traitPredicate;
if (classificationType == SearchContext.MATCH_ALL_CLASSIFICATION) { if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, null, List.class), traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getIsNullPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else { } else {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class), traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class)); SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class));
} }
processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes); processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
...@@ -154,12 +163,15 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -154,12 +163,15 @@ public class EntitySearchProcessor extends SearchProcessor {
if (filterClassification) { if (filterClassification) {
List<AtlasGraphQuery> orConditions = new LinkedList<>(); List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType == SearchContext.MATCH_ALL_CLASSIFICATION) { if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
orConditions.add(query.createChildQuery().has(Constants.TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
orConditions.add(query.createChildQuery().has(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null)); orConditions.add(query.createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null)
.has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
} else { } else {
orConditions.add(query.createChildQuery().in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes)); orConditions.add(query.createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
orConditions.add(query.createChildQuery().in(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes)); orConditions.add(query.createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
} }
query.or(orConditions); query.or(orConditions);
......
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
...@@ -31,6 +31,10 @@ import java.util.ArrayList; ...@@ -31,6 +31,10 @@ import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
public class FullTextSearchProcessor extends SearchProcessor { public class FullTextSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(FullTextSearchProcessor.class); private static final Logger LOG = LoggerFactory.getLogger(FullTextSearchProcessor.class);
...@@ -61,7 +65,9 @@ public class FullTextSearchProcessor extends SearchProcessor { ...@@ -61,7 +65,9 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes classification criteria, adding a filter here can help avoid unnecessary // if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if (context.getClassificationType() != null && context.getClassificationType() != SearchContext.MATCH_ALL_CLASSIFICATION) { if (context.getClassificationType() != null && context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION &&
context.getClassificationType() != MATCH_ALL_CLASSIFIED &&
context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) {
String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr(); String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) { if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
......
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
...@@ -45,6 +45,10 @@ import java.util.Iterator; ...@@ -45,6 +45,10 @@ import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
/* /*
* Search context captures elements required for performing a basic search * Search context captures elements required for performing a basic search
* For every search request the search context will determine the execution sequence of the search processor(s) and the * For every search request the search context will determine the execution sequence of the search processor(s) and the
...@@ -61,7 +65,9 @@ public class SearchContext { ...@@ -61,7 +65,9 @@ public class SearchContext {
private SearchProcessor searchProcessor; private SearchProcessor searchProcessor;
private boolean terminateSearch = false; private boolean terminateSearch = false;
public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef("*")); public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException { public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
String classificationName = searchParameters.getClassification(); String classificationName = searchParameters.getClassification();
...@@ -199,8 +205,12 @@ public class SearchContext { ...@@ -199,8 +205,12 @@ public class SearchContext {
private AtlasClassificationType getClassificationType(String classificationName) { private AtlasClassificationType getClassificationType(String classificationName) {
AtlasClassificationType ret; AtlasClassificationType ret;
if (StringUtils.equals(classificationName, MATCH_ALL_CLASSIFICATION.getTypeName())) { if (StringUtils.equals(classificationName, MATCH_ALL_WILDCARD_CLASSIFICATION.getTypeName())) {
ret = MATCH_ALL_CLASSIFICATION; ret = MATCH_ALL_WILDCARD_CLASSIFICATION;
} else if (StringUtils.equals(classificationName, MATCH_ALL_CLASSIFIED.getTypeName())) {
ret = MATCH_ALL_CLASSIFIED;
} else if (StringUtils.equals(classificationName, MATCH_ALL_NOT_CLASSIFIED.getTypeName())) {
ret = MATCH_ALL_NOT_CLASSIFIED;
} else { } else {
ret = typeRegistry.getClassificationTypeByName(classificationName); ret = typeRegistry.getClassificationTypeByName(classificationName);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment