Skip to content

A
atlas
Commit 8b5cb9d9 by Pinal Shah Committed by Sarath Subramanian
Options

ATLAS-3838: Support multiple tag/classification in basic/quick search API

ATLAS-3652: Quick Search: API requirement for GET request on multiple entity types Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org> both JIRA's addressed in the same commit.
parent e691a32c
Showing with 640 additions and 267 deletions
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
...@@ -27,7 +27,7 @@ import java.util.Set; ...@@ -27,7 +27,7 @@ import java.util.Set;
public class AggregationContext { public class AggregationContext {
private final String queryString; private final String queryString;
private final FilterCriteria filterCriteria; private final FilterCriteria filterCriteria;
private final AtlasEntityType searchForEntityType; private final Set<AtlasEntityType> searchForEntityTypes;
private final Set<String> aggregationFieldNames; private final Set<String> aggregationFieldNames;
private final Set<AtlasAttribute> aggregationAttributes; private final Set<AtlasAttribute> aggregationAttributes;
private final Map<String, String> indexFieldNameCache; private final Map<String, String> indexFieldNameCache;
...@@ -43,7 +43,7 @@ public class AggregationContext { ...@@ -43,7 +43,7 @@ public class AggregationContext {
*/ */
public AggregationContext(String queryString, public AggregationContext(String queryString,
FilterCriteria filterCriteria, FilterCriteria filterCriteria,
AtlasEntityType searchForEntityType, Set<AtlasEntityType> searchForEntityType,
Set<String> aggregationFieldNames, Set<String> aggregationFieldNames,
Set<AtlasAttribute> aggregationAttributes, Set<AtlasAttribute> aggregationAttributes,
Map<String, String> indexFieldNameCache, Map<String, String> indexFieldNameCache,
...@@ -51,7 +51,7 @@ public class AggregationContext { ...@@ -51,7 +51,7 @@ public class AggregationContext {
boolean includeSubTypes) { boolean includeSubTypes) {
this.queryString = queryString; this.queryString = queryString;
this.filterCriteria = filterCriteria; this.filterCriteria = filterCriteria;
this.searchForEntityType = searchForEntityType; this.searchForEntityTypes = searchForEntityType;
this.aggregationFieldNames = aggregationFieldNames; this.aggregationFieldNames = aggregationFieldNames;
this.aggregationAttributes = aggregationAttributes; this.aggregationAttributes = aggregationAttributes;
this.indexFieldNameCache = indexFieldNameCache; this.indexFieldNameCache = indexFieldNameCache;
...@@ -67,8 +67,8 @@ public class AggregationContext { ...@@ -67,8 +67,8 @@ public class AggregationContext {
return filterCriteria; return filterCriteria;
} }
public AtlasEntityType getSearchForEntityType() { public Set<AtlasEntityType> getSearchForEntityTypes() {
return searchForEntityType; return searchForEntityTypes;
} }
public Set<String> getAggregationFieldNames() { public Set<String> getAggregationFieldNames() {
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
...@@ -175,7 +175,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient { ...@@ -175,7 +175,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
Map<String, String> indexFieldName2PropertyKeyNameMap = new HashMap<>(); Map<String, String> indexFieldName2PropertyKeyNameMap = new HashMap<>();
AtlasSolrQueryBuilder solrQueryBuilder = new AtlasSolrQueryBuilder(); AtlasSolrQueryBuilder solrQueryBuilder = new AtlasSolrQueryBuilder();
solrQueryBuilder.withEntityType(aggregationContext.getSearchForEntityType()) solrQueryBuilder.withEntityTypes(aggregationContext.getSearchForEntityTypes())
.withQueryString(aggregationContext.getQueryString()) .withQueryString(aggregationContext.getQueryString())
.withCriteria(aggregationContext.getFilterCriteria()) .withCriteria(aggregationContext.getFilterCriteria())
.withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities()) .withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities())
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
...@@ -29,17 +29,14 @@ import org.apache.commons.lang.StringUtils; ...@@ -29,17 +29,14 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.Iterator; import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
public class AtlasSolrQueryBuilder { public class AtlasSolrQueryBuilder {
private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class); private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class);
private AtlasEntityType entityType; private Set<AtlasEntityType> entityTypes;
private String queryString; private String queryString;
private FilterCriteria criteria; private FilterCriteria criteria;
private boolean excludeDeletedEntities; private boolean excludeDeletedEntities;
...@@ -52,8 +49,8 @@ public class AtlasSolrQueryBuilder { ...@@ -52,8 +49,8 @@ public class AtlasSolrQueryBuilder {
public AtlasSolrQueryBuilder() { public AtlasSolrQueryBuilder() {
} }
public AtlasSolrQueryBuilder withEntityType(AtlasEntityType searchForEntityType) { public AtlasSolrQueryBuilder withEntityTypes(Set<AtlasEntityType> searchForEntityTypes) {
this.entityType = searchForEntityType; this.entityTypes = searchForEntityTypes;
return this; return this;
} }
...@@ -112,7 +109,7 @@ public class AtlasSolrQueryBuilder { ...@@ -112,7 +109,7 @@ public class AtlasSolrQueryBuilder {
isAndNeeded = true; isAndNeeded = true;
} }
if (entityType != null) { if (CollectionUtils.isNotEmpty(entityTypes)) {
if (isAndNeeded) { if (isAndNeeded) {
queryBuilder.append(" AND "); queryBuilder.append(" AND ");
} }
...@@ -140,29 +137,25 @@ public class AtlasSolrQueryBuilder { ...@@ -140,29 +137,25 @@ public class AtlasSolrQueryBuilder {
} }
private void buildForEntityType(StringBuilder queryBuilder) { private void buildForEntityType(StringBuilder queryBuilder) {
if (LOG.isDebugEnabled()) {
LOG.debug("Search is being done for entities of type {}", entityType.getTypeName());
}
String typeIndexFieldName = indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY); String typeIndexFieldName = indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY);
queryBuilder.append(" +") queryBuilder.append(" +")
.append(typeIndexFieldName) .append(typeIndexFieldName)
.append(":(") .append(":(");
.append(entityType.getTypeName())
.append(" ");
if (includeSubtypes) { Set<String> typesToSearch = new HashSet<>();
Set<String> allSubTypes = entityType.getAllSubTypes(); for (AtlasEntityType type : entityTypes) {
if(allSubTypes.size() != 0 ) { if (includeSubtypes) {
for(String subTypeName: allSubTypes) { typesToSearch.addAll(type.getTypeAndAllSubTypes());
queryBuilder.append(subTypeName).append(" "); } else {
} typesToSearch.add(type.getTypeName());
} }
} }
queryBuilder.append(" ) "); queryBuilder.append(StringUtils.join(typesToSearch, " ")).append(" ) ");
} }
private void dropDeletedEntities(StringBuilder queryBuilder) throws AtlasBaseException { private void dropDeletedEntities(StringBuilder queryBuilder) throws AtlasBaseException {
...@@ -173,9 +166,8 @@ public class AtlasSolrQueryBuilder { ...@@ -173,9 +166,8 @@ public class AtlasSolrQueryBuilder {
String indexFieldName = indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY); String indexFieldName = indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY);
if (indexFieldName == null) { if (indexFieldName == null) {
String msg = String.format("There is no index field name defined for attribute '%s' for entity '%s'", String msg = String.format("There is no index field name defined for attribute '%s'",
Constants.STATE_PROPERTY_KEY, Constants.STATE_PROPERTY_KEY);
entityType.getTypeName());
LOG.error(msg); LOG.error(msg);
...@@ -187,10 +179,46 @@ public class AtlasSolrQueryBuilder { ...@@ -187,10 +179,46 @@ public class AtlasSolrQueryBuilder {
private AtlasSolrQueryBuilder withCriteria(StringBuilder queryBuilder, FilterCriteria criteria) throws AtlasBaseException { private AtlasSolrQueryBuilder withCriteria(StringBuilder queryBuilder, FilterCriteria criteria) throws AtlasBaseException {
List<FilterCriteria> criterion = criteria.getCriterion(); List<FilterCriteria> criterion = criteria.getCriterion();
Set<String> indexAttributes = new HashSet<>();
if (StringUtils.isNotEmpty(criteria.getAttributeName()) && CollectionUtils.isEmpty(criterion)) { // no child criterion
String attributeName = criteria.getAttributeName();
String attributeValue = criteria.getAttributeValue();
Operator operator = criteria.getOperator();
ArrayList<StringBuilder> orExpQuery = new ArrayList<>();
for (AtlasEntityType type : entityTypes) {
String indexAttributeName = getIndexAttributeName(type, attributeName);
//check to remove duplicate attribute query (for eg. name)
if (!indexAttributes.contains(indexAttributeName)) {
StringBuilder sb = new StringBuilder();
if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works
operator = Operator.EQ;
attributeValue = getIndexQueryAttributeValue(attributeValue);
}
withPropertyCondition(sb, indexAttributeName, operator, attributeValue);
indexAttributes.add(indexAttributeName);
orExpQuery.add(sb);
}
}
if(criterion == null || CollectionUtils.isEmpty(criteria.getCriterion())) { // no child criterion if (CollectionUtils.isNotEmpty(orExpQuery)) {
withPropertyCondition(queryBuilder, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue()); if (orExpQuery.size() > 1) {
String orExpStr = StringUtils.join(orExpQuery, FilterCriteria.Condition.OR.name());
queryBuilder.append(" ( ").append(orExpStr).append(" ) ");
} else { } else {
queryBuilder.append(orExpQuery.iterator().next());
}
}
} else if (CollectionUtils.isNotEmpty(criterion)) {
beginCriteria(queryBuilder); beginCriteria(queryBuilder);
for (Iterator<FilterCriteria> iterator = criterion.iterator(); iterator.hasNext(); ) { for (Iterator<FilterCriteria> iterator = criterion.iterator(); iterator.hasNext(); ) {
...@@ -209,40 +237,12 @@ public class AtlasSolrQueryBuilder { ...@@ -209,40 +237,12 @@ public class AtlasSolrQueryBuilder {
return this; return this;
} }
private void withPropertyCondition(StringBuilder queryBuilder, String attributeName, Operator operator, String attributeValue) throws AtlasBaseException { private void withPropertyCondition(StringBuilder queryBuilder, String indexFieldName, Operator operator, String attributeValue) throws AtlasBaseException {
if (StringUtils.isNotEmpty(attributeName) && operator != null) { if (StringUtils.isNotEmpty(indexFieldName) && operator != null) {
if (attributeValue != null) { if (attributeValue != null) {
attributeValue = attributeValue.trim(); attributeValue = attributeValue.trim();
} }
if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works
operator = Operator.EQ;
attributeValue = getIndexQueryAttributeValue(attributeValue);
}
AtlasAttribute attribute = entityType.getAttribute(attributeName);
if (attribute == null) {
String msg = String.format("Received unknown attribute '%s' for type '%s'.", attributeName, entityType.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
String indexFieldName = attribute.getIndexFieldName();
if (indexFieldName == null) {
String msg = String.format("Received non-index attribute %s for type %s.", attributeName, entityType.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
beginCriteria(queryBuilder); beginCriteria(queryBuilder);
switch (operator) { switch (operator) {
...@@ -308,6 +308,29 @@ public class AtlasSolrQueryBuilder { ...@@ -308,6 +308,29 @@ public class AtlasSolrQueryBuilder {
return attributeValue; return attributeValue;
} }
private String getIndexAttributeName(AtlasEntityType type, String attrName) throws AtlasBaseException {
AtlasAttribute ret = type.getAttribute(attrName);
if (ret == null) {
String msg = String.format("Received unknown attribute '%s' for type '%s'.", attrName, type.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
String indexFieldName = ret.getIndexFieldName();
if (indexFieldName == null) {
String msg = String.format("Received non-index attribute %s for type %s.", attrName, type.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
return indexFieldName;
}
private void beginCriteria(StringBuilder queryBuilder) { private void beginCriteria(StringBuilder queryBuilder) {
queryBuilder.append("( "); queryBuilder.append("( ");
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
...@@ -33,8 +33,10 @@ import org.testng.annotations.Test; ...@@ -33,8 +33,10 @@ import org.testng.annotations.Test;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
...@@ -45,6 +47,9 @@ public class AtlasSolrQueryBuilderTest { ...@@ -45,6 +47,9 @@ public class AtlasSolrQueryBuilderTest {
private AtlasEntityType hiveTableEntityTypeMock; private AtlasEntityType hiveTableEntityTypeMock;
@Mock @Mock
private AtlasEntityType hiveTableEntityTypeMock2;
@Mock
private AtlasStructType.AtlasAttribute nameAttributeMock; private AtlasStructType.AtlasAttribute nameAttributeMock;
@Mock @Mock
...@@ -95,6 +100,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -95,6 +100,7 @@ public class AtlasSolrQueryBuilderTest {
when(hiveTableEntityTypeMock.getTypeName()).thenReturn("hive_table"); when(hiveTableEntityTypeMock.getTypeName()).thenReturn("hive_table");
when(hiveTableEntityTypeMock2.getTypeName()).thenReturn("hive_db");
when(nameAttributeMock.getIndexFieldName()).thenReturn("name_index"); when(nameAttributeMock.getIndexFieldName()).thenReturn("name_index");
when(commentAttributeMock.getIndexFieldName()).thenReturn("comment_index"); when(commentAttributeMock.getIndexFieldName()).thenReturn("comment_index");
...@@ -217,6 +223,15 @@ public class AtlasSolrQueryBuilderTest { ...@@ -217,6 +223,15 @@ public class AtlasSolrQueryBuilderTest {
Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )"); Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )");
} }
@Test
public void testGenerateSolrQueryString2TypeNames() throws IOException, AtlasBaseException {
final String fileName = "src/test/resources/searchparameters2Types.json";
AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
processSearchParametersForMultipleTypeNames(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table hive_db ) ");
}
...@@ -243,8 +258,23 @@ public class AtlasSolrQueryBuilderTest { ...@@ -243,8 +258,23 @@ public class AtlasSolrQueryBuilderTest {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class); SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class);
Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
underTest.withEntityTypes(hiveTableEntityTypeMocks)
.withQueryString(searchParameters.getQuery())
.withCriteria(searchParameters.getEntityFilters())
.withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
.withCommonIndexFieldNames(indexFieldNamesMap);
}
private void processSearchParametersForMultipleTypeNames(String fileName, AtlasSolrQueryBuilder underTest) throws IOException, AtlasBaseException {
ObjectMapper mapper = new ObjectMapper();
SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class);
underTest.withEntityType(hiveTableEntityTypeMock) Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock2);
underTest.withEntityTypes(hiveTableEntityTypeMocks)
.withQueryString(searchParameters.getQuery()) .withQueryString(searchParameters.getQuery())
.withCriteria(searchParameters.getEntityFilters()) .withCriteria(searchParameters.getEntityFilters())
.withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities()) .withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
......
graphdb/janus/src/test/resources/searchparameters2Types.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t",
"limit":25,
"offset":0,
"typeName":"hive_table,hive_db",
"classification":null,
"termName":null
}
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
...@@ -67,17 +67,16 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -67,17 +67,16 @@ public class ClassificationSearchProcessor extends SearchProcessor {
public ClassificationSearchProcessor(SearchContext context) { public ClassificationSearchProcessor(SearchContext context) {
super(context); super(context);
final AtlasClassificationType classificationType = context.getClassificationType();
final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters(); final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
final Set<String> indexAttributes = new HashSet<>(); final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>(); final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>(); final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getClassificationTypes(); final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr(); final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
final boolean isBuiltInType = context.isBuiltInClassificationType();
final boolean isWildcardSearch = context.isWildCardSearch(); final boolean isWildcardSearch = context.isWildCardSearch();
final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes); processSearchAttributes(classificationTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes);
/* for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query /* for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
There are four cases in the classification type : There are four cases in the classification type :
...@@ -87,19 +86,25 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -87,19 +86,25 @@ public class ClassificationSearchProcessor extends SearchProcessor {
4. classification is not present in the search parameter 4. classification is not present in the search parameter
each of above cases with either has empty/or not tagFilters each of above cases with either has empty/or not tagFilters
*/ */
final boolean useIndexSearchForEntity = (classificationType != null || isWildcardSearch) && final boolean useIndexSearchForEntity = (CollectionUtils.isNotEmpty(classificationTypes) || isWildcardSearch) &&
!context.hasAttributeFilter(filterCriteria) && !context.hasAttributeFilter(filterCriteria) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS); (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
/* If classification's attributes can be applied index filter, we can use direct index /* If classification's attributes can be applied index filter, we can use direct index
* to query classification index as well. * to query classification index as well.
*/ */
final boolean useIndexSearchForClassification = (!isBuiltInType && !isWildcardSearch) && final boolean useIndexSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
!isWildcardSearch) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) && (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
CollectionUtils.isNotEmpty(indexAttributes) && CollectionUtils.isNotEmpty(indexAttributes) &&
canApplyIndexFilter(classificationType, filterCriteria, false); canApplyIndexFilter(classificationTypes, filterCriteria, false);
traitPredicate = buildTraitPredict(classificationType); final boolean useGraphSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
!isWildcardSearch && CollectionUtils.isNotEmpty(graphAttributes));
traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry()); isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
AtlasGraph graph = context.getGraph(); AtlasGraph graph = context.getGraph();
...@@ -115,8 +120,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -115,8 +120,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
// tagFilters is not allowed in wildcard search // tagFilters is not allowed in wildcard search
graphIndexQueryBuilder.addClassificationTypeFilter(queryString); graphIndexQueryBuilder.addClassificationTypeFilter(queryString);
} else { } else {
if (isBuiltInType) { if (classificationTypes.iterator().next() == SearchContext.MATCH_ALL_NOT_CLASSIFIED) {
// tagFilters is not allowed in unique classificationType search // tagFilters is not allowed in unique classificationType search
graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString); graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString);
...@@ -146,7 +150,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -146,7 +150,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString); graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString, typeAndSubTypesQryStr); graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString, typeAndSubTypesQryStr);
constructFilterQuery(queryString, classificationType, filterCriteria, indexAttributes); constructFilterQuery(queryString, classificationTypes, filterCriteria, indexAttributes);
String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")"); String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")"); indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
...@@ -158,7 +162,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -158,7 +162,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate); inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
} }
Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes); Predicate attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) { if (attributePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate); inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
...@@ -170,7 +174,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -170,7 +174,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
} }
// only registered classification will search with tag filters // only registered classification will search with tag filters
if (!isWildcardSearch && !isBuiltInType && !graphAttributes.isEmpty()) { if (useGraphSearchForClassification) {
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE; AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
AtlasGraphQuery query = graph.query(); AtlasGraphQuery query = graph.query();
...@@ -179,7 +183,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -179,7 +183,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes); query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
} }
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationType, filterCriteria, allAttributes, query); tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
gremlinQueryBindings = new HashMap<>(); gremlinQueryBindings = new HashMap<>();
StringBuilder gremlinQuery = new StringBuilder(); StringBuilder gremlinQuery = new StringBuilder();
...@@ -188,7 +192,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -188,7 +192,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
gremlinQuery.append(".as('e').filter(out()"); gremlinQuery.append(".as('e').filter(out()");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER)); gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters()); // constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
// After filtering on tags go back to e and output the list of entity vertices // After filtering on tags go back to e and output the list of entity vertices
gremlinQuery.append(").toList()"); gremlinQuery.append(").toList()");
......
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
...@@ -486,8 +486,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -486,8 +486,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
resultAttributes.addAll(searchContext.getEntityAttributes()); resultAttributes.addAll(searchContext.getEntityAttributes());
} }
AtlasEntityType entityType = searchContext.getEntityType(); if (CollectionUtils.isNotEmpty(searchContext.getEntityTypes())) {
if (entityType != null) {
AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
for (String resultAttribute : resultAttributes) { for (String resultAttribute : resultAttributes) {
AtlasAttribute attribute = entityType.getAttribute(resultAttribute); AtlasAttribute attribute = entityType.getAttribute(resultAttribute);
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
...@@ -31,27 +31,15 @@ import org.apache.atlas.utils.AtlasPerfTracer; ...@@ -31,27 +31,15 @@ import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate; import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils; import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.*;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.stream.StreamSupport; import java.util.stream.StreamSupport;
import static org.apache.atlas.SortOrder.ASCENDING; import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES; import static org.apache.atlas.discovery.SearchContext.*;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
...@@ -69,28 +57,28 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -69,28 +57,28 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) { public EntitySearchProcessor(SearchContext context) {
super(context); super(context);
final AtlasEntityType entityType = context.getEntityType(); final Set<AtlasEntityType> entityTypes = context.getEntityTypes();
final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
final Set<String> indexAttributes = new HashSet<>(); final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>(); final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>(); final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getEntityTypes(); final Set<String> typeAndSubTypes = context.getEntityTypeNames();
final String typeAndSubTypesQryStr = context.getEntityTypesQryStr(); final String typeAndSubTypesQryStr = context.getEntityTypesQryStr();
final String sortBy = context.getSearchParameters().getSortBy(); final String sortBy = context.getSearchParameters().getSortBy();
final SortOrder sortOrder = context.getSearchParameters().getSortOrder(); final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
final AtlasClassificationType classificationType = context.getClassificationType(); final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
final Set<String> classificationTypeAndSubTypes = context.getClassificationTypes(); final Set<String> classificationTypeAndSubTypes = context.getClassificationTypeNames();
final boolean filterClassification; final boolean filterClassification;
if (classificationType != null) { if (CollectionUtils.isNotEmpty(classificationTypes)) {
filterClassification = !context.needClassificationProcessor(); filterClassification = !context.needClassificationProcessor();
} else { } else {
filterClassification = false; filterClassification = false;
} }
final Predicate typeNamePredicate; final Predicate typeNamePredicate;
final Predicate traitPredicate = buildTraitPredict(classificationType); final Predicate traitPredicate = buildTraitPredict(classificationTypes);
final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator() final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class); .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
...@@ -101,10 +89,10 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -101,10 +89,10 @@ public class EntitySearchProcessor extends SearchProcessor {
typeNamePredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry()); typeNamePredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
} }
processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes); processSearchAttributes(entityTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes);
final boolean typeSearchByIndex = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES; final boolean typeSearchByIndex = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES;
final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityType, filterCriteria, false); final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityTypes, filterCriteria, false);
StringBuilder indexQuery = new StringBuilder(); StringBuilder indexQuery = new StringBuilder();
...@@ -116,9 +104,9 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -116,9 +104,9 @@ public class EntitySearchProcessor extends SearchProcessor {
} }
if (attrSearchByIndex) { if (attrSearchByIndex) {
constructFilterQuery(indexQuery, entityType, filterCriteria, indexAttributes); constructFilterQuery(indexQuery, entityTypes, filterCriteria, indexAttributes);
Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, indexAttributes); Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) { if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate); inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
} }
...@@ -149,6 +137,7 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -149,6 +137,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// If we need to filter on the trait names then we need to build the query and equivalent in-memory predicate // If we need to filter on the trait names then we need to build the query and equivalent in-memory predicate
if (filterClassification) { if (filterClassification) {
AtlasClassificationType classificationType = classificationTypes.iterator().next();
List<AtlasGraphQuery> orConditions = new LinkedList<>(); List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
...@@ -176,10 +165,10 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -176,10 +165,10 @@ public class EntitySearchProcessor extends SearchProcessor {
} }
} }
graphQuery = toGraphFilterQuery(entityType, filterCriteria, graphAttributes, query); graphQuery = toGraphFilterQuery(entityTypes, filterCriteria, graphAttributes, query);
// Prepare in-memory predicate for attribute filtering // Prepare in-memory predicate for attribute filtering
Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, graphAttributes); Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, graphAttributes);
if (attributePredicate != null) { if (attributePredicate != null) {
if (graphQueryPredicate != null) { if (graphQueryPredicate != null) {
...@@ -199,7 +188,8 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -199,7 +188,8 @@ public class EntitySearchProcessor extends SearchProcessor {
} }
} }
if (sortBy != null && !sortBy.isEmpty()) { if (sortBy != null && !sortBy.isEmpty()) {
AtlasAttribute sortByAttribute = context.getEntityType().getAttribute(sortBy); final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute != null) { if (sortByAttribute != null) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC; AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
...@@ -215,8 +205,7 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -215,8 +205,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// Prepare the graph query and in-memory filter for the filtering phase // Prepare the graph query and in-memory filter for the filtering phase
filterGraphQueryPredicate = typeNamePredicate; filterGraphQueryPredicate = typeNamePredicate;
Predicate attributesPredicate = constructInMemoryPredicate(entityTypes, filterCriteria, allAttributes);
Predicate attributesPredicate = constructInMemoryPredicate(entityType, filterCriteria, allAttributes);
if (attributesPredicate != null) { if (attributesPredicate != null) {
filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate : filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate :
...@@ -265,7 +254,7 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -265,7 +254,7 @@ public class EntitySearchProcessor extends SearchProcessor {
SortOrder sortOrder = context.getSearchParameters().getSortOrder(); SortOrder sortOrder = context.getSearchParameters().getSortOrder();
String sortBy = context.getSearchParameters().getSortBy(); String sortBy = context.getSearchParameters().getSortBy();
final AtlasEntityType entityType = context.getEntityType(); final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy); AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute == null) { if (sortByAttribute == null) {
sortBy = null; sortBy = null;
......
repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
...@@ -18,7 +18,6 @@ ...@@ -18,7 +18,6 @@
package org.apache.atlas.discovery; package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.graphdb.*;
...@@ -53,13 +52,13 @@ public class FreeTextSearchProcessor extends SearchProcessor { ...@@ -53,13 +52,13 @@ public class FreeTextSearchProcessor extends SearchProcessor {
queryString.append(searchParameters.getQuery()); queryString.append(searchParameters.getQuery());
if (CollectionUtils.isNotEmpty(context.getEntityTypes()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) { if (CollectionUtils.isNotEmpty(context.getEntityTypeNames()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getEntityTypesQryStr()); queryString.append(AND_STR).append(context.getEntityTypesQryStr());
} }
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString); graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) { if (CollectionUtils.isNotEmpty(context.getClassificationTypeNames()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getClassificationTypesQryStr()); queryString.append(AND_STR).append(context.getClassificationTypesQryStr());
} }
...@@ -138,7 +137,7 @@ public class FreeTextSearchProcessor extends SearchProcessor { ...@@ -138,7 +137,7 @@ public class FreeTextSearchProcessor extends SearchProcessor {
continue; continue;
} }
if (context.getClassificationType() != null) { if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
List<String> entityClassifications = GraphHelper.getAllTraitNames(vertex); List<String> entityClassifications = GraphHelper.getAllTraitNames(vertex);
if (!context.includeClassificationTypes(entityClassifications)) { if (!context.includeClassificationTypes(entityClassifications)) {
......
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
...@@ -24,6 +24,7 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery; ...@@ -24,6 +24,7 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -31,10 +32,7 @@ import java.util.ArrayList; ...@@ -31,10 +32,7 @@ import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_ENTITY_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
public class FullTextSearchProcessor extends SearchProcessor { public class FullTextSearchProcessor extends SearchProcessor {
...@@ -53,29 +51,28 @@ public class FullTextSearchProcessor extends SearchProcessor { ...@@ -53,29 +51,28 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary // if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor // processing (and rejection) by subsequent EntitySearchProcessor
if (context.getEntityType() != null && context.getEntityType() != MATCH_ALL_ENTITY_TYPES) { if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr(); String typeAndSubTypeNamesQryStr = context.getEntityTypesQryStr();
if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) { if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr); queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
} else { } else {
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length()); searchParameters.getTypeName(), typeAndSubTypeNamesQryStr.length());
} }
} }
// if search includes classification criteria, adding a filter here can help avoid unnecessary // if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if (context.getClassificationType() != null && context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION && if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) &&
context.getClassificationType() != MATCH_ALL_CLASSIFIED && context.getClassificationTypes().iterator().next() != MATCH_ALL_NOT_CLASSIFIED) {
context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) { String typeAndSubTypeNamesStr = context.getClassificationTypesQryStr();
String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) { if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
queryString.append(AND_STR).append(typeAndSubTypeNamesStr); queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
} else { } else {
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length()); searchParameters.getClassification(), typeAndSubTypeNamesStr.length());
} }
} }
......
repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
...@@ -26,6 +26,8 @@ import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NA ...@@ -26,6 +26,8 @@ import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NA
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.type.AtlasStructType;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
public class GraphIndexQueryBuilder { public class GraphIndexQueryBuilder {
...@@ -36,20 +38,20 @@ public class GraphIndexQueryBuilder { ...@@ -36,20 +38,20 @@ public class GraphIndexQueryBuilder {
} }
void addClassificationTypeFilter(StringBuilder indexQuery) { void addClassificationTypeFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) { if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationNames())) {
String classificationName = context.getSearchParameters().getClassification(); String classificationNames = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(context.getClassificationNames());
if (indexQuery.length() != 0) { if (indexQuery.length() != 0) {
indexQuery.append(" AND "); indexQuery.append(" AND ");
} }
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName) indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationNames)
.append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY) .append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
.append('\"').append(':').append(classificationName).append(")"); .append('\"').append(':').append(classificationNames).append(")");
} }
} }
void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) { void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) { if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
String classificationTypesQryStr = context.getClassificationTypesQryStr(); String classificationTypesQryStr = context.getClassificationTypesQryStr();
if (indexQuery.length() != 0) { if (indexQuery.length() != 0) {
...@@ -63,17 +65,8 @@ public class GraphIndexQueryBuilder { ...@@ -63,17 +65,8 @@ public class GraphIndexQueryBuilder {
} }
void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) { void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) {
if (indexQuery != null && context.getClassificationType() != null) { if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
if (context.getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() == MATCH_ALL_CLASSIFIED) { if (context.getClassificationTypes().iterator().next() == MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"")
.append(CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]")
.append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"")
.append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]").append(")");
} else if (context.getClassificationType() == MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery.length() != 0) { if (indexQuery.length() != 0) {
indexQuery.append(" AND "); indexQuery.append(" AND ");
} }
......
repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
...@@ -56,12 +56,7 @@ public class SearchAggregatorImpl implements SearchAggregator { ...@@ -56,12 +56,7 @@ public class SearchAggregatorImpl implements SearchAggregator {
try { try {
AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient(); AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
String searchedOnTypeName = searchParameters.getTypeName(); Set<AtlasEntityType> searchForEntityType = searchContext.getEntityTypes();
AtlasEntityType searchForEntityType = null;
if (searchedOnTypeName != null) {
searchForEntityType = typeRegistry.getEntityTypeByName(searchedOnTypeName);
}
Map<String, String> indexFieldNameCache = new HashMap<>(); Map<String, String> indexFieldNameCache = new HashMap<>();
......
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
...@@ -21,7 +21,7 @@ package org.apache.atlas.discovery; ...@@ -21,7 +21,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria; import org.apache.atlas.model.discovery.SearchParameters.*;
import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
...@@ -44,13 +44,10 @@ import org.slf4j.Logger; ...@@ -44,13 +44,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
import static org.apache.atlas.discovery.SearchProcessor.ALL_TYPE_QUERY; import static org.apache.atlas.discovery.SearchProcessor.ALL_TYPE_QUERY;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS; import static org.apache.atlas.model.discovery.SearchParameters.*;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
/* /*
* Search context captures elements required for performing a basic search * Search context captures elements required for performing a basic search
...@@ -62,12 +59,12 @@ public class SearchContext { ...@@ -62,12 +59,12 @@ public class SearchContext {
private final AtlasTypeRegistry typeRegistry; private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph; private final AtlasGraph graph;
private final AtlasEntityType entityType; private final Set<AtlasEntityType> entityTypes;
private final Set<String> indexedKeys; private final Set<String> indexedKeys;
private final Set<String> entityAttributes; private final Set<String> entityAttributes;
private final SearchParameters searchParameters; private final SearchParameters searchParameters;
private final AtlasClassificationType classificationType; private final Set<AtlasClassificationType> classificationTypes;
private final String classificationName; private final Set<String> classificationNames;
private final Set<String> typeAndSubTypes; private final Set<String> typeAndSubTypes;
private final Set<String> classificationTypeAndSubTypes; private final Set<String> classificationTypeAndSubTypes;
private final String typeAndSubTypesQryStr; private final String typeAndSubTypesQryStr;
...@@ -80,27 +77,18 @@ public class SearchContext { ...@@ -80,27 +77,18 @@ public class SearchContext {
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS)); public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot(); public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot();
public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot(); public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot();
public final static String TYPENAME_DELIMITER = ",";
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException { public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
this.classificationName = searchParameters.getClassification();
this.searchParameters = searchParameters; this.searchParameters = searchParameters;
this.typeRegistry = typeRegistry; this.typeRegistry = typeRegistry;
this.graph = graph; this.graph = graph;
this.indexedKeys = indexedKeys; this.indexedKeys = indexedKeys;
this.entityAttributes = new HashSet<>(); this.entityAttributes = new HashSet<>();
this.entityType = getEntityType(searchParameters.getTypeName()); this.entityTypes = getEntityTypes(searchParameters.getTypeName());
this.classificationType = getClassificationType(classificationName); this.classificationNames = getClassificationNames(searchParameters.getClassification());
this.classificationTypes = getClassificationTypes(this.classificationNames);
// Validate if the type name exists
if (StringUtils.isNotEmpty(searchParameters.getTypeName()) && entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, searchParameters.getTypeName());
}
// Validate if the classification exists
if ((StringUtils.isNotEmpty(classificationName) && classificationType == null && !classificationName.contains(WILDCARD_CLASSIFICATIONS))) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName);
}
AtlasVertex glossaryTermVertex = getGlossaryTermVertex(searchParameters.getTermName()); AtlasVertex glossaryTermVertex = getGlossaryTermVertex(searchParameters.getTermName());
...@@ -109,40 +97,77 @@ public class SearchContext { ...@@ -109,40 +97,77 @@ public class SearchContext {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_GLOSSARY_TERM, searchParameters.getTermName()); throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_GLOSSARY_TERM, searchParameters.getTermName());
} }
// Invalid attributes will raise an exception with 400 error code // Invalid attributes or unsupported attribute in a type, will raise an exception with 400 error code
if (CollectionUtils.isNotEmpty(entityTypes)) {
for (AtlasEntityType entityType : entityTypes) {
validateAttributes(entityType, searchParameters.getEntityFilters()); validateAttributes(entityType, searchParameters.getEntityFilters());
// Invalid attribute will raise an exception with 400 error code
validateAttributes(entityType, searchParameters.getSortBy()); validateAttributes(entityType, searchParameters.getSortBy());
}
}
// Invalid attributes will raise an exception with 400 error code // Invalid attributes will raise an exception with 400 error code
if (CollectionUtils.isNotEmpty(classificationTypes)) {
for (AtlasClassificationType classificationType : classificationTypes) {
validateAttributes(classificationType, searchParameters.getTagFilters()); validateAttributes(classificationType, searchParameters.getTagFilters());
}
}
//remove other types if builtin type is present
filterStructTypes();
//gather all classifications and its corresponding subtypes
Set<String> classificationTypeAndSubTypes = new HashSet<>();
String classificationTypeAndSubTypesQryStr = null;
if (CollectionUtils.isNotEmpty(classificationTypes) && classificationTypes.iterator().next() != MATCH_ALL_NOT_CLASSIFIED ) {
for (AtlasClassificationType classificationType : classificationTypes) {
if (classificationType != null && !isBuiltInClassificationType()) {
if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
classificationTypeAndSubTypes = Collections.emptySet(); classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY; classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
break;
} else { } else {
classificationTypeAndSubTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName()); Set<String> allTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName());
classificationTypeAndSubTypesQryStr = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypesQryStr() : classificationType.getTypeQryStr(); classificationTypeAndSubTypes.addAll(allTypes); }
}
if (CollectionUtils.isNotEmpty(classificationTypeAndSubTypes)) {
classificationTypeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(classificationTypeAndSubTypes);
} }
} else { } else {
classificationTypeAndSubTypes = Collections.emptySet(); classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = ""; classificationTypeAndSubTypesQryStr = "";
} }
this.classificationTypeAndSubTypes = classificationTypeAndSubTypes;
this.classificationTypeAndSubTypesQryStr = classificationTypeAndSubTypesQryStr;
//gather all types and its corresponding subtypes
Set<String> typeAndSubTypes = new HashSet<>();
String typeAndSubTypesQryStr = null;
if (CollectionUtils.isNotEmpty(entityTypes)) {
for (AtlasEntityType entityType : entityTypes) {
if (entityType != null) {
if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) { if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
typeAndSubTypes = Collections.emptySet(); typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = ALL_TYPE_QUERY; typeAndSubTypesQryStr = ALL_TYPE_QUERY;
break;
} else { } else {
typeAndSubTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName()); Set<String> allTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
typeAndSubTypesQryStr = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypesQryStr() : entityType.getTypeQryStr(); typeAndSubTypes.addAll(allTypes);
}
}
if (CollectionUtils.isNotEmpty(typeAndSubTypes)) {
typeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndSubTypes);
} }
} else { } else {
typeAndSubTypes = Collections.emptySet(); typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = ""; typeAndSubTypesQryStr = "";
} }
this.typeAndSubTypes = typeAndSubTypes;
this.typeAndSubTypesQryStr = typeAndSubTypesQryStr;
if (glossaryTermVertex != null) { if (glossaryTermVertex != null) {
addProcessor(new TermSearchProcessor(this, getAssignedEntities(glossaryTermVertex))); addProcessor(new TermSearchProcessor(this, getAssignedEntities(glossaryTermVertex)));
...@@ -179,37 +204,37 @@ public class SearchContext { ...@@ -179,37 +204,37 @@ public class SearchContext {
public Set<String> getEntityAttributes() { return entityAttributes; } public Set<String> getEntityAttributes() { return entityAttributes; }
public AtlasEntityType getEntityType() { return entityType; } public Set<AtlasClassificationType> getClassificationTypes() { return classificationTypes; }
public AtlasClassificationType getClassificationType() { return classificationType; }
public Set<String> getEntityTypes() { return typeAndSubTypes; } public Set<String> getEntityTypeNames() { return typeAndSubTypes; }
public Set<String> getClassificationTypes() { return classificationTypeAndSubTypes; } public Set<String> getClassificationTypeNames() { return classificationTypeAndSubTypes; }
public String getEntityTypesQryStr() { return typeAndSubTypesQryStr; } public String getEntityTypesQryStr() { return typeAndSubTypesQryStr; }
public String getClassificationTypesQryStr() { return classificationTypeAndSubTypesQryStr; } public String getClassificationTypesQryStr() { return classificationTypeAndSubTypesQryStr; }
public Set<AtlasEntityType> getEntityTypes() { return entityTypes; }
public SearchProcessor getSearchProcessor() { return searchProcessor; } public SearchProcessor getSearchProcessor() { return searchProcessor; }
public String getClassificationName() {return classificationName;} public Set<String> getClassificationNames() {return classificationNames;}
public boolean includeEntityType(String entityType) { public boolean includeEntityType(String entityType) {
return typeAndSubTypes.isEmpty() || typeAndSubTypes.contains(entityType); return typeAndSubTypes.isEmpty() || typeAndSubTypes.contains(entityType);
} }
public boolean includeClassificationTypes(Collection<String> classificationTypes) { public boolean includeClassificationTypes(Collection<String> traitNames) {
final boolean ret; final boolean ret;
if (classificationType == null || classificationTypeAndSubTypes.isEmpty()) { if (CollectionUtils.isEmpty(classificationTypes) || classificationTypeAndSubTypes.isEmpty()) {
ret = true; ret = true;
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { } else if (classificationTypes.iterator().next() == MATCH_ALL_NOT_CLASSIFIED) {
ret = CollectionUtils.isEmpty(classificationTypes); ret = CollectionUtils.isEmpty(traitNames);
} else if (classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION) { } else if (classificationTypes.iterator().next() == MATCH_ALL_CLASSIFICATION_TYPES) {
ret = CollectionUtils.isNotEmpty(classificationTypes); ret = CollectionUtils.isNotEmpty(traitNames);
} else { } else {
ret = CollectionUtils.containsAny(classificationTypeAndSubTypes, classificationTypes); ret = CollectionUtils.containsAny(classificationTypeAndSubTypes, traitNames);
} }
return ret; return ret;
...@@ -243,25 +268,18 @@ public class SearchContext { ...@@ -243,25 +268,18 @@ public class SearchContext {
} }
boolean needClassificationProcessor() { boolean needClassificationProcessor() {
return (classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ; return (CollectionUtils.isNotEmpty(classificationTypes) && (CollectionUtils.isEmpty(entityTypes) || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
}
boolean isBuiltInClassificationType() {
return getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION
|| getClassificationType() == MATCH_ALL_CLASSIFIED
|| getClassificationType() == MATCH_ALL_NOT_CLASSIFIED;
} }
boolean isWildCardSearch () { boolean isWildCardSearch () {
String classification = getSearchParameters().getClassification(); if (CollectionUtils.isNotEmpty(classificationNames)) {
if (StringUtils.isNotEmpty(classification) && getClassificationType() == null) { return classificationNames.stream().anyMatch(classification -> classification.contains(WILDCARD_CLASSIFICATIONS));
return classification.contains("*");
} }
return false; return false;
} }
boolean needEntityProcessor() { boolean needEntityProcessor() {
return entityType != null; return CollectionUtils.isNotEmpty(entityTypes);
} }
private void validateAttributes(final AtlasStructType structType, final FilterCriteria filterCriteria) throws AtlasBaseException { private void validateAttributes(final AtlasStructType structType, final FilterCriteria filterCriteria) throws AtlasBaseException {
...@@ -285,7 +303,14 @@ public class SearchContext { ...@@ -285,7 +303,14 @@ public class SearchContext {
if (structType == null) { if (structType == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL"); throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL");
} }
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, structType.getTypeName());
String name = structType.getTypeName();
if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
name = ALL_ENTITY_TYPES;
} else if (name.equals(MATCH_ALL_CLASSIFICATION_TYPES.getTypeName())) {
name = ALL_CLASSIFICATION_TYPES;
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, name);
} }
} }
} }
...@@ -321,11 +346,107 @@ public class SearchContext { ...@@ -321,11 +346,107 @@ public class SearchContext {
return ret; return ret;
} }
private Set<AtlasClassificationType> getClassificationTypes(Set<String> classificationNames) {
if (CollectionUtils.isNotEmpty(classificationNames)) {
return classificationNames.stream().map(n ->
getClassificationType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
}
return null;
}
private Set<String> getClassificationNames(String classification) throws AtlasBaseException {
Set<String> classificationNames = new HashSet<>();
if (StringUtils.isNotEmpty(classification)) {
String[] types = classification.split(TYPENAME_DELIMITER);
Set<String> names = new HashSet<>(Arrays.asList(types));
names.forEach(name -> {
AtlasClassificationType type = getClassificationType(name);
if (type != null || name.contains(WILDCARD_CLASSIFICATIONS)) {
classificationNames.add(name);
}
});
// Validate if the classification exists
if (CollectionUtils.isEmpty(classificationNames) || classificationNames.size() != names.size()) {
if (CollectionUtils.isNotEmpty(classificationNames)) {
names.removeAll(classificationNames);
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, String.join(TYPENAME_DELIMITER, names));
}
}
return classificationNames;
}
private AtlasEntityType getEntityType(String entityName) { private AtlasEntityType getEntityType(String entityName) {
return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ? MATCH_ALL_ENTITY_TYPES : return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ? MATCH_ALL_ENTITY_TYPES :
typeRegistry.getEntityTypeByName(entityName); typeRegistry.getEntityTypeByName(entityName);
} }
private Set<AtlasEntityType> getEntityTypes(String typeName) throws AtlasBaseException {
Set<AtlasEntityType> entityTypes = null;
//split multiple typeNames by comma
if (StringUtils.isNotEmpty(typeName)) {
String[] types = typeName.split(TYPENAME_DELIMITER);
Set<String> typeNames = new HashSet<>(Arrays.asList(types));
entityTypes = typeNames.stream().map(n ->
getEntityType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
// Validate if the type name is incorrect
if (CollectionUtils.isEmpty(entityTypes) || entityTypes.size() != typeNames.size()) {
if (CollectionUtils.isNotEmpty(entityTypes)) {
Set<String> validEntityTypes = new HashSet<>();
for (AtlasEntityType entityType : entityTypes) {
String name = entityType.getTypeName();
if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
validEntityTypes.add(ALL_ENTITY_TYPES);
continue;
}
validEntityTypes.add(entityType.getTypeName());
}
typeNames.removeAll(validEntityTypes);
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, String.join(TYPENAME_DELIMITER, typeNames));
}
}
return entityTypes;
}
private void filterStructTypes(){
//if typeName contains ALL_ENTITY_TYPES, remove others as OR condition will not effect any other
if (CollectionUtils.isNotEmpty(entityTypes) && entityTypes.contains(MATCH_ALL_ENTITY_TYPES)) {
entityTypes.clear();
entityTypes.add(MATCH_ALL_ENTITY_TYPES);
}
//No Builtin Classification can be together
if (CollectionUtils.isNotEmpty(classificationTypes)) {
if (classificationTypes.contains(MATCH_ALL_NOT_CLASSIFIED)) {
classificationTypes.clear();
classificationTypes.add(MATCH_ALL_NOT_CLASSIFIED);
classificationNames.clear();
classificationNames.add(MATCH_ALL_NOT_CLASSIFIED.getTypeName());
} else if (classificationTypes.contains(MATCH_ALL_WILDCARD_CLASSIFICATION) || classificationTypes.contains(MATCH_ALL_CLASSIFICATION_TYPES) || classificationTypes.contains(MATCH_ALL_CLASSIFIED)) {
classificationTypes.clear();
classificationTypes.add(MATCH_ALL_CLASSIFICATION_TYPES);
classificationNames.clear();
classificationNames.add(ALL_CLASSIFICATION_TYPES);
}
}
}
private AtlasVertex getGlossaryTermVertex(String termName) { private AtlasVertex getGlossaryTermVertex(String termName) {
AtlasVertex ret = null; AtlasVertex ret = null;
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
...@@ -51,16 +51,8 @@ import java.util.regex.Pattern; ...@@ -51,16 +51,8 @@ import java.util.regex.Pattern;
import static org.apache.atlas.SortOrder.ASCENDING; import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION; import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.util.SearchPredicateUtil.*; import static org.apache.atlas.util.SearchPredicateUtil.*;
public abstract class SearchProcessor { public abstract class SearchProcessor {
...@@ -153,11 +145,19 @@ public abstract class SearchProcessor { ...@@ -153,11 +145,19 @@ public abstract class SearchProcessor {
public abstract long getResultCount(); public abstract long getResultCount();
protected boolean isEntityRootType() { protected boolean isEntityRootType() {
return context.getEntityType() == SearchContext.MATCH_ALL_ENTITY_TYPES; //always size will be one if in case of _ALL_ENTITY_TYPES
if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
return context.getEntityTypes().iterator().next() == SearchContext.MATCH_ALL_ENTITY_TYPES;
}
return false;
} }
protected boolean isClassificationRootType() { protected boolean isClassificationRootType() {
return context.getClassificationType() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES; //always size will be one if in case of _ALL_CLASSIFICATION_TYPES
if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
return context.getClassificationTypes().iterator().next() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
}
return false;
} }
protected boolean isSystemAttribute(String attrName) { protected boolean isSystemAttribute(String attrName) {
...@@ -195,9 +195,14 @@ public abstract class SearchProcessor { ...@@ -195,9 +195,14 @@ public abstract class SearchProcessor {
} }
} }
protected Predicate buildTraitPredict(AtlasClassificationType classificationType) { protected Predicate buildTraitPredict(Set<AtlasClassificationType> classificationTypes) {
Predicate traitPredicate; Predicate traitPredicate;
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { AtlasClassificationType classificationType = null;
if (CollectionUtils.isNotEmpty(classificationTypes)) {
classificationType = classificationTypes.iterator().next();
}
if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
...@@ -205,20 +210,27 @@ public abstract class SearchProcessor { ...@@ -205,20 +210,27 @@ public abstract class SearchProcessor {
SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (context.isWildCardSearch()) { } else if (context.isWildCardSearch()) {
//For wildcard search __classificationNames which of String type is taken instead of _traitNames which is of Array type //For wildcard search __classificationNames which of String type is taken instead of _traitNames which is of Array type
Set<String> classificationNames = context.getClassificationNames();
List<Predicate> predicates = new ArrayList<>();
classificationNames.forEach(classificationName -> {
//No need to escape, as classification Names only support letters,numbers,space and underscore //No need to escape, as classification Names only support letters,numbers,space and underscore
String regexString = getRegexString("\\|" + context.getClassificationName() + "\\|"); String regexString = getRegexString("\\|" + classificationName + "\\|");
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, regexString, String.class), predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, regexString, String.class));
SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, regexString, String.class)); predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, regexString, String.class));
});
traitPredicate = PredicateUtils.anyPredicate(predicates);
} else { } else {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class), traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class)); SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class));
} }
return traitPredicate; return traitPredicate;
} }
protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) { protected void processSearchAttributes(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) {
if (structType == null || filterCriteria == null) { if (CollectionUtils.isEmpty(structTypes) || filterCriteria == null) {
return; return;
} }
...@@ -227,7 +239,7 @@ public abstract class SearchProcessor { ...@@ -227,7 +239,7 @@ public abstract class SearchProcessor {
if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) { if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
for (SearchParameters.FilterCriteria criteria : criterion) { for (SearchParameters.FilterCriteria criteria : criterion) {
processSearchAttributes(structType, criteria, indexFiltered, graphFiltered, allAttributes); processSearchAttributes(structTypes, criteria, indexFiltered, graphFiltered, allAttributes);
} }
} else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) { } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
String attributeName = filterCriteria.getAttributeName(); String attributeName = filterCriteria.getAttributeName();
...@@ -267,12 +279,14 @@ public abstract class SearchProcessor { ...@@ -267,12 +279,14 @@ public abstract class SearchProcessor {
} }
try { try {
for (AtlasStructType structType : structTypes) {
String qualifiedName = structType.getVertexPropertyName(attributeName);
if (isIndexSearchable(filterCriteria, structType)) { if (isIndexSearchable(filterCriteria, structType)) {
indexFiltered.add(attributeName); indexFiltered.add(qualifiedName);
} else { } else {
LOG.warn("not using index-search for attribute '{}'; might cause poor performance", structType.getVertexPropertyName(attributeName)); LOG.warn("not using index-search for attribute '{}'; might cause poor performance", structType.getVertexPropertyName(attributeName));
graphFiltered.add(attributeName); graphFiltered.add(qualifiedName);
} }
if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) { if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) {
...@@ -280,7 +294,8 @@ public abstract class SearchProcessor { ...@@ -280,7 +294,8 @@ public abstract class SearchProcessor {
context.getEntityAttributes().add(attributeName); context.getEntityAttributes().add(attributeName);
} }
allAttributes.add(attributeName); allAttributes.add(qualifiedName);
}
} catch (AtlasBaseException e) { } catch (AtlasBaseException e) {
LOG.warn(e.getMessage()); LOG.warn(e.getMessage());
} }
...@@ -298,7 +313,7 @@ public abstract class SearchProcessor { ...@@ -298,7 +313,7 @@ public abstract class SearchProcessor {
// (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z) // (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z)
// (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z (AND idx-attr2=xyz idx-attr2=abc)) // (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z (AND idx-attr2=xyz idx-attr2=abc))
// //
protected boolean canApplyIndexFilter(AtlasStructType structType, FilterCriteria filterCriteria, boolean insideOrCondition) { protected boolean canApplyIndexFilter(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, boolean insideOrCondition) {
if (!context.hasAttributeFilter(filterCriteria)) { if (!context.hasAttributeFilter(filterCriteria)) {
return true; return true;
} }
...@@ -314,7 +329,7 @@ public abstract class SearchProcessor { ...@@ -314,7 +329,7 @@ public abstract class SearchProcessor {
// If we have nested criterion let's find any nested ORs with non-indexed attr // If we have nested criterion let's find any nested ORs with non-indexed attr
for (FilterCriteria criteria : criterion) { for (FilterCriteria criteria : criterion) {
ret = canApplyIndexFilter(structType, criteria, insideOrCondition); ret = canApplyIndexFilter(structTypes, criteria, insideOrCondition);
if (!ret) { if (!ret) {
break; break;
...@@ -322,10 +337,11 @@ public abstract class SearchProcessor { ...@@ -322,10 +337,11 @@ public abstract class SearchProcessor {
} }
} else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) { } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
try { try {
for (AtlasStructType structType : structTypes) {
if (insideOrCondition && !isIndexSearchable(filterCriteria, structType)) { if (insideOrCondition && !isIndexSearchable(filterCriteria, structType)) {
ret = false; ret = false;
break;
}
} }
} catch (AtlasBaseException e) { } catch (AtlasBaseException e) {
LOG.warn(e.getMessage()); LOG.warn(e.getMessage());
...@@ -338,7 +354,7 @@ public abstract class SearchProcessor { ...@@ -338,7 +354,7 @@ public abstract class SearchProcessor {
protected void filterWhiteSpaceClassification(List<AtlasVertex> entityVertices) { protected void filterWhiteSpaceClassification(List<AtlasVertex> entityVertices) {
if (CollectionUtils.isNotEmpty(entityVertices)) { if (CollectionUtils.isNotEmpty(entityVertices)) {
final Iterator<AtlasVertex> it = entityVertices.iterator(); final Iterator<AtlasVertex> it = entityVertices.iterator();
final Set<String> typeAndSubTypes = context.getClassificationTypes(); final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
while (it.hasNext()) { while (it.hasNext()) {
AtlasVertex entityVertex = it.next(); AtlasVertex entityVertex = it.next();
...@@ -363,13 +379,13 @@ public abstract class SearchProcessor { ...@@ -363,13 +379,13 @@ public abstract class SearchProcessor {
} }
} }
protected void constructFilterQuery(StringBuilder indexQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> indexAttributes) { protected void constructFilterQuery(StringBuilder indexQuery, Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) {
if (filterCriteria != null) { if (filterCriteria != null) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Processing Filters"); LOG.debug("Processing Filters");
} }
String filterQuery = toIndexQuery(type, filterCriteria, indexAttributes, 0); String filterQuery = toIndexQuery(structTypes, filterCriteria, indexAttributes, 0);
if (StringUtils.isNotEmpty(filterQuery)) { if (StringUtils.isNotEmpty(filterQuery)) {
if (indexQuery.length() > 0) { if (indexQuery.length() > 0) {
...@@ -381,14 +397,14 @@ public abstract class SearchProcessor { ...@@ -381,14 +397,14 @@ public abstract class SearchProcessor {
} }
} }
protected Predicate constructInMemoryPredicate(AtlasStructType type, FilterCriteria filterCriteria, Set<String> indexAttributes) { protected Predicate constructInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) {
Predicate ret = null; Predicate ret = null;
if (filterCriteria != null) { if (filterCriteria != null) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Processing Filters"); LOG.debug("Processing Filters");
} }
ret = toInMemoryPredicate(type, filterCriteria, indexAttributes); ret = toInMemoryPredicate(structTypes, filterCriteria, indexAttributes);
} }
return ret; return ret;
} }
...@@ -473,17 +489,20 @@ public abstract class SearchProcessor { ...@@ -473,17 +489,20 @@ public abstract class SearchProcessor {
return ret; return ret;
} }
private String toIndexQuery(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes, int level) { private String toIndexQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes, int level) {
return toIndexQuery(type, criteria, indexAttributes, new StringBuilder(), level); return toIndexQuery(structTypes, criteria, indexAttributes, new StringBuilder(), level);
} }
private String toIndexQuery(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int level) { private String toIndexQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int level) {
Set<String> filterAttributes = new HashSet<>();
filterAttributes.addAll(indexAttributes);
Condition condition = criteria.getCondition(); Condition condition = criteria.getCondition();
if (condition != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) { if (condition != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
StringBuilder nestedExpression = new StringBuilder(); StringBuilder nestedExpression = new StringBuilder();
for (FilterCriteria filterCriteria : criteria.getCriterion()) { for (FilterCriteria filterCriteria : criteria.getCriterion()) {
String nestedQuery = toIndexQuery(type, filterCriteria, indexAttributes, level + 1); String nestedQuery = toIndexQuery(structTypes, filterCriteria, filterAttributes, level + 1);
if (StringUtils.isNotEmpty(nestedQuery)) { if (StringUtils.isNotEmpty(nestedQuery)) {
if (nestedExpression.length() > 0) { if (nestedExpression.length() > 0) {
...@@ -502,19 +521,45 @@ public abstract class SearchProcessor { ...@@ -502,19 +521,45 @@ public abstract class SearchProcessor {
} else { } else {
return EMPTY_STRING; return EMPTY_STRING;
} }
} else if (indexAttributes.contains(criteria.getAttributeName())){ } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
return toIndexExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue()); try {
ArrayList<String> orExpQuery = new ArrayList<>();
for (AtlasStructType structType : structTypes) {
String name = structType.getVertexPropertyName(criteria.getAttributeName());
if (filterAttributes.contains(name)) {
String nestedQuery = toIndexExpression(structType, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
orExpQuery.add(nestedQuery);
filterAttributes.remove(name);
}
}
if (CollectionUtils.isNotEmpty(orExpQuery)) {
if (orExpQuery.size() > 1) {
String orExpStr = StringUtils.join(orExpQuery, " "+Condition.OR.name()+" ");
return BRACE_OPEN_STR + " " + orExpStr + " " + BRACE_CLOSE_STR;
} else {
return orExpQuery.iterator().next();
}
} else { } else {
return EMPTY_STRING; return EMPTY_STRING;
} }
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
}
}
return EMPTY_STRING;
} }
private Predicate toInMemoryPredicate(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes) { private Predicate toInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes) {
Set<String> filterAttributes = new HashSet<>();
filterAttributes.addAll(indexAttributes);
if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) { if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
List<Predicate> predicates = new ArrayList<>(); List<Predicate> predicates = new ArrayList<>();
for (FilterCriteria filterCriteria : criteria.getCriterion()) { for (FilterCriteria filterCriteria : criteria.getCriterion()) {
Predicate predicate = toInMemoryPredicate(type, filterCriteria, indexAttributes); Predicate predicate = toInMemoryPredicate(structTypes, filterCriteria, filterAttributes);
if (predicate != null) { if (predicate != null) {
predicates.add(predicate); predicates.add(predicate);
...@@ -528,7 +573,13 @@ public abstract class SearchProcessor { ...@@ -528,7 +573,13 @@ public abstract class SearchProcessor {
return PredicateUtils.anyPredicate(predicates); return PredicateUtils.anyPredicate(predicates);
} }
} }
} else if (indexAttributes.contains(criteria.getAttributeName())) { } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
try {
ArrayList<Predicate> predicates = new ArrayList<>();
for (AtlasStructType structType : structTypes) {
String name = structType.getVertexPropertyName(criteria.getAttributeName());
if (filterAttributes.contains(name)) {
String attrName = criteria.getAttributeName(); String attrName = criteria.getAttributeName();
String attrValue = criteria.getAttributeValue(); String attrValue = criteria.getAttributeValue();
SearchParameters.Operator operator = criteria.getOperator(); SearchParameters.Operator operator = criteria.getOperator();
...@@ -540,9 +591,22 @@ public abstract class SearchProcessor { ...@@ -540,9 +591,22 @@ public abstract class SearchProcessor {
operator = processedCriteria.getOperator(); operator = processedCriteria.getOperator();
} }
return toInMemoryPredicate(type, attrName, operator, attrValue); predicates.add(toInMemoryPredicate(structType, attrName, operator, attrValue));
filterAttributes.remove(name);
}
} }
if (CollectionUtils.isNotEmpty(predicates)) {
if (predicates.size() > 1) {
return PredicateUtils.anyPredicate(predicates);
} else {
return predicates.iterator().next();
}
}
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
}
}
return null; return null;
} }
...@@ -704,12 +768,15 @@ public abstract class SearchProcessor { ...@@ -704,12 +768,15 @@ public abstract class SearchProcessor {
return ret; return ret;
} }
protected AtlasGraphQuery toGraphFilterQuery(AtlasStructType type, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) { protected AtlasGraphQuery toGraphFilterQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) {
Set<String> filterAttributes = new HashSet<>();
filterAttributes.addAll(graphAttributes);
if (criteria != null) { if (criteria != null) {
if (criteria.getCondition() != null) { if (criteria.getCondition() != null) {
if (criteria.getCondition() == Condition.AND) { if (criteria.getCondition() == Condition.AND) {
for (FilterCriteria filterCriteria : criteria.getCriterion()) { for (FilterCriteria filterCriteria : criteria.getCriterion()) {
AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, graphAttributes, context.getGraph().query()); AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
query.addConditionsFrom(nestedQuery); query.addConditionsFrom(nestedQuery);
} }
...@@ -717,7 +784,7 @@ public abstract class SearchProcessor { ...@@ -717,7 +784,7 @@ public abstract class SearchProcessor {
List<AtlasGraphQuery> orConditions = new LinkedList<>(); List<AtlasGraphQuery> orConditions = new LinkedList<>();
for (FilterCriteria filterCriteria : criteria.getCriterion()) { for (FilterCriteria filterCriteria : criteria.getCriterion()) {
AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, graphAttributes, context.getGraph().query()); AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery)); orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery));
} }
...@@ -726,7 +793,13 @@ public abstract class SearchProcessor { ...@@ -726,7 +793,13 @@ public abstract class SearchProcessor {
query.or(orConditions); query.or(orConditions);
} }
} }
} else if (graphAttributes.contains(criteria.getAttributeName())) { } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
try {
ArrayList<AtlasGraphQuery> queries = new ArrayList<>();
for (AtlasStructType structType : structTypes) {
String qualifiedName = structType.getVertexPropertyName(criteria.getAttributeName());
if (filterAttributes.contains(qualifiedName)) {
String attrName = criteria.getAttributeName(); String attrName = criteria.getAttributeName();
String attrValue = criteria.getAttributeValue(); String attrValue = criteria.getAttributeValue();
SearchParameters.Operator operator = criteria.getOperator(); SearchParameters.Operator operator = criteria.getOperator();
...@@ -738,49 +811,63 @@ public abstract class SearchProcessor { ...@@ -738,49 +811,63 @@ public abstract class SearchProcessor {
operator = processedCriteria.getOperator(); operator = processedCriteria.getOperator();
} }
final String qualifiedName = type.getAttribute(attrName).getVertexPropertyName(); AtlasGraphQuery innerQry = context.getGraph().query().createChildQuery();
switch (operator) { switch (operator) {
case LT: case LT:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue); innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
break; break;
case LTE: case LTE:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue); innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
break; break;
case GT: case GT:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue); innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
break; break;
case GTE: case GTE:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue); innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
break; break;
case EQ: case EQ:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue); innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
break; break;
case NEQ: case NEQ:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue); innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
break; break;
case LIKE: case LIKE:
query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue); innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
break; break;
case CONTAINS: case CONTAINS:
query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue)); innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
break; break;
case STARTS_WITH: case STARTS_WITH:
query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue); innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
break; break;
case ENDS_WITH: case ENDS_WITH:
query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue)); innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
break; break;
case IS_NULL: case IS_NULL:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null); innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null);
break; break;
case NOT_NULL: case NOT_NULL:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null); innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
break; break;
default: default:
LOG.warn("{}: unsupported operator. Ignored", operator); LOG.warn("{}: unsupported operator. Ignored", operator);
break; break;
} }
queries.add(context.getGraph().query().createChildQuery().addConditionsFrom(innerQry));
filterAttributes.remove(qualifiedName);
}
}
if (CollectionUtils.isNotEmpty(queries)) {
if (queries.size() > 1) {
return context.getGraph().query().createChildQuery().or(queries);
} else {
return queries.iterator().next();
}
}
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
}
} }
} }
...@@ -987,12 +1074,14 @@ public abstract class SearchProcessor { ...@@ -987,12 +1074,14 @@ public abstract class SearchProcessor {
} }
private static String getSortByAttribute(SearchContext context) { private static String getSortByAttribute(SearchContext context) {
final AtlasEntityType entityType = context.getEntityType(); if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
String sortBy = context.getSearchParameters().getSortBy(); String sortBy = context.getSearchParameters().getSortBy();
AtlasStructType.AtlasAttribute sortByAttribute = entityType != null ? entityType.getAttribute(sortBy) : null; AtlasStructType.AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute != null) { if (sortByAttribute != null) {
return sortByAttribute.getVertexPropertyName(); return sortByAttribute.getVertexPropertyName();
} }
}
return null; return null;
} }
......
repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
...@@ -130,7 +130,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup { ...@@ -130,7 +130,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
new EntitySearchProcessor(context); new EntitySearchProcessor(context);
} }
@Test @Test(priority = -1)
public void searchWithNEQ_stringAttr() throws AtlasBaseException { public void searchWithNEQ_stringAttr() throws AtlasBaseException {
String expectedEntityName = "hive_Table_Null_tableType"; String expectedEntityName = "hive_Table_Null_tableType";
createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE); createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
...@@ -197,4 +197,116 @@ public class EntitySearchProcessorTest extends BasicTestSetup { ...@@ -197,4 +197,116 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertTrue(nameList.contains("hive_Table_Null_tableType")); assertTrue(nameList.contains("hive_Table_Null_tableType"));
} }
@Test
public void ALLEntityType() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 20);
}
@Test
public void ALLEntityTypeWithTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
params.setClassification(FACT_CLASSIFICATION);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 5);
}
@Test
public void entityType() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 3);
}
@Test
public void entityTypes() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 14);
}
@Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Not_Exists: Unknown/invalid typename")
public void entityTypesNotAllowed() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+",Not_Exists");
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
}
@Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Attribute tableType not found for type "+DATABASE_TYPE)
public void entityFiltersNotAllowed() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.CONTAINS, "ETL");
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
}
@Test
public void entityTypesAndTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
params.setClassification(FACT_CLASSIFICATION);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 3);
}
@Test
public void searchWithEntityTypesAndEntityFilters() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 4);
}
@Test
public void searchWithEntityTypesAndEntityFiltersAndTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
params.setEntityFilters(filterCriteria);
params.setClassification(LOGDATA_CLASSIFICATION);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 2);
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment