Skip to content

A
atlas
Commit 8b5cb9d9 by Pinal Shah Committed by Sarath Subramanian
Options

ATLAS-3838: Support multiple tag/classification in basic/quick search API

ATLAS-3652: Quick Search: API requirement for GET request on multiple entity types Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org> both JIRA's addressed in the same commit.
parent e691a32c
Showing with 750 additions and 377 deletions
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
......@@ -25,14 +25,14 @@ import java.util.Map;
import java.util.Set;
public class AggregationContext {
private final String queryString;
private final FilterCriteria filterCriteria;
private final AtlasEntityType searchForEntityType;
private final Set<String> aggregationFieldNames;
private final Set<AtlasAttribute> aggregationAttributes;
private final Map<String, String> indexFieldNameCache;
private final boolean excludeDeletedEntities;
private final boolean includeSubTypes;
private final String queryString;
private final FilterCriteria filterCriteria;
private final Set<AtlasEntityType> searchForEntityTypes;
private final Set<String> aggregationFieldNames;
private final Set<AtlasAttribute> aggregationAttributes;
private final Map<String, String> indexFieldNameCache;
private final boolean excludeDeletedEntities;
private final boolean includeSubTypes;
/**
* @param queryString the query string whose aggregation metrics need to be retrieved.
......@@ -41,17 +41,17 @@ public class AggregationContext {
* @param indexFieldNameCache
* @param excludeDeletedEntities a boolean flag to indicate if the deleted entities need to be excluded in search
*/
public AggregationContext(String queryString,
FilterCriteria filterCriteria,
AtlasEntityType searchForEntityType,
Set<String> aggregationFieldNames,
Set<AtlasAttribute> aggregationAttributes,
Map<String, String> indexFieldNameCache,
boolean excludeDeletedEntities,
boolean includeSubTypes) {
public AggregationContext(String queryString,
FilterCriteria filterCriteria,
Set<AtlasEntityType> searchForEntityType,
Set<String> aggregationFieldNames,
Set<AtlasAttribute> aggregationAttributes,
Map<String, String> indexFieldNameCache,
boolean excludeDeletedEntities,
boolean includeSubTypes) {
this.queryString = queryString;
this.filterCriteria = filterCriteria;
this.searchForEntityType = searchForEntityType;
this.searchForEntityTypes = searchForEntityType;
this.aggregationFieldNames = aggregationFieldNames;
this.aggregationAttributes = aggregationAttributes;
this.indexFieldNameCache = indexFieldNameCache;
......@@ -67,8 +67,8 @@ public class AggregationContext {
return filterCriteria;
}
public AtlasEntityType getSearchForEntityType() {
return searchForEntityType;
public Set<AtlasEntityType> getSearchForEntityTypes() {
return searchForEntityTypes;
}
public Set<String> getAggregationFieldNames() {
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
......@@ -175,7 +175,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
Map<String, String> indexFieldName2PropertyKeyNameMap = new HashMap<>();
AtlasSolrQueryBuilder solrQueryBuilder = new AtlasSolrQueryBuilder();
solrQueryBuilder.withEntityType(aggregationContext.getSearchForEntityType())
solrQueryBuilder.withEntityTypes(aggregationContext.getSearchForEntityTypes())
.withQueryString(aggregationContext.getQueryString())
.withCriteria(aggregationContext.getFilterCriteria())
.withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities())
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
......@@ -29,31 +29,28 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
public class AtlasSolrQueryBuilder {
private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class);
private AtlasEntityType entityType;
private String queryString;
private FilterCriteria criteria;
private boolean excludeDeletedEntities;
private boolean includeSubtypes;
private Map<String, String> indexFieldNameCache;
public static final char CUSTOM_ATTR_SEPARATOR = '=';
public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
private Set<AtlasEntityType> entityTypes;
private String queryString;
private FilterCriteria criteria;
private boolean excludeDeletedEntities;
private boolean includeSubtypes;
private Map<String, String> indexFieldNameCache;
public static final char CUSTOM_ATTR_SEPARATOR = '=';
public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
public AtlasSolrQueryBuilder() {
}
public AtlasSolrQueryBuilder withEntityType(AtlasEntityType searchForEntityType) {
this.entityType = searchForEntityType;
public AtlasSolrQueryBuilder withEntityTypes(Set<AtlasEntityType> searchForEntityTypes) {
this.entityTypes = searchForEntityTypes;
return this;
}
......@@ -112,7 +109,7 @@ public class AtlasSolrQueryBuilder {
isAndNeeded = true;
}
if (entityType != null) {
if (CollectionUtils.isNotEmpty(entityTypes)) {
if (isAndNeeded) {
queryBuilder.append(" AND ");
}
......@@ -140,29 +137,25 @@ public class AtlasSolrQueryBuilder {
}
private void buildForEntityType(StringBuilder queryBuilder) {
if (LOG.isDebugEnabled()) {
LOG.debug("Search is being done for entities of type {}", entityType.getTypeName());
}
String typeIndexFieldName = indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY);
queryBuilder.append(" +")
.append(typeIndexFieldName)
.append(":(")
.append(entityType.getTypeName())
.append(" ");
.append(":(");
if (includeSubtypes) {
Set<String> allSubTypes = entityType.getAllSubTypes();
Set<String> typesToSearch = new HashSet<>();
for (AtlasEntityType type : entityTypes) {
if(allSubTypes.size() != 0 ) {
for(String subTypeName: allSubTypes) {
queryBuilder.append(subTypeName).append(" ");
}
if (includeSubtypes) {
typesToSearch.addAll(type.getTypeAndAllSubTypes());
} else {
typesToSearch.add(type.getTypeName());
}
}
queryBuilder.append(" ) ");
queryBuilder.append(StringUtils.join(typesToSearch, " ")).append(" ) ");
}
private void dropDeletedEntities(StringBuilder queryBuilder) throws AtlasBaseException {
......@@ -173,9 +166,8 @@ public class AtlasSolrQueryBuilder {
String indexFieldName = indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY);
if (indexFieldName == null) {
String msg = String.format("There is no index field name defined for attribute '%s' for entity '%s'",
Constants.STATE_PROPERTY_KEY,
entityType.getTypeName());
String msg = String.format("There is no index field name defined for attribute '%s'",
Constants.STATE_PROPERTY_KEY);
LOG.error(msg);
......@@ -187,10 +179,46 @@ public class AtlasSolrQueryBuilder {
private AtlasSolrQueryBuilder withCriteria(StringBuilder queryBuilder, FilterCriteria criteria) throws AtlasBaseException {
List<FilterCriteria> criterion = criteria.getCriterion();
Set<String> indexAttributes = new HashSet<>();
if (StringUtils.isNotEmpty(criteria.getAttributeName()) && CollectionUtils.isEmpty(criterion)) { // no child criterion
String attributeName = criteria.getAttributeName();
String attributeValue = criteria.getAttributeValue();
Operator operator = criteria.getOperator();
ArrayList<StringBuilder> orExpQuery = new ArrayList<>();
for (AtlasEntityType type : entityTypes) {
String indexAttributeName = getIndexAttributeName(type, attributeName);
//check to remove duplicate attribute query (for eg. name)
if (!indexAttributes.contains(indexAttributeName)) {
StringBuilder sb = new StringBuilder();
if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works
operator = Operator.EQ;
attributeValue = getIndexQueryAttributeValue(attributeValue);
}
withPropertyCondition(sb, indexAttributeName, operator, attributeValue);
indexAttributes.add(indexAttributeName);
orExpQuery.add(sb);
}
}
if (CollectionUtils.isNotEmpty(orExpQuery)) {
if (orExpQuery.size() > 1) {
String orExpStr = StringUtils.join(orExpQuery, FilterCriteria.Condition.OR.name());
queryBuilder.append(" ( ").append(orExpStr).append(" ) ");
} else {
queryBuilder.append(orExpQuery.iterator().next());
}
}
if(criterion == null || CollectionUtils.isEmpty(criteria.getCriterion())) { // no child criterion
withPropertyCondition(queryBuilder, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
} else {
} else if (CollectionUtils.isNotEmpty(criterion)) {
beginCriteria(queryBuilder);
for (Iterator<FilterCriteria> iterator = criterion.iterator(); iterator.hasNext(); ) {
......@@ -209,40 +237,12 @@ public class AtlasSolrQueryBuilder {
return this;
}
private void withPropertyCondition(StringBuilder queryBuilder, String attributeName, Operator operator, String attributeValue) throws AtlasBaseException {
if (StringUtils.isNotEmpty(attributeName) && operator != null) {
private void withPropertyCondition(StringBuilder queryBuilder, String indexFieldName, Operator operator, String attributeValue) throws AtlasBaseException {
if (StringUtils.isNotEmpty(indexFieldName) && operator != null) {
if (attributeValue != null) {
attributeValue = attributeValue.trim();
}
if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works
operator = Operator.EQ;
attributeValue = getIndexQueryAttributeValue(attributeValue);
}
AtlasAttribute attribute = entityType.getAttribute(attributeName);
if (attribute == null) {
String msg = String.format("Received unknown attribute '%s' for type '%s'.", attributeName, entityType.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
String indexFieldName = attribute.getIndexFieldName();
if (indexFieldName == null) {
String msg = String.format("Received non-index attribute %s for type %s.", attributeName, entityType.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
beginCriteria(queryBuilder);
switch (operator) {
......@@ -308,6 +308,29 @@ public class AtlasSolrQueryBuilder {
return attributeValue;
}
private String getIndexAttributeName(AtlasEntityType type, String attrName) throws AtlasBaseException {
AtlasAttribute ret = type.getAttribute(attrName);
if (ret == null) {
String msg = String.format("Received unknown attribute '%s' for type '%s'.", attrName, type.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
String indexFieldName = ret.getIndexFieldName();
if (indexFieldName == null) {
String msg = String.format("Received non-index attribute %s for type %s.", attrName, type.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
return indexFieldName;
}
private void beginCriteria(StringBuilder queryBuilder) {
queryBuilder.append("( ");
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
......@@ -33,8 +33,10 @@ import org.testng.annotations.Test;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.mockito.Mockito.when;
......@@ -45,6 +47,9 @@ public class AtlasSolrQueryBuilderTest {
private AtlasEntityType hiveTableEntityTypeMock;
@Mock
private AtlasEntityType hiveTableEntityTypeMock2;
@Mock
private AtlasStructType.AtlasAttribute nameAttributeMock;
@Mock
......@@ -95,6 +100,7 @@ public class AtlasSolrQueryBuilderTest {
when(hiveTableEntityTypeMock.getTypeName()).thenReturn("hive_table");
when(hiveTableEntityTypeMock2.getTypeName()).thenReturn("hive_db");
when(nameAttributeMock.getIndexFieldName()).thenReturn("name_index");
when(commentAttributeMock.getIndexFieldName()).thenReturn("comment_index");
......@@ -114,7 +120,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )");
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )");
}
@Test
......@@ -124,7 +130,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
}
@Test
......@@ -134,7 +140,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )");
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )");
}
@Test
......@@ -144,7 +150,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
}
@Test
......@@ -154,7 +160,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
}
@Test
......@@ -164,7 +170,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )");
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )");
}
@Test
......@@ -174,7 +180,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )");
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )");
}
@Test
......@@ -184,7 +190,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )");
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )");
}
@Test
......@@ -194,7 +200,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )");
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )");
}
@Test
......@@ -204,7 +210,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )");
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )");
}
@Test
......@@ -214,9 +220,18 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )");
Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )");
}
@Test
public void testGenerateSolrQueryString2TypeNames() throws IOException, AtlasBaseException {
final String fileName = "src/test/resources/searchparameters2Types.json";
AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
processSearchParametersForMultipleTypeNames(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table hive_db ) ");
}
......@@ -243,8 +258,23 @@ public class AtlasSolrQueryBuilderTest {
ObjectMapper mapper = new ObjectMapper();
SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class);
Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
underTest.withEntityTypes(hiveTableEntityTypeMocks)
.withQueryString(searchParameters.getQuery())
.withCriteria(searchParameters.getEntityFilters())
.withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
.withCommonIndexFieldNames(indexFieldNamesMap);
}
private void processSearchParametersForMultipleTypeNames(String fileName, AtlasSolrQueryBuilder underTest) throws IOException, AtlasBaseException {
ObjectMapper mapper = new ObjectMapper();
SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class);
underTest.withEntityType(hiveTableEntityTypeMock)
Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock2);
underTest.withEntityTypes(hiveTableEntityTypeMocks)
.withQueryString(searchParameters.getQuery())
.withCriteria(searchParameters.getEntityFilters())
.withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
......
graphdb/janus/src/test/resources/searchparameters2Types.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t",
"limit":25,
"offset":0,
"typeName":"hive_table,hive_db",
"classification":null,
"termName":null
}
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
......@@ -67,17 +67,16 @@ public class ClassificationSearchProcessor extends SearchProcessor {
public ClassificationSearchProcessor(SearchContext context) {
super(context);
final AtlasClassificationType classificationType = context.getClassificationType();
final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getClassificationTypes();
final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
final boolean isBuiltInType = context.isBuiltInClassificationType();
final boolean isWildcardSearch = context.isWildCardSearch();
final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
processSearchAttributes(classificationTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes);
/* for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
There are four cases in the classification type :
......@@ -87,19 +86,25 @@ public class ClassificationSearchProcessor extends SearchProcessor {
4. classification is not present in the search parameter
each of above cases with either has empty/or not tagFilters
*/
final boolean useIndexSearchForEntity = (classificationType != null || isWildcardSearch) &&
final boolean useIndexSearchForEntity = (CollectionUtils.isNotEmpty(classificationTypes) || isWildcardSearch) &&
!context.hasAttributeFilter(filterCriteria) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
/* If classification's attributes can be applied index filter, we can use direct index
* to query classification index as well.
*/
final boolean useIndexSearchForClassification = (!isBuiltInType && !isWildcardSearch) &&
final boolean useIndexSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
!isWildcardSearch) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
CollectionUtils.isNotEmpty(indexAttributes) &&
canApplyIndexFilter(classificationType, filterCriteria, false);
canApplyIndexFilter(classificationTypes, filterCriteria, false);
traitPredicate = buildTraitPredict(classificationType);
final boolean useGraphSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
!isWildcardSearch && CollectionUtils.isNotEmpty(graphAttributes));
traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
AtlasGraph graph = context.getGraph();
......@@ -115,8 +120,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
// tagFilters is not allowed in wildcard search
graphIndexQueryBuilder.addClassificationTypeFilter(queryString);
} else {
if (isBuiltInType) {
if (classificationTypes.iterator().next() == SearchContext.MATCH_ALL_NOT_CLASSIFIED) {
// tagFilters is not allowed in unique classificationType search
graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString);
......@@ -146,7 +150,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString, typeAndSubTypesQryStr);
constructFilterQuery(queryString, classificationType, filterCriteria, indexAttributes);
constructFilterQuery(queryString, classificationTypes, filterCriteria, indexAttributes);
String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
......@@ -158,7 +162,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
}
Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
Predicate attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
......@@ -170,7 +174,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
// only registered classification will search with tag filters
if (!isWildcardSearch && !isBuiltInType && !graphAttributes.isEmpty()) {
if (useGraphSearchForClassification) {
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
AtlasGraphQuery query = graph.query();
......@@ -179,7 +183,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
}
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationType, filterCriteria, allAttributes, query);
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
gremlinQueryBindings = new HashMap<>();
StringBuilder gremlinQuery = new StringBuilder();
......@@ -188,7 +192,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
gremlinQuery.append(".as('e').filter(out()");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
// constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
// After filtering on tags go back to e and output the list of entity vertices
gremlinQuery.append(").toList()");
......
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
......@@ -486,9 +486,11 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
resultAttributes.addAll(searchContext.getEntityAttributes());
}
AtlasEntityType entityType = searchContext.getEntityType();
if (entityType != null) {
for (String resultAttribute : resultAttributes) {
if (CollectionUtils.isNotEmpty(searchContext.getEntityTypes())) {
AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
for (String resultAttribute : resultAttributes) {
AtlasAttribute attribute = entityType.getAttribute(resultAttribute);
if (attribute == null) {
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
......@@ -31,27 +31,15 @@ import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.stream.StreamSupport;
import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.discovery.SearchContext.*;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
......@@ -69,28 +57,28 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) {
super(context);
final AtlasEntityType entityType = context.getEntityType();
final Set<AtlasEntityType> entityTypes = context.getEntityTypes();
final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getEntityTypes();
final Set<String> typeAndSubTypes = context.getEntityTypeNames();
final String typeAndSubTypesQryStr = context.getEntityTypesQryStr();
final String sortBy = context.getSearchParameters().getSortBy();
final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
final AtlasClassificationType classificationType = context.getClassificationType();
final Set<String> classificationTypeAndSubTypes = context.getClassificationTypes();
final boolean filterClassification;
final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
final Set<String> classificationTypeAndSubTypes = context.getClassificationTypeNames();
final boolean filterClassification;
if (classificationType != null) {
if (CollectionUtils.isNotEmpty(classificationTypes)) {
filterClassification = !context.needClassificationProcessor();
} else {
filterClassification = false;
}
final Predicate typeNamePredicate;
final Predicate traitPredicate = buildTraitPredict(classificationType);
final Predicate traitPredicate = buildTraitPredict(classificationTypes);
final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
......@@ -101,10 +89,10 @@ public class EntitySearchProcessor extends SearchProcessor {
typeNamePredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
}
processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
processSearchAttributes(entityTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes);
final boolean typeSearchByIndex = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES;
final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityType, filterCriteria, false);
final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityTypes, filterCriteria, false);
StringBuilder indexQuery = new StringBuilder();
......@@ -116,9 +104,9 @@ public class EntitySearchProcessor extends SearchProcessor {
}
if (attrSearchByIndex) {
constructFilterQuery(indexQuery, entityType, filterCriteria, indexAttributes);
constructFilterQuery(indexQuery, entityTypes, filterCriteria, indexAttributes);
Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, indexAttributes);
Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
......@@ -149,6 +137,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// If we need to filter on the trait names then we need to build the query and equivalent in-memory predicate
if (filterClassification) {
AtlasClassificationType classificationType = classificationTypes.iterator().next();
List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
......@@ -176,10 +165,10 @@ public class EntitySearchProcessor extends SearchProcessor {
}
}
graphQuery = toGraphFilterQuery(entityType, filterCriteria, graphAttributes, query);
graphQuery = toGraphFilterQuery(entityTypes, filterCriteria, graphAttributes, query);
// Prepare in-memory predicate for attribute filtering
Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, graphAttributes);
Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, graphAttributes);
if (attributePredicate != null) {
if (graphQueryPredicate != null) {
......@@ -199,7 +188,8 @@ public class EntitySearchProcessor extends SearchProcessor {
}
}
if (sortBy != null && !sortBy.isEmpty()) {
AtlasAttribute sortByAttribute = context.getEntityType().getAttribute(sortBy);
final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute != null) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
......@@ -215,8 +205,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// Prepare the graph query and in-memory filter for the filtering phase
filterGraphQueryPredicate = typeNamePredicate;
Predicate attributesPredicate = constructInMemoryPredicate(entityType, filterCriteria, allAttributes);
Predicate attributesPredicate = constructInMemoryPredicate(entityTypes, filterCriteria, allAttributes);
if (attributesPredicate != null) {
filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate :
......@@ -265,7 +254,7 @@ public class EntitySearchProcessor extends SearchProcessor {
SortOrder sortOrder = context.getSearchParameters().getSortOrder();
String sortBy = context.getSearchParameters().getSortBy();
final AtlasEntityType entityType = context.getEntityType();
final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute == null) {
sortBy = null;
......
repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
......@@ -18,7 +18,6 @@
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.*;
......@@ -53,13 +52,13 @@ public class FreeTextSearchProcessor extends SearchProcessor {
queryString.append(searchParameters.getQuery());
if (CollectionUtils.isNotEmpty(context.getEntityTypes()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
if (CollectionUtils.isNotEmpty(context.getEntityTypeNames()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getEntityTypesQryStr());
}
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
if (CollectionUtils.isNotEmpty(context.getClassificationTypeNames()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getClassificationTypesQryStr());
}
......@@ -138,7 +137,7 @@ public class FreeTextSearchProcessor extends SearchProcessor {
continue;
}
if (context.getClassificationType() != null) {
if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
List<String> entityClassifications = GraphHelper.getAllTraitNames(vertex);
if (!context.includeClassificationTypes(entityClassifications)) {
......
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
......@@ -24,6 +24,7 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -31,10 +32,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_ENTITY_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
public class FullTextSearchProcessor extends SearchProcessor {
......@@ -53,29 +51,28 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
if (context.getEntityType() != null && context.getEntityType() != MATCH_ALL_ENTITY_TYPES) {
String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr();
if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
String typeAndSubTypeNamesQryStr = context.getEntityTypesQryStr();
if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
} else {
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length());
searchParameters.getTypeName(), typeAndSubTypeNamesQryStr.length());
}
}
// if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if (context.getClassificationType() != null && context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION &&
context.getClassificationType() != MATCH_ALL_CLASSIFIED &&
context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) {
String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) &&
context.getClassificationTypes().iterator().next() != MATCH_ALL_NOT_CLASSIFIED) {
String typeAndSubTypeNamesStr = context.getClassificationTypesQryStr();
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
} else {
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length());
searchParameters.getClassification(), typeAndSubTypeNamesStr.length());
}
}
......
repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
......@@ -26,6 +26,8 @@ import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NA
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.type.AtlasStructType;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
public class GraphIndexQueryBuilder {
......@@ -36,20 +38,20 @@ public class GraphIndexQueryBuilder {
}
void addClassificationTypeFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
String classificationName = context.getSearchParameters().getClassification();
if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationNames())) {
String classificationNames = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(context.getClassificationNames());
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName)
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationNames)
.append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
.append('\"').append(':').append(classificationName).append(")");
.append('\"').append(':').append(classificationNames).append(")");
}
}
void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
String classificationTypesQryStr = context.getClassificationTypesQryStr();
if (indexQuery.length() != 0) {
......@@ -63,17 +65,8 @@ public class GraphIndexQueryBuilder {
}
void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) {
if (indexQuery != null && context.getClassificationType() != null) {
if (context.getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() == MATCH_ALL_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"")
.append(CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]")
.append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"")
.append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]").append(")");
} else if (context.getClassificationType() == MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
if (context.getClassificationTypes().iterator().next() == MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
......
repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
......@@ -56,12 +56,7 @@ public class SearchAggregatorImpl implements SearchAggregator {
try {
AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
String searchedOnTypeName = searchParameters.getTypeName();
AtlasEntityType searchForEntityType = null;
if (searchedOnTypeName != null) {
searchForEntityType = typeRegistry.getEntityTypeByName(searchedOnTypeName);
}
Set<AtlasEntityType> searchForEntityType = searchContext.getEntityTypes();
Map<String, String> indexFieldNameCache = new HashMap<>();
......
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
......@@ -21,7 +21,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.discovery.SearchParameters.*;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.repository.Constants;
......@@ -44,13 +44,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.stream.Collectors;
import static org.apache.atlas.discovery.SearchProcessor.ALL_TYPE_QUERY;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.*;
/*
* Search context captures elements required for performing a basic search
......@@ -62,12 +59,12 @@ public class SearchContext {
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
private final AtlasEntityType entityType;
private final Set<AtlasEntityType> entityTypes;
private final Set<String> indexedKeys;
private final Set<String> entityAttributes;
private final SearchParameters searchParameters;
private final AtlasClassificationType classificationType;
private final String classificationName;
private final Set<AtlasClassificationType> classificationTypes;
private final Set<String> classificationNames;
private final Set<String> typeAndSubTypes;
private final Set<String> classificationTypeAndSubTypes;
private final String typeAndSubTypesQryStr;
......@@ -80,27 +77,18 @@ public class SearchContext {
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot();
public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot();
public final static String TYPENAME_DELIMITER = ",";
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
this.classificationName = searchParameters.getClassification();
this.searchParameters = searchParameters;
this.typeRegistry = typeRegistry;
this.graph = graph;
this.indexedKeys = indexedKeys;
this.entityAttributes = new HashSet<>();
this.entityType = getEntityType(searchParameters.getTypeName());
this.classificationType = getClassificationType(classificationName);
// Validate if the type name exists
if (StringUtils.isNotEmpty(searchParameters.getTypeName()) && entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, searchParameters.getTypeName());
}
// Validate if the classification exists
if ((StringUtils.isNotEmpty(classificationName) && classificationType == null && !classificationName.contains(WILDCARD_CLASSIFICATIONS))) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName);
}
this.entityTypes = getEntityTypes(searchParameters.getTypeName());
this.classificationNames = getClassificationNames(searchParameters.getClassification());
this.classificationTypes = getClassificationTypes(this.classificationNames);
AtlasVertex glossaryTermVertex = getGlossaryTermVertex(searchParameters.getTermName());
......@@ -109,40 +97,77 @@ public class SearchContext {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_GLOSSARY_TERM, searchParameters.getTermName());
}
// Invalid attributes will raise an exception with 400 error code
validateAttributes(entityType, searchParameters.getEntityFilters());
// Invalid attributes or unsupported attribute in a type, will raise an exception with 400 error code
if (CollectionUtils.isNotEmpty(entityTypes)) {
for (AtlasEntityType entityType : entityTypes) {
validateAttributes(entityType, searchParameters.getEntityFilters());
// Invalid attribute will raise an exception with 400 error code
validateAttributes(entityType, searchParameters.getSortBy());
validateAttributes(entityType, searchParameters.getSortBy());
}
}
// Invalid attributes will raise an exception with 400 error code
validateAttributes(classificationType, searchParameters.getTagFilters());
if (CollectionUtils.isNotEmpty(classificationTypes)) {
for (AtlasClassificationType classificationType : classificationTypes) {
validateAttributes(classificationType, searchParameters.getTagFilters());
}
}
if (classificationType != null && !isBuiltInClassificationType()) {
if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
} else {
classificationTypeAndSubTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName());
classificationTypeAndSubTypesQryStr = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypesQryStr() : classificationType.getTypeQryStr();
//remove other types if builtin type is present
filterStructTypes();
//gather all classifications and its corresponding subtypes
Set<String> classificationTypeAndSubTypes = new HashSet<>();
String classificationTypeAndSubTypesQryStr = null;
if (CollectionUtils.isNotEmpty(classificationTypes) && classificationTypes.iterator().next() != MATCH_ALL_NOT_CLASSIFIED ) {
for (AtlasClassificationType classificationType : classificationTypes) {
if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
break;
} else {
Set<String> allTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName());
classificationTypeAndSubTypes.addAll(allTypes); }
}
if (CollectionUtils.isNotEmpty(classificationTypeAndSubTypes)) {
classificationTypeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(classificationTypeAndSubTypes);
}
} else {
classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = "";
}
this.classificationTypeAndSubTypes = classificationTypeAndSubTypes;
this.classificationTypeAndSubTypesQryStr = classificationTypeAndSubTypesQryStr;
//gather all types and its corresponding subtypes
Set<String> typeAndSubTypes = new HashSet<>();
String typeAndSubTypesQryStr = null;
if (CollectionUtils.isNotEmpty(entityTypes)) {
for (AtlasEntityType entityType : entityTypes) {
if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = ALL_TYPE_QUERY;
break;
} else {
Set<String> allTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
typeAndSubTypes.addAll(allTypes);
}
}
if (entityType != null) {
if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = ALL_TYPE_QUERY;
} else {
typeAndSubTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
typeAndSubTypesQryStr = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypesQryStr() : entityType.getTypeQryStr();
if (CollectionUtils.isNotEmpty(typeAndSubTypes)) {
typeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndSubTypes);
}
} else {
typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = "";
}
this.typeAndSubTypes = typeAndSubTypes;
this.typeAndSubTypesQryStr = typeAndSubTypesQryStr;
if (glossaryTermVertex != null) {
addProcessor(new TermSearchProcessor(this, getAssignedEntities(glossaryTermVertex)));
......@@ -179,37 +204,37 @@ public class SearchContext {
public Set<String> getEntityAttributes() { return entityAttributes; }
public AtlasEntityType getEntityType() { return entityType; }
public Set<AtlasClassificationType> getClassificationTypes() { return classificationTypes; }
public AtlasClassificationType getClassificationType() { return classificationType; }
public Set<String> getEntityTypeNames() { return typeAndSubTypes; }
public Set<String> getEntityTypes() { return typeAndSubTypes; }
public Set<String> getClassificationTypes() { return classificationTypeAndSubTypes; }
public Set<String> getClassificationTypeNames() { return classificationTypeAndSubTypes; }
public String getEntityTypesQryStr() { return typeAndSubTypesQryStr; }
public String getClassificationTypesQryStr() { return classificationTypeAndSubTypesQryStr; }
public Set<AtlasEntityType> getEntityTypes() { return entityTypes; }
public SearchProcessor getSearchProcessor() { return searchProcessor; }
public String getClassificationName() {return classificationName;}
public Set<String> getClassificationNames() {return classificationNames;}
public boolean includeEntityType(String entityType) {
return typeAndSubTypes.isEmpty() || typeAndSubTypes.contains(entityType);
}
public boolean includeClassificationTypes(Collection<String> classificationTypes) {
public boolean includeClassificationTypes(Collection<String> traitNames) {
final boolean ret;
if (classificationType == null || classificationTypeAndSubTypes.isEmpty()) {
if (CollectionUtils.isEmpty(classificationTypes) || classificationTypeAndSubTypes.isEmpty()) {
ret = true;
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
ret = CollectionUtils.isEmpty(classificationTypes);
} else if (classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION) {
ret = CollectionUtils.isNotEmpty(classificationTypes);
} else if (classificationTypes.iterator().next() == MATCH_ALL_NOT_CLASSIFIED) {
ret = CollectionUtils.isEmpty(traitNames);
} else if (classificationTypes.iterator().next() == MATCH_ALL_CLASSIFICATION_TYPES) {
ret = CollectionUtils.isNotEmpty(traitNames);
} else {
ret = CollectionUtils.containsAny(classificationTypeAndSubTypes, classificationTypes);
ret = CollectionUtils.containsAny(classificationTypeAndSubTypes, traitNames);
}
return ret;
......@@ -243,25 +268,18 @@ public class SearchContext {
}
boolean needClassificationProcessor() {
return (classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
}
boolean isBuiltInClassificationType() {
return getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION
|| getClassificationType() == MATCH_ALL_CLASSIFIED
|| getClassificationType() == MATCH_ALL_NOT_CLASSIFIED;
return (CollectionUtils.isNotEmpty(classificationTypes) && (CollectionUtils.isEmpty(entityTypes) || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
}
boolean isWildCardSearch () {
String classification = getSearchParameters().getClassification();
if (StringUtils.isNotEmpty(classification) && getClassificationType() == null) {
return classification.contains("*");
if (CollectionUtils.isNotEmpty(classificationNames)) {
return classificationNames.stream().anyMatch(classification -> classification.contains(WILDCARD_CLASSIFICATIONS));
}
return false;
}
boolean needEntityProcessor() {
return entityType != null;
return CollectionUtils.isNotEmpty(entityTypes);
}
private void validateAttributes(final AtlasStructType structType, final FilterCriteria filterCriteria) throws AtlasBaseException {
......@@ -285,7 +303,14 @@ public class SearchContext {
if (structType == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL");
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, structType.getTypeName());
String name = structType.getTypeName();
if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
name = ALL_ENTITY_TYPES;
} else if (name.equals(MATCH_ALL_CLASSIFICATION_TYPES.getTypeName())) {
name = ALL_CLASSIFICATION_TYPES;
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, name);
}
}
}
......@@ -321,11 +346,107 @@ public class SearchContext {
return ret;
}
private Set<AtlasClassificationType> getClassificationTypes(Set<String> classificationNames) {
if (CollectionUtils.isNotEmpty(classificationNames)) {
return classificationNames.stream().map(n ->
getClassificationType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
}
return null;
}
private Set<String> getClassificationNames(String classification) throws AtlasBaseException {
Set<String> classificationNames = new HashSet<>();
if (StringUtils.isNotEmpty(classification)) {
String[] types = classification.split(TYPENAME_DELIMITER);
Set<String> names = new HashSet<>(Arrays.asList(types));
names.forEach(name -> {
AtlasClassificationType type = getClassificationType(name);
if (type != null || name.contains(WILDCARD_CLASSIFICATIONS)) {
classificationNames.add(name);
}
});
// Validate if the classification exists
if (CollectionUtils.isEmpty(classificationNames) || classificationNames.size() != names.size()) {
if (CollectionUtils.isNotEmpty(classificationNames)) {
names.removeAll(classificationNames);
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, String.join(TYPENAME_DELIMITER, names));
}
}
return classificationNames;
}
private AtlasEntityType getEntityType(String entityName) {
return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ? MATCH_ALL_ENTITY_TYPES :
typeRegistry.getEntityTypeByName(entityName);
}
private Set<AtlasEntityType> getEntityTypes(String typeName) throws AtlasBaseException {
Set<AtlasEntityType> entityTypes = null;
//split multiple typeNames by comma
if (StringUtils.isNotEmpty(typeName)) {
String[] types = typeName.split(TYPENAME_DELIMITER);
Set<String> typeNames = new HashSet<>(Arrays.asList(types));
entityTypes = typeNames.stream().map(n ->
getEntityType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
// Validate if the type name is incorrect
if (CollectionUtils.isEmpty(entityTypes) || entityTypes.size() != typeNames.size()) {
if (CollectionUtils.isNotEmpty(entityTypes)) {
Set<String> validEntityTypes = new HashSet<>();
for (AtlasEntityType entityType : entityTypes) {
String name = entityType.getTypeName();
if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
validEntityTypes.add(ALL_ENTITY_TYPES);
continue;
}
validEntityTypes.add(entityType.getTypeName());
}
typeNames.removeAll(validEntityTypes);
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, String.join(TYPENAME_DELIMITER, typeNames));
}
}
return entityTypes;
}
private void filterStructTypes(){
//if typeName contains ALL_ENTITY_TYPES, remove others as OR condition will not effect any other
if (CollectionUtils.isNotEmpty(entityTypes) && entityTypes.contains(MATCH_ALL_ENTITY_TYPES)) {
entityTypes.clear();
entityTypes.add(MATCH_ALL_ENTITY_TYPES);
}
//No Builtin Classification can be together
if (CollectionUtils.isNotEmpty(classificationTypes)) {
if (classificationTypes.contains(MATCH_ALL_NOT_CLASSIFIED)) {
classificationTypes.clear();
classificationTypes.add(MATCH_ALL_NOT_CLASSIFIED);
classificationNames.clear();
classificationNames.add(MATCH_ALL_NOT_CLASSIFIED.getTypeName());
} else if (classificationTypes.contains(MATCH_ALL_WILDCARD_CLASSIFICATION) || classificationTypes.contains(MATCH_ALL_CLASSIFICATION_TYPES) || classificationTypes.contains(MATCH_ALL_CLASSIFIED)) {
classificationTypes.clear();
classificationTypes.add(MATCH_ALL_CLASSIFICATION_TYPES);
classificationNames.clear();
classificationNames.add(ALL_CLASSIFICATION_TYPES);
}
}
}
private AtlasVertex getGlossaryTermVertex(String termName) {
AtlasVertex ret = null;
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
......@@ -51,16 +51,8 @@ import java.util.regex.Pattern;
import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.util.SearchPredicateUtil.*;
public abstract class SearchProcessor {
......@@ -153,11 +145,19 @@ public abstract class SearchProcessor {
public abstract long getResultCount();
protected boolean isEntityRootType() {
return context.getEntityType() == SearchContext.MATCH_ALL_ENTITY_TYPES;
//always size will be one if in case of _ALL_ENTITY_TYPES
if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
return context.getEntityTypes().iterator().next() == SearchContext.MATCH_ALL_ENTITY_TYPES;
}
return false;
}
protected boolean isClassificationRootType() {
return context.getClassificationType() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
//always size will be one if in case of _ALL_CLASSIFICATION_TYPES
if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
return context.getClassificationTypes().iterator().next() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
}
return false;
}
protected boolean isSystemAttribute(String attrName) {
......@@ -195,9 +195,14 @@ public abstract class SearchProcessor {
}
}
protected Predicate buildTraitPredict(AtlasClassificationType classificationType) {
protected Predicate buildTraitPredict(Set<AtlasClassificationType> classificationTypes) {
Predicate traitPredicate;
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
AtlasClassificationType classificationType = null;
if (CollectionUtils.isNotEmpty(classificationTypes)) {
classificationType = classificationTypes.iterator().next();
}
if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
......@@ -205,20 +210,27 @@ public abstract class SearchProcessor {
SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (context.isWildCardSearch()) {
//For wildcard search __classificationNames which of String type is taken instead of _traitNames which is of Array type
//No need to escape, as classification Names only support letters,numbers,space and underscore
String regexString = getRegexString("\\|" + context.getClassificationName() + "\\|");
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, regexString, String.class),
SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, regexString, String.class));
Set<String> classificationNames = context.getClassificationNames();
List<Predicate> predicates = new ArrayList<>();
classificationNames.forEach(classificationName -> {
//No need to escape, as classification Names only support letters,numbers,space and underscore
String regexString = getRegexString("\\|" + classificationName + "\\|");
predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, regexString, String.class));
predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, regexString, String.class));
});
traitPredicate = PredicateUtils.anyPredicate(predicates);
} else {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class));
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class));
}
return traitPredicate;
}
protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) {
if (structType == null || filterCriteria == null) {
protected void processSearchAttributes(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) {
if (CollectionUtils.isEmpty(structTypes) || filterCriteria == null) {
return;
}
......@@ -227,7 +239,7 @@ public abstract class SearchProcessor {
if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
for (SearchParameters.FilterCriteria criteria : criterion) {
processSearchAttributes(structType, criteria, indexFiltered, graphFiltered, allAttributes);
processSearchAttributes(structTypes, criteria, indexFiltered, graphFiltered, allAttributes);
}
} else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
String attributeName = filterCriteria.getAttributeName();
......@@ -267,20 +279,23 @@ public abstract class SearchProcessor {
}
try {
if (isIndexSearchable(filterCriteria, structType)) {
indexFiltered.add(attributeName);
} else {
LOG.warn("not using index-search for attribute '{}'; might cause poor performance", structType.getVertexPropertyName(attributeName));
for (AtlasStructType structType : structTypes) {
String qualifiedName = structType.getVertexPropertyName(attributeName);
if (isIndexSearchable(filterCriteria, structType)) {
indexFiltered.add(qualifiedName);
} else {
LOG.warn("not using index-search for attribute '{}'; might cause poor performance", structType.getVertexPropertyName(attributeName));
graphFiltered.add(attributeName);
}
graphFiltered.add(qualifiedName);
}
if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) {
// Capture the entity attributes
context.getEntityAttributes().add(attributeName);
}
if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) {
// Capture the entity attributes
context.getEntityAttributes().add(attributeName);
}
allAttributes.add(attributeName);
allAttributes.add(qualifiedName);
}
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
}
......@@ -298,7 +313,7 @@ public abstract class SearchProcessor {
// (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z)
// (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z (AND idx-attr2=xyz idx-attr2=abc))
//
protected boolean canApplyIndexFilter(AtlasStructType structType, FilterCriteria filterCriteria, boolean insideOrCondition) {
protected boolean canApplyIndexFilter(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, boolean insideOrCondition) {
if (!context.hasAttributeFilter(filterCriteria)) {
return true;
}
......@@ -314,7 +329,7 @@ public abstract class SearchProcessor {
// If we have nested criterion let's find any nested ORs with non-indexed attr
for (FilterCriteria criteria : criterion) {
ret = canApplyIndexFilter(structType, criteria, insideOrCondition);
ret = canApplyIndexFilter(structTypes, criteria, insideOrCondition);
if (!ret) {
break;
......@@ -322,10 +337,11 @@ public abstract class SearchProcessor {
}
} else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
try {
if (insideOrCondition && !isIndexSearchable(filterCriteria, structType)) {
ret = false;
for (AtlasStructType structType : structTypes) {
if (insideOrCondition && !isIndexSearchable(filterCriteria, structType)) {
ret = false;
break;
}
}
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
......@@ -338,7 +354,7 @@ public abstract class SearchProcessor {
protected void filterWhiteSpaceClassification(List<AtlasVertex> entityVertices) {
if (CollectionUtils.isNotEmpty(entityVertices)) {
final Iterator<AtlasVertex> it = entityVertices.iterator();
final Set<String> typeAndSubTypes = context.getClassificationTypes();
final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
while (it.hasNext()) {
AtlasVertex entityVertex = it.next();
......@@ -363,13 +379,13 @@ public abstract class SearchProcessor {
}
}
protected void constructFilterQuery(StringBuilder indexQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> indexAttributes) {
protected void constructFilterQuery(StringBuilder indexQuery, Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) {
if (filterCriteria != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing Filters");
}
String filterQuery = toIndexQuery(type, filterCriteria, indexAttributes, 0);
String filterQuery = toIndexQuery(structTypes, filterCriteria, indexAttributes, 0);
if (StringUtils.isNotEmpty(filterQuery)) {
if (indexQuery.length() > 0) {
......@@ -381,14 +397,14 @@ public abstract class SearchProcessor {
}
}
protected Predicate constructInMemoryPredicate(AtlasStructType type, FilterCriteria filterCriteria, Set<String> indexAttributes) {
protected Predicate constructInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) {
Predicate ret = null;
if (filterCriteria != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing Filters");
}
ret = toInMemoryPredicate(type, filterCriteria, indexAttributes);
ret = toInMemoryPredicate(structTypes, filterCriteria, indexAttributes);
}
return ret;
}
......@@ -473,17 +489,20 @@ public abstract class SearchProcessor {
return ret;
}
private String toIndexQuery(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes, int level) {
return toIndexQuery(type, criteria, indexAttributes, new StringBuilder(), level);
private String toIndexQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes, int level) {
return toIndexQuery(structTypes, criteria, indexAttributes, new StringBuilder(), level);
}
private String toIndexQuery(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int level) {
private String toIndexQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int level) {
Set<String> filterAttributes = new HashSet<>();
filterAttributes.addAll(indexAttributes);
Condition condition = criteria.getCondition();
if (condition != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
StringBuilder nestedExpression = new StringBuilder();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
String nestedQuery = toIndexQuery(type, filterCriteria, indexAttributes, level + 1);
String nestedQuery = toIndexQuery(structTypes, filterCriteria, filterAttributes, level + 1);
if (StringUtils.isNotEmpty(nestedQuery)) {
if (nestedExpression.length() > 0) {
......@@ -502,19 +521,45 @@ public abstract class SearchProcessor {
} else {
return EMPTY_STRING;
}
} else if (indexAttributes.contains(criteria.getAttributeName())){
return toIndexExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
} else {
return EMPTY_STRING;
} else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
try {
ArrayList<String> orExpQuery = new ArrayList<>();
for (AtlasStructType structType : structTypes) {
String name = structType.getVertexPropertyName(criteria.getAttributeName());
if (filterAttributes.contains(name)) {
String nestedQuery = toIndexExpression(structType, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
orExpQuery.add(nestedQuery);
filterAttributes.remove(name);
}
}
if (CollectionUtils.isNotEmpty(orExpQuery)) {
if (orExpQuery.size() > 1) {
String orExpStr = StringUtils.join(orExpQuery, " "+Condition.OR.name()+" ");
return BRACE_OPEN_STR + " " + orExpStr + " " + BRACE_CLOSE_STR;
} else {
return orExpQuery.iterator().next();
}
} else {
return EMPTY_STRING;
}
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
}
}
return EMPTY_STRING;
}
private Predicate toInMemoryPredicate(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes) {
private Predicate toInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes) {
Set<String> filterAttributes = new HashSet<>();
filterAttributes.addAll(indexAttributes);
if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
List<Predicate> predicates = new ArrayList<>();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
Predicate predicate = toInMemoryPredicate(type, filterCriteria, indexAttributes);
Predicate predicate = toInMemoryPredicate(structTypes, filterCriteria, filterAttributes);
if (predicate != null) {
predicates.add(predicate);
......@@ -528,21 +573,40 @@ public abstract class SearchProcessor {
return PredicateUtils.anyPredicate(predicates);
}
}
} else if (indexAttributes.contains(criteria.getAttributeName())) {
String attrName = criteria.getAttributeName();
String attrValue = criteria.getAttributeValue();
SearchParameters.Operator operator = criteria.getOperator();
} else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
try {
ArrayList<Predicate> predicates = new ArrayList<>();
for (AtlasStructType structType : structTypes) {
String name = structType.getVertexPropertyName(criteria.getAttributeName());
if (filterAttributes.contains(name)) {
String attrName = criteria.getAttributeName();
String attrValue = criteria.getAttributeValue();
SearchParameters.Operator operator = criteria.getOperator();
//process attribute value and attribute operator for pipeSeperated fields
if (isPipeSeparatedSystemAttribute(attrName)) {
FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
attrValue = processedCriteria.getAttributeValue();
operator = processedCriteria.getOperator();
}
//process attribute value and attribute operator for pipeSeperated fields
if (isPipeSeparatedSystemAttribute(attrName)) {
FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
attrValue = processedCriteria.getAttributeValue();
operator = processedCriteria.getOperator();
}
predicates.add(toInMemoryPredicate(structType, attrName, operator, attrValue));
filterAttributes.remove(name);
}
}
return toInMemoryPredicate(type, attrName, operator, attrValue);
if (CollectionUtils.isNotEmpty(predicates)) {
if (predicates.size() > 1) {
return PredicateUtils.anyPredicate(predicates);
} else {
return predicates.iterator().next();
}
}
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
}
}
return null;
}
......@@ -704,12 +768,15 @@ public abstract class SearchProcessor {
return ret;
}
protected AtlasGraphQuery toGraphFilterQuery(AtlasStructType type, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) {
protected AtlasGraphQuery toGraphFilterQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) {
Set<String> filterAttributes = new HashSet<>();
filterAttributes.addAll(graphAttributes);
if (criteria != null) {
if (criteria.getCondition() != null) {
if (criteria.getCondition() == Condition.AND) {
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, graphAttributes, context.getGraph().query());
AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
query.addConditionsFrom(nestedQuery);
}
......@@ -717,7 +784,7 @@ public abstract class SearchProcessor {
List<AtlasGraphQuery> orConditions = new LinkedList<>();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, graphAttributes, context.getGraph().query());
AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery));
}
......@@ -726,60 +793,80 @@ public abstract class SearchProcessor {
query.or(orConditions);
}
}
} else if (graphAttributes.contains(criteria.getAttributeName())) {
String attrName = criteria.getAttributeName();
String attrValue = criteria.getAttributeValue();
SearchParameters.Operator operator = criteria.getOperator();
//process attribute value and attribute operator for pipeSeperated fields
if (isPipeSeparatedSystemAttribute(attrName)) {
FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
attrValue = processedCriteria.getAttributeValue();
operator = processedCriteria.getOperator();
}
final String qualifiedName = type.getAttribute(attrName).getVertexPropertyName();
} else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
try {
ArrayList<AtlasGraphQuery> queries = new ArrayList<>();
for (AtlasStructType structType : structTypes) {
String qualifiedName = structType.getVertexPropertyName(criteria.getAttributeName());
if (filterAttributes.contains(qualifiedName)) {
String attrName = criteria.getAttributeName();
String attrValue = criteria.getAttributeValue();
SearchParameters.Operator operator = criteria.getOperator();
//process attribute value and attribute operator for pipeSeperated fields
if (isPipeSeparatedSystemAttribute(attrName)) {
FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
attrValue = processedCriteria.getAttributeValue();
operator = processedCriteria.getOperator();
}
AtlasGraphQuery innerQry = context.getGraph().query().createChildQuery();
switch (operator) {
case LT:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
break;
case LTE:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
break;
case GT:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
break;
case GTE:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
break;
case EQ:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
break;
case NEQ:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
break;
case LIKE:
innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
break;
case CONTAINS:
innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
break;
case STARTS_WITH:
innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
break;
case ENDS_WITH:
innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
break;
case IS_NULL:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null);
break;
case NOT_NULL:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
break;
default:
LOG.warn("{}: unsupported operator. Ignored", operator);
break;
}
queries.add(context.getGraph().query().createChildQuery().addConditionsFrom(innerQry));
filterAttributes.remove(qualifiedName);
}
}
switch (operator) {
case LT:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
break;
case LTE:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
break;
case GT:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
break;
case GTE:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
break;
case EQ:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
break;
case NEQ:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
break;
case LIKE:
query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
break;
case CONTAINS:
query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
break;
case STARTS_WITH:
query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
break;
case ENDS_WITH:
query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
break;
case IS_NULL:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null);
break;
case NOT_NULL:
query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
break;
default:
LOG.warn("{}: unsupported operator. Ignored", operator);
break;
if (CollectionUtils.isNotEmpty(queries)) {
if (queries.size() > 1) {
return context.getGraph().query().createChildQuery().or(queries);
} else {
return queries.iterator().next();
}
}
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
}
}
}
......@@ -987,11 +1074,13 @@ public abstract class SearchProcessor {
}
private static String getSortByAttribute(SearchContext context) {
final AtlasEntityType entityType = context.getEntityType();
String sortBy = context.getSearchParameters().getSortBy();
AtlasStructType.AtlasAttribute sortByAttribute = entityType != null ? entityType.getAttribute(sortBy) : null;
if (sortByAttribute != null) {
return sortByAttribute.getVertexPropertyName();
if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
String sortBy = context.getSearchParameters().getSortBy();
AtlasStructType.AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute != null) {
return sortByAttribute.getVertexPropertyName();
}
}
return null;
}
......
repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
......@@ -130,7 +130,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
new EntitySearchProcessor(context);
}
@Test
@Test(priority = -1)
public void searchWithNEQ_stringAttr() throws AtlasBaseException {
String expectedEntityName = "hive_Table_Null_tableType";
createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
......@@ -197,4 +197,116 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertTrue(nameList.contains("hive_Table_Null_tableType"));
}
@Test
public void ALLEntityType() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 20);
}
@Test
public void ALLEntityTypeWithTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
params.setClassification(FACT_CLASSIFICATION);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 5);
}
@Test
public void entityType() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 3);
}
@Test
public void entityTypes() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 14);
}
@Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Not_Exists: Unknown/invalid typename")
public void entityTypesNotAllowed() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+",Not_Exists");
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
}
@Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Attribute tableType not found for type "+DATABASE_TYPE)
public void entityFiltersNotAllowed() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.CONTAINS, "ETL");
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
}
@Test
public void entityTypesAndTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
params.setClassification(FACT_CLASSIFICATION);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 3);
}
@Test
public void searchWithEntityTypesAndEntityFilters() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 4);
}
@Test
public void searchWithEntityTypesAndEntityFiltersAndTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
params.setEntityFilters(filterCriteria);
params.setClassification(LOGDATA_CLASSIFICATION);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 2);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment