Skip to content

A
atlas
Commit 8b5cb9d9 by Pinal Shah Committed by Sarath Subramanian
Options

ATLAS-3838: Support multiple tag/classification in basic/quick search API

ATLAS-3652: Quick Search: API requirement for GET request on multiple entity types Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org> both JIRA's addressed in the same commit.
parent e691a32c
Showing with 347 additions and 184 deletions
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
...@@ -25,14 +25,14 @@ import java.util.Map; ...@@ -25,14 +25,14 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
public class AggregationContext { public class AggregationContext {
private final String queryString; private final String queryString;
private final FilterCriteria filterCriteria; private final FilterCriteria filterCriteria;
private final AtlasEntityType searchForEntityType; private final Set<AtlasEntityType> searchForEntityTypes;
private final Set<String> aggregationFieldNames; private final Set<String> aggregationFieldNames;
private final Set<AtlasAttribute> aggregationAttributes; private final Set<AtlasAttribute> aggregationAttributes;
private final Map<String, String> indexFieldNameCache; private final Map<String, String> indexFieldNameCache;
private final boolean excludeDeletedEntities; private final boolean excludeDeletedEntities;
private final boolean includeSubTypes; private final boolean includeSubTypes;
/** /**
* @param queryString the query string whose aggregation metrics need to be retrieved. * @param queryString the query string whose aggregation metrics need to be retrieved.
...@@ -41,17 +41,17 @@ public class AggregationContext { ...@@ -41,17 +41,17 @@ public class AggregationContext {
* @param indexFieldNameCache * @param indexFieldNameCache
* @param excludeDeletedEntities a boolean flag to indicate if the deleted entities need to be excluded in search * @param excludeDeletedEntities a boolean flag to indicate if the deleted entities need to be excluded in search
*/ */
public AggregationContext(String queryString, public AggregationContext(String queryString,
FilterCriteria filterCriteria, FilterCriteria filterCriteria,
AtlasEntityType searchForEntityType, Set<AtlasEntityType> searchForEntityType,
Set<String> aggregationFieldNames, Set<String> aggregationFieldNames,
Set<AtlasAttribute> aggregationAttributes, Set<AtlasAttribute> aggregationAttributes,
Map<String, String> indexFieldNameCache, Map<String, String> indexFieldNameCache,
boolean excludeDeletedEntities, boolean excludeDeletedEntities,
boolean includeSubTypes) { boolean includeSubTypes) {
this.queryString = queryString; this.queryString = queryString;
this.filterCriteria = filterCriteria; this.filterCriteria = filterCriteria;
this.searchForEntityType = searchForEntityType; this.searchForEntityTypes = searchForEntityType;
this.aggregationFieldNames = aggregationFieldNames; this.aggregationFieldNames = aggregationFieldNames;
this.aggregationAttributes = aggregationAttributes; this.aggregationAttributes = aggregationAttributes;
this.indexFieldNameCache = indexFieldNameCache; this.indexFieldNameCache = indexFieldNameCache;
...@@ -67,8 +67,8 @@ public class AggregationContext { ...@@ -67,8 +67,8 @@ public class AggregationContext {
return filterCriteria; return filterCriteria;
} }
public AtlasEntityType getSearchForEntityType() { public Set<AtlasEntityType> getSearchForEntityTypes() {
return searchForEntityType; return searchForEntityTypes;
} }
public Set<String> getAggregationFieldNames() { public Set<String> getAggregationFieldNames() {
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
...@@ -175,7 +175,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient { ...@@ -175,7 +175,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
Map<String, String> indexFieldName2PropertyKeyNameMap = new HashMap<>(); Map<String, String> indexFieldName2PropertyKeyNameMap = new HashMap<>();
AtlasSolrQueryBuilder solrQueryBuilder = new AtlasSolrQueryBuilder(); AtlasSolrQueryBuilder solrQueryBuilder = new AtlasSolrQueryBuilder();
solrQueryBuilder.withEntityType(aggregationContext.getSearchForEntityType()) solrQueryBuilder.withEntityTypes(aggregationContext.getSearchForEntityTypes())
.withQueryString(aggregationContext.getQueryString()) .withQueryString(aggregationContext.getQueryString())
.withCriteria(aggregationContext.getFilterCriteria()) .withCriteria(aggregationContext.getFilterCriteria())
.withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities()) .withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities())
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
...@@ -29,31 +29,28 @@ import org.apache.commons.lang.StringUtils; ...@@ -29,31 +29,28 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.Iterator; import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
public class AtlasSolrQueryBuilder { public class AtlasSolrQueryBuilder {
private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class); private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class);
private AtlasEntityType entityType; private Set<AtlasEntityType> entityTypes;
private String queryString; private String queryString;
private FilterCriteria criteria; private FilterCriteria criteria;
private boolean excludeDeletedEntities; private boolean excludeDeletedEntities;
private boolean includeSubtypes; private boolean includeSubtypes;
private Map<String, String> indexFieldNameCache; private Map<String, String> indexFieldNameCache;
public static final char CUSTOM_ATTR_SEPARATOR = '='; public static final char CUSTOM_ATTR_SEPARATOR = '=';
public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\""; public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
public AtlasSolrQueryBuilder() { public AtlasSolrQueryBuilder() {
} }
public AtlasSolrQueryBuilder withEntityType(AtlasEntityType searchForEntityType) { public AtlasSolrQueryBuilder withEntityTypes(Set<AtlasEntityType> searchForEntityTypes) {
this.entityType = searchForEntityType; this.entityTypes = searchForEntityTypes;
return this; return this;
} }
...@@ -112,7 +109,7 @@ public class AtlasSolrQueryBuilder { ...@@ -112,7 +109,7 @@ public class AtlasSolrQueryBuilder {
isAndNeeded = true; isAndNeeded = true;
} }
if (entityType != null) { if (CollectionUtils.isNotEmpty(entityTypes)) {
if (isAndNeeded) { if (isAndNeeded) {
queryBuilder.append(" AND "); queryBuilder.append(" AND ");
} }
...@@ -140,29 +137,25 @@ public class AtlasSolrQueryBuilder { ...@@ -140,29 +137,25 @@ public class AtlasSolrQueryBuilder {
} }
private void buildForEntityType(StringBuilder queryBuilder) { private void buildForEntityType(StringBuilder queryBuilder) {
if (LOG.isDebugEnabled()) {
LOG.debug("Search is being done for entities of type {}", entityType.getTypeName());
}
String typeIndexFieldName = indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY); String typeIndexFieldName = indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY);
queryBuilder.append(" +") queryBuilder.append(" +")
.append(typeIndexFieldName) .append(typeIndexFieldName)
.append(":(") .append(":(");
.append(entityType.getTypeName())
.append(" ");
if (includeSubtypes) { Set<String> typesToSearch = new HashSet<>();
Set<String> allSubTypes = entityType.getAllSubTypes(); for (AtlasEntityType type : entityTypes) {
if(allSubTypes.size() != 0 ) { if (includeSubtypes) {
for(String subTypeName: allSubTypes) { typesToSearch.addAll(type.getTypeAndAllSubTypes());
queryBuilder.append(subTypeName).append(" "); } else {
} typesToSearch.add(type.getTypeName());
} }
} }
queryBuilder.append(" ) "); queryBuilder.append(StringUtils.join(typesToSearch, " ")).append(" ) ");
} }
private void dropDeletedEntities(StringBuilder queryBuilder) throws AtlasBaseException { private void dropDeletedEntities(StringBuilder queryBuilder) throws AtlasBaseException {
...@@ -173,9 +166,8 @@ public class AtlasSolrQueryBuilder { ...@@ -173,9 +166,8 @@ public class AtlasSolrQueryBuilder {
String indexFieldName = indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY); String indexFieldName = indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY);
if (indexFieldName == null) { if (indexFieldName == null) {
String msg = String.format("There is no index field name defined for attribute '%s' for entity '%s'", String msg = String.format("There is no index field name defined for attribute '%s'",
Constants.STATE_PROPERTY_KEY, Constants.STATE_PROPERTY_KEY);
entityType.getTypeName());
LOG.error(msg); LOG.error(msg);
...@@ -187,10 +179,46 @@ public class AtlasSolrQueryBuilder { ...@@ -187,10 +179,46 @@ public class AtlasSolrQueryBuilder {
private AtlasSolrQueryBuilder withCriteria(StringBuilder queryBuilder, FilterCriteria criteria) throws AtlasBaseException { private AtlasSolrQueryBuilder withCriteria(StringBuilder queryBuilder, FilterCriteria criteria) throws AtlasBaseException {
List<FilterCriteria> criterion = criteria.getCriterion(); List<FilterCriteria> criterion = criteria.getCriterion();
Set<String> indexAttributes = new HashSet<>();
if (StringUtils.isNotEmpty(criteria.getAttributeName()) && CollectionUtils.isEmpty(criterion)) { // no child criterion
String attributeName = criteria.getAttributeName();
String attributeValue = criteria.getAttributeValue();
Operator operator = criteria.getOperator();
ArrayList<StringBuilder> orExpQuery = new ArrayList<>();
for (AtlasEntityType type : entityTypes) {
String indexAttributeName = getIndexAttributeName(type, attributeName);
//check to remove duplicate attribute query (for eg. name)
if (!indexAttributes.contains(indexAttributeName)) {
StringBuilder sb = new StringBuilder();
if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works
operator = Operator.EQ;
attributeValue = getIndexQueryAttributeValue(attributeValue);
}
withPropertyCondition(sb, indexAttributeName, operator, attributeValue);
indexAttributes.add(indexAttributeName);
orExpQuery.add(sb);
}
}
if (CollectionUtils.isNotEmpty(orExpQuery)) {
if (orExpQuery.size() > 1) {
String orExpStr = StringUtils.join(orExpQuery, FilterCriteria.Condition.OR.name());
queryBuilder.append(" ( ").append(orExpStr).append(" ) ");
} else {
queryBuilder.append(orExpQuery.iterator().next());
}
}
if(criterion == null || CollectionUtils.isEmpty(criteria.getCriterion())) { // no child criterion } else if (CollectionUtils.isNotEmpty(criterion)) {
withPropertyCondition(queryBuilder, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
} else {
beginCriteria(queryBuilder); beginCriteria(queryBuilder);
for (Iterator<FilterCriteria> iterator = criterion.iterator(); iterator.hasNext(); ) { for (Iterator<FilterCriteria> iterator = criterion.iterator(); iterator.hasNext(); ) {
...@@ -209,40 +237,12 @@ public class AtlasSolrQueryBuilder { ...@@ -209,40 +237,12 @@ public class AtlasSolrQueryBuilder {
return this; return this;
} }
private void withPropertyCondition(StringBuilder queryBuilder, String attributeName, Operator operator, String attributeValue) throws AtlasBaseException { private void withPropertyCondition(StringBuilder queryBuilder, String indexFieldName, Operator operator, String attributeValue) throws AtlasBaseException {
if (StringUtils.isNotEmpty(attributeName) && operator != null) { if (StringUtils.isNotEmpty(indexFieldName) && operator != null) {
if (attributeValue != null) { if (attributeValue != null) {
attributeValue = attributeValue.trim(); attributeValue = attributeValue.trim();
} }
if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works
operator = Operator.EQ;
attributeValue = getIndexQueryAttributeValue(attributeValue);
}
AtlasAttribute attribute = entityType.getAttribute(attributeName);
if (attribute == null) {
String msg = String.format("Received unknown attribute '%s' for type '%s'.", attributeName, entityType.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
String indexFieldName = attribute.getIndexFieldName();
if (indexFieldName == null) {
String msg = String.format("Received non-index attribute %s for type %s.", attributeName, entityType.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
beginCriteria(queryBuilder); beginCriteria(queryBuilder);
switch (operator) { switch (operator) {
...@@ -308,6 +308,29 @@ public class AtlasSolrQueryBuilder { ...@@ -308,6 +308,29 @@ public class AtlasSolrQueryBuilder {
return attributeValue; return attributeValue;
} }
private String getIndexAttributeName(AtlasEntityType type, String attrName) throws AtlasBaseException {
AtlasAttribute ret = type.getAttribute(attrName);
if (ret == null) {
String msg = String.format("Received unknown attribute '%s' for type '%s'.", attrName, type.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
String indexFieldName = ret.getIndexFieldName();
if (indexFieldName == null) {
String msg = String.format("Received non-index attribute %s for type %s.", attrName, type.getTypeName());
LOG.error(msg);
throw new AtlasBaseException(msg);
}
return indexFieldName;
}
private void beginCriteria(StringBuilder queryBuilder) { private void beginCriteria(StringBuilder queryBuilder) {
queryBuilder.append("( "); queryBuilder.append("( ");
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
...@@ -33,8 +33,10 @@ import org.testng.annotations.Test; ...@@ -33,8 +33,10 @@ import org.testng.annotations.Test;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
...@@ -45,6 +47,9 @@ public class AtlasSolrQueryBuilderTest { ...@@ -45,6 +47,9 @@ public class AtlasSolrQueryBuilderTest {
private AtlasEntityType hiveTableEntityTypeMock; private AtlasEntityType hiveTableEntityTypeMock;
@Mock @Mock
private AtlasEntityType hiveTableEntityTypeMock2;
@Mock
private AtlasStructType.AtlasAttribute nameAttributeMock; private AtlasStructType.AtlasAttribute nameAttributeMock;
@Mock @Mock
...@@ -95,6 +100,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -95,6 +100,7 @@ public class AtlasSolrQueryBuilderTest {
when(hiveTableEntityTypeMock.getTypeName()).thenReturn("hive_table"); when(hiveTableEntityTypeMock.getTypeName()).thenReturn("hive_table");
when(hiveTableEntityTypeMock2.getTypeName()).thenReturn("hive_db");
when(nameAttributeMock.getIndexFieldName()).thenReturn("name_index"); when(nameAttributeMock.getIndexFieldName()).thenReturn("name_index");
when(commentAttributeMock.getIndexFieldName()).thenReturn("comment_index"); when(commentAttributeMock.getIndexFieldName()).thenReturn("comment_index");
...@@ -114,7 +120,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -114,7 +120,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )"); Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )");
} }
@Test @Test
...@@ -124,7 +130,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -124,7 +130,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )"); Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
} }
@Test @Test
...@@ -134,7 +140,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -134,7 +140,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )"); Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )");
} }
@Test @Test
...@@ -144,7 +150,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -144,7 +150,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )"); Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
} }
@Test @Test
...@@ -154,7 +160,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -154,7 +160,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )"); Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
} }
@Test @Test
...@@ -164,7 +170,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -164,7 +170,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )"); Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )");
} }
@Test @Test
...@@ -174,7 +180,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -174,7 +180,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )"); Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )");
} }
@Test @Test
...@@ -184,7 +190,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -184,7 +190,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )"); Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )");
} }
@Test @Test
...@@ -194,7 +200,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -194,7 +200,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )"); Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )");
} }
@Test @Test
...@@ -204,7 +210,7 @@ public class AtlasSolrQueryBuilderTest { ...@@ -204,7 +210,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )"); Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )");
} }
@Test @Test
...@@ -214,9 +220,18 @@ public class AtlasSolrQueryBuilderTest { ...@@ -214,9 +220,18 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest); processSearchParameters(fileName, underTest);
Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )"); Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )");
} }
@Test
public void testGenerateSolrQueryString2TypeNames() throws IOException, AtlasBaseException {
final String fileName = "src/test/resources/searchparameters2Types.json";
AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
processSearchParametersForMultipleTypeNames(fileName, underTest);
Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table hive_db ) ");
}
...@@ -243,8 +258,23 @@ public class AtlasSolrQueryBuilderTest { ...@@ -243,8 +258,23 @@ public class AtlasSolrQueryBuilderTest {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class); SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class);
Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
underTest.withEntityTypes(hiveTableEntityTypeMocks)
.withQueryString(searchParameters.getQuery())
.withCriteria(searchParameters.getEntityFilters())
.withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
.withCommonIndexFieldNames(indexFieldNamesMap);
}
private void processSearchParametersForMultipleTypeNames(String fileName, AtlasSolrQueryBuilder underTest) throws IOException, AtlasBaseException {
ObjectMapper mapper = new ObjectMapper();
SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class);
underTest.withEntityType(hiveTableEntityTypeMock) Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock2);
underTest.withEntityTypes(hiveTableEntityTypeMocks)
.withQueryString(searchParameters.getQuery()) .withQueryString(searchParameters.getQuery())
.withCriteria(searchParameters.getEntityFilters()) .withCriteria(searchParameters.getEntityFilters())
.withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities()) .withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
......
graphdb/janus/src/test/resources/searchparameters2Types.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t",
"limit":25,
"offset":0,
"typeName":"hive_table,hive_db",
"classification":null,
"termName":null
}
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
...@@ -67,17 +67,16 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -67,17 +67,16 @@ public class ClassificationSearchProcessor extends SearchProcessor {
public ClassificationSearchProcessor(SearchContext context) { public ClassificationSearchProcessor(SearchContext context) {
super(context); super(context);
final AtlasClassificationType classificationType = context.getClassificationType();
final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters(); final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
final Set<String> indexAttributes = new HashSet<>(); final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>(); final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>(); final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getClassificationTypes(); final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr(); final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
final boolean isBuiltInType = context.isBuiltInClassificationType();
final boolean isWildcardSearch = context.isWildCardSearch(); final boolean isWildcardSearch = context.isWildCardSearch();
final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes); processSearchAttributes(classificationTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes);
/* for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query /* for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
There are four cases in the classification type : There are four cases in the classification type :
...@@ -87,19 +86,25 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -87,19 +86,25 @@ public class ClassificationSearchProcessor extends SearchProcessor {
4. classification is not present in the search parameter 4. classification is not present in the search parameter
each of above cases with either has empty/or not tagFilters each of above cases with either has empty/or not tagFilters
*/ */
final boolean useIndexSearchForEntity = (classificationType != null || isWildcardSearch) && final boolean useIndexSearchForEntity = (CollectionUtils.isNotEmpty(classificationTypes) || isWildcardSearch) &&
!context.hasAttributeFilter(filterCriteria) && !context.hasAttributeFilter(filterCriteria) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS); (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
/* If classification's attributes can be applied index filter, we can use direct index /* If classification's attributes can be applied index filter, we can use direct index
* to query classification index as well. * to query classification index as well.
*/ */
final boolean useIndexSearchForClassification = (!isBuiltInType && !isWildcardSearch) && final boolean useIndexSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
!isWildcardSearch) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) && (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
CollectionUtils.isNotEmpty(indexAttributes) && CollectionUtils.isNotEmpty(indexAttributes) &&
canApplyIndexFilter(classificationType, filterCriteria, false); canApplyIndexFilter(classificationTypes, filterCriteria, false);
traitPredicate = buildTraitPredict(classificationType); final boolean useGraphSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
!isWildcardSearch && CollectionUtils.isNotEmpty(graphAttributes));
traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry()); isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
AtlasGraph graph = context.getGraph(); AtlasGraph graph = context.getGraph();
...@@ -115,8 +120,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -115,8 +120,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
// tagFilters is not allowed in wildcard search // tagFilters is not allowed in wildcard search
graphIndexQueryBuilder.addClassificationTypeFilter(queryString); graphIndexQueryBuilder.addClassificationTypeFilter(queryString);
} else { } else {
if (isBuiltInType) { if (classificationTypes.iterator().next() == SearchContext.MATCH_ALL_NOT_CLASSIFIED) {
// tagFilters is not allowed in unique classificationType search // tagFilters is not allowed in unique classificationType search
graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString); graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString);
...@@ -146,7 +150,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -146,7 +150,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString); graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString, typeAndSubTypesQryStr); graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString, typeAndSubTypesQryStr);
constructFilterQuery(queryString, classificationType, filterCriteria, indexAttributes); constructFilterQuery(queryString, classificationTypes, filterCriteria, indexAttributes);
String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")"); String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")"); indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
...@@ -158,7 +162,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -158,7 +162,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate); inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
} }
Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes); Predicate attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) { if (attributePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate); inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
...@@ -170,7 +174,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -170,7 +174,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
} }
// only registered classification will search with tag filters // only registered classification will search with tag filters
if (!isWildcardSearch && !isBuiltInType && !graphAttributes.isEmpty()) { if (useGraphSearchForClassification) {
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE; AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
AtlasGraphQuery query = graph.query(); AtlasGraphQuery query = graph.query();
...@@ -179,7 +183,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -179,7 +183,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes); query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
} }
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationType, filterCriteria, allAttributes, query); tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
gremlinQueryBindings = new HashMap<>(); gremlinQueryBindings = new HashMap<>();
StringBuilder gremlinQuery = new StringBuilder(); StringBuilder gremlinQuery = new StringBuilder();
...@@ -188,7 +192,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -188,7 +192,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
gremlinQuery.append(".as('e').filter(out()"); gremlinQuery.append(".as('e').filter(out()");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER)); gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters()); // constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
// After filtering on tags go back to e and output the list of entity vertices // After filtering on tags go back to e and output the list of entity vertices
gremlinQuery.append(").toList()"); gremlinQuery.append(").toList()");
......
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
...@@ -486,9 +486,11 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -486,9 +486,11 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
resultAttributes.addAll(searchContext.getEntityAttributes()); resultAttributes.addAll(searchContext.getEntityAttributes());
} }
AtlasEntityType entityType = searchContext.getEntityType(); if (CollectionUtils.isNotEmpty(searchContext.getEntityTypes())) {
if (entityType != null) {
for (String resultAttribute : resultAttributes) { AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
for (String resultAttribute : resultAttributes) {
AtlasAttribute attribute = entityType.getAttribute(resultAttribute); AtlasAttribute attribute = entityType.getAttribute(resultAttribute);
if (attribute == null) { if (attribute == null) {
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
...@@ -31,27 +31,15 @@ import org.apache.atlas.utils.AtlasPerfTracer; ...@@ -31,27 +31,15 @@ import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate; import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils; import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.*;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.stream.StreamSupport; import java.util.stream.StreamSupport;
import static org.apache.atlas.SortOrder.ASCENDING; import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES; import static org.apache.atlas.discovery.SearchContext.*;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
...@@ -69,28 +57,28 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -69,28 +57,28 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) { public EntitySearchProcessor(SearchContext context) {
super(context); super(context);
final AtlasEntityType entityType = context.getEntityType(); final Set<AtlasEntityType> entityTypes = context.getEntityTypes();
final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
final Set<String> indexAttributes = new HashSet<>(); final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>(); final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>(); final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getEntityTypes(); final Set<String> typeAndSubTypes = context.getEntityTypeNames();
final String typeAndSubTypesQryStr = context.getEntityTypesQryStr(); final String typeAndSubTypesQryStr = context.getEntityTypesQryStr();
final String sortBy = context.getSearchParameters().getSortBy(); final String sortBy = context.getSearchParameters().getSortBy();
final SortOrder sortOrder = context.getSearchParameters().getSortOrder(); final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
final AtlasClassificationType classificationType = context.getClassificationType(); final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
final Set<String> classificationTypeAndSubTypes = context.getClassificationTypes(); final Set<String> classificationTypeAndSubTypes = context.getClassificationTypeNames();
final boolean filterClassification; final boolean filterClassification;
if (classificationType != null) { if (CollectionUtils.isNotEmpty(classificationTypes)) {
filterClassification = !context.needClassificationProcessor(); filterClassification = !context.needClassificationProcessor();
} else { } else {
filterClassification = false; filterClassification = false;
} }
final Predicate typeNamePredicate; final Predicate typeNamePredicate;
final Predicate traitPredicate = buildTraitPredict(classificationType); final Predicate traitPredicate = buildTraitPredict(classificationTypes);
final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator() final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class); .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
...@@ -101,10 +89,10 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -101,10 +89,10 @@ public class EntitySearchProcessor extends SearchProcessor {
typeNamePredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry()); typeNamePredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
} }
processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes); processSearchAttributes(entityTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes);
final boolean typeSearchByIndex = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES; final boolean typeSearchByIndex = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES;
final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityType, filterCriteria, false); final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityTypes, filterCriteria, false);
StringBuilder indexQuery = new StringBuilder(); StringBuilder indexQuery = new StringBuilder();
...@@ -116,9 +104,9 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -116,9 +104,9 @@ public class EntitySearchProcessor extends SearchProcessor {
} }
if (attrSearchByIndex) { if (attrSearchByIndex) {
constructFilterQuery(indexQuery, entityType, filterCriteria, indexAttributes); constructFilterQuery(indexQuery, entityTypes, filterCriteria, indexAttributes);
Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, indexAttributes); Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) { if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate); inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
} }
...@@ -149,6 +137,7 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -149,6 +137,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// If we need to filter on the trait names then we need to build the query and equivalent in-memory predicate // If we need to filter on the trait names then we need to build the query and equivalent in-memory predicate
if (filterClassification) { if (filterClassification) {
AtlasClassificationType classificationType = classificationTypes.iterator().next();
List<AtlasGraphQuery> orConditions = new LinkedList<>(); List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) { if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
...@@ -176,10 +165,10 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -176,10 +165,10 @@ public class EntitySearchProcessor extends SearchProcessor {
} }
} }
graphQuery = toGraphFilterQuery(entityType, filterCriteria, graphAttributes, query); graphQuery = toGraphFilterQuery(entityTypes, filterCriteria, graphAttributes, query);
// Prepare in-memory predicate for attribute filtering // Prepare in-memory predicate for attribute filtering
Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, graphAttributes); Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, graphAttributes);
if (attributePredicate != null) { if (attributePredicate != null) {
if (graphQueryPredicate != null) { if (graphQueryPredicate != null) {
...@@ -199,7 +188,8 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -199,7 +188,8 @@ public class EntitySearchProcessor extends SearchProcessor {
} }
} }
if (sortBy != null && !sortBy.isEmpty()) { if (sortBy != null && !sortBy.isEmpty()) {
AtlasAttribute sortByAttribute = context.getEntityType().getAttribute(sortBy); final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute != null) { if (sortByAttribute != null) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC; AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
...@@ -215,8 +205,7 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -215,8 +205,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// Prepare the graph query and in-memory filter for the filtering phase // Prepare the graph query and in-memory filter for the filtering phase
filterGraphQueryPredicate = typeNamePredicate; filterGraphQueryPredicate = typeNamePredicate;
Predicate attributesPredicate = constructInMemoryPredicate(entityTypes, filterCriteria, allAttributes);
Predicate attributesPredicate = constructInMemoryPredicate(entityType, filterCriteria, allAttributes);
if (attributesPredicate != null) { if (attributesPredicate != null) {
filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate : filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate :
...@@ -265,7 +254,7 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -265,7 +254,7 @@ public class EntitySearchProcessor extends SearchProcessor {
SortOrder sortOrder = context.getSearchParameters().getSortOrder(); SortOrder sortOrder = context.getSearchParameters().getSortOrder();
String sortBy = context.getSearchParameters().getSortBy(); String sortBy = context.getSearchParameters().getSortBy();
final AtlasEntityType entityType = context.getEntityType(); final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy); AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute == null) { if (sortByAttribute == null) {
sortBy = null; sortBy = null;
......
repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
...@@ -18,7 +18,6 @@ ...@@ -18,7 +18,6 @@
package org.apache.atlas.discovery; package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.graphdb.*;
...@@ -53,13 +52,13 @@ public class FreeTextSearchProcessor extends SearchProcessor { ...@@ -53,13 +52,13 @@ public class FreeTextSearchProcessor extends SearchProcessor {
queryString.append(searchParameters.getQuery()); queryString.append(searchParameters.getQuery());
if (CollectionUtils.isNotEmpty(context.getEntityTypes()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) { if (CollectionUtils.isNotEmpty(context.getEntityTypeNames()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getEntityTypesQryStr()); queryString.append(AND_STR).append(context.getEntityTypesQryStr());
} }
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString); graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) { if (CollectionUtils.isNotEmpty(context.getClassificationTypeNames()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getClassificationTypesQryStr()); queryString.append(AND_STR).append(context.getClassificationTypesQryStr());
} }
...@@ -138,7 +137,7 @@ public class FreeTextSearchProcessor extends SearchProcessor { ...@@ -138,7 +137,7 @@ public class FreeTextSearchProcessor extends SearchProcessor {
continue; continue;
} }
if (context.getClassificationType() != null) { if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
List<String> entityClassifications = GraphHelper.getAllTraitNames(vertex); List<String> entityClassifications = GraphHelper.getAllTraitNames(vertex);
if (!context.includeClassificationTypes(entityClassifications)) { if (!context.includeClassificationTypes(entityClassifications)) {
......
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
...@@ -24,6 +24,7 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery; ...@@ -24,6 +24,7 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -31,10 +32,7 @@ import java.util.ArrayList; ...@@ -31,10 +32,7 @@ import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_ENTITY_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
public class FullTextSearchProcessor extends SearchProcessor { public class FullTextSearchProcessor extends SearchProcessor {
...@@ -53,29 +51,28 @@ public class FullTextSearchProcessor extends SearchProcessor { ...@@ -53,29 +51,28 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary // if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor // processing (and rejection) by subsequent EntitySearchProcessor
if (context.getEntityType() != null && context.getEntityType() != MATCH_ALL_ENTITY_TYPES) { if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr(); String typeAndSubTypeNamesQryStr = context.getEntityTypesQryStr();
if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) { if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr); queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
} else { } else {
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length()); searchParameters.getTypeName(), typeAndSubTypeNamesQryStr.length());
} }
} }
// if search includes classification criteria, adding a filter here can help avoid unnecessary // if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if (context.getClassificationType() != null && context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION && if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) &&
context.getClassificationType() != MATCH_ALL_CLASSIFIED && context.getClassificationTypes().iterator().next() != MATCH_ALL_NOT_CLASSIFIED) {
context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) { String typeAndSubTypeNamesStr = context.getClassificationTypesQryStr();
String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) { if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
queryString.append(AND_STR).append(typeAndSubTypeNamesStr); queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
} else { } else {
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length()); searchParameters.getClassification(), typeAndSubTypeNamesStr.length());
} }
} }
......
repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
...@@ -26,6 +26,8 @@ import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NA ...@@ -26,6 +26,8 @@ import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NA
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.type.AtlasStructType;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
public class GraphIndexQueryBuilder { public class GraphIndexQueryBuilder {
...@@ -36,20 +38,20 @@ public class GraphIndexQueryBuilder { ...@@ -36,20 +38,20 @@ public class GraphIndexQueryBuilder {
} }
void addClassificationTypeFilter(StringBuilder indexQuery) { void addClassificationTypeFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) { if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationNames())) {
String classificationName = context.getSearchParameters().getClassification(); String classificationNames = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(context.getClassificationNames());
if (indexQuery.length() != 0) { if (indexQuery.length() != 0) {
indexQuery.append(" AND "); indexQuery.append(" AND ");
} }
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName) indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationNames)
.append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY) .append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
.append('\"').append(':').append(classificationName).append(")"); .append('\"').append(':').append(classificationNames).append(")");
} }
} }
void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) { void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) { if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
String classificationTypesQryStr = context.getClassificationTypesQryStr(); String classificationTypesQryStr = context.getClassificationTypesQryStr();
if (indexQuery.length() != 0) { if (indexQuery.length() != 0) {
...@@ -63,17 +65,8 @@ public class GraphIndexQueryBuilder { ...@@ -63,17 +65,8 @@ public class GraphIndexQueryBuilder {
} }
void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) { void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) {
if (indexQuery != null && context.getClassificationType() != null) { if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
if (context.getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() == MATCH_ALL_CLASSIFIED) { if (context.getClassificationTypes().iterator().next() == MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"")
.append(CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]")
.append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"")
.append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]").append(")");
} else if (context.getClassificationType() == MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery.length() != 0) { if (indexQuery.length() != 0) {
indexQuery.append(" AND "); indexQuery.append(" AND ");
} }
......
repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
...@@ -56,12 +56,7 @@ public class SearchAggregatorImpl implements SearchAggregator { ...@@ -56,12 +56,7 @@ public class SearchAggregatorImpl implements SearchAggregator {
try { try {
AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient(); AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
String searchedOnTypeName = searchParameters.getTypeName(); Set<AtlasEntityType> searchForEntityType = searchContext.getEntityTypes();
AtlasEntityType searchForEntityType = null;
if (searchedOnTypeName != null) {
searchForEntityType = typeRegistry.getEntityTypeByName(searchedOnTypeName);
}
Map<String, String> indexFieldNameCache = new HashMap<>(); Map<String, String> indexFieldNameCache = new HashMap<>();
......
repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
...@@ -130,7 +130,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup { ...@@ -130,7 +130,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
new EntitySearchProcessor(context); new EntitySearchProcessor(context);
} }
@Test @Test(priority = -1)
public void searchWithNEQ_stringAttr() throws AtlasBaseException { public void searchWithNEQ_stringAttr() throws AtlasBaseException {
String expectedEntityName = "hive_Table_Null_tableType"; String expectedEntityName = "hive_Table_Null_tableType";
createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE); createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
...@@ -197,4 +197,116 @@ public class EntitySearchProcessorTest extends BasicTestSetup { ...@@ -197,4 +197,116 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertTrue(nameList.contains("hive_Table_Null_tableType")); assertTrue(nameList.contains("hive_Table_Null_tableType"));
} }
@Test
public void ALLEntityType() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 20);
}
@Test
public void ALLEntityTypeWithTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
params.setClassification(FACT_CLASSIFICATION);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 5);
}
@Test
public void entityType() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 3);
}
@Test
public void entityTypes() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 14);
}
@Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Not_Exists: Unknown/invalid typename")
public void entityTypesNotAllowed() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+",Not_Exists");
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
}
@Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Attribute tableType not found for type "+DATABASE_TYPE)
public void entityFiltersNotAllowed() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.CONTAINS, "ETL");
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
}
@Test
public void entityTypesAndTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
params.setClassification(FACT_CLASSIFICATION);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 3);
}
@Test
public void searchWithEntityTypesAndEntityFilters() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 4);
}
@Test
public void searchWithEntityTypesAndEntityFiltersAndTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
params.setEntityFilters(filterCriteria);
params.setClassification(LOGDATA_CLASSIFICATION);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
assertEquals(processor.execute().size(), 2);
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment