Skip to content

A
atlas
Commit abc4856e by apoorvnaik Committed by Madhan Neethiraj
Options

ATLAS-1925: basic search fixes for ATLAS-1917, ATLAS-1922, ATLAS-1926

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 3962057c
Showing with 39 additions and 17 deletions
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
...@@ -55,6 +55,7 @@ import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; ...@@ -55,6 +55,7 @@ import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.util.AtlasGremlinQueryProvider; import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
import org.apache.atlas.util.SearchTracker;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
...@@ -84,17 +85,19 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -84,17 +85,19 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private final AtlasGremlinQueryProvider gremlinQueryProvider; private final AtlasGremlinQueryProvider gremlinQueryProvider;
private final AtlasTypeRegistry typeRegistry; private final AtlasTypeRegistry typeRegistry;
private final GraphBackedSearchIndexer indexer; private final GraphBackedSearchIndexer indexer;
private final SearchTracker searchTracker;
private final int maxResultSetSize; private final int maxResultSetSize;
private final int maxTypesCountInIdxQuery; private final int maxTypesCountInIdxQuery;
private final int maxTagsCountInIdxQuery; private final int maxTagsCountInIdxQuery;
@Inject @Inject
EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry, EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry,
AtlasGraph graph, GraphBackedSearchIndexer indexer) throws AtlasException { AtlasGraph graph, GraphBackedSearchIndexer indexer, SearchTracker searchTracker) throws AtlasException {
this.graph = graph; this.graph = graph;
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository); this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
this.entityRetriever = new EntityGraphRetriever(typeRegistry); this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.indexer = indexer; this.indexer = indexer;
this.searchTracker = searchTracker;
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
this.typeRegistry = typeRegistry; this.typeRegistry = typeRegistry;
this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150); this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
...@@ -402,7 +405,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -402,7 +405,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
AtlasSearchResult ret = new AtlasSearchResult(searchParameters); AtlasSearchResult ret = new AtlasSearchResult(searchParameters);
SearchContext context = new SearchContext(searchParameters, typeRegistry, graph, indexer.getVertexIndexKeys()); SearchContext context = new SearchContext(searchParameters, typeRegistry, graph, indexer.getVertexIndexKeys());
String searchID = searchTracker.add(context); // For future cancellations
try {
List<AtlasVertex> resultList = context.getSearchProcessor().execute(); List<AtlasVertex> resultList = context.getSearchProcessor().execute();
// By default any attribute that shows up in the search parameter should be sent back in the response // By default any attribute that shows up in the search parameter should be sent back in the response
...@@ -441,7 +446,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -441,7 +446,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
Object attrValue = entity.getAttribute(entityAttribute); Object attrValue = entity.getAttribute(entityAttribute);
if (attrValue instanceof AtlasObjectId) { if (attrValue instanceof AtlasObjectId) {
AtlasObjectId objId = (AtlasObjectId)attrValue; AtlasObjectId objId = (AtlasObjectId) attrValue;
if (ret.getReferredEntities() == null) { if (ret.getReferredEntities() == null) {
ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>()); ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>());
...@@ -451,11 +456,11 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -451,11 +456,11 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid())); ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
} }
} else if (attrValue instanceof Collection) { } else if (attrValue instanceof Collection) {
Collection objIds = (Collection)attrValue; Collection objIds = (Collection) attrValue;
for (Object obj : objIds) { for (Object obj : objIds) {
if (obj instanceof AtlasObjectId) { if (obj instanceof AtlasObjectId) {
AtlasObjectId objId = (AtlasObjectId)obj; AtlasObjectId objId = (AtlasObjectId) obj;
if (ret.getReferredEntities() == null) { if (ret.getReferredEntities() == null) {
ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>()); ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>());
...@@ -469,6 +474,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -469,6 +474,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
} }
} }
} }
} finally {
searchTracker.remove(searchID);
}
return ret; return ret;
} }
...@@ -479,7 +487,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -479,7 +487,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) { private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) {
String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery); String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery);
String classficationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery); String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery);
StringBuilder queryText = new StringBuilder(); StringBuilder queryText = new StringBuilder();
...@@ -495,12 +503,12 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -495,12 +503,12 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
queryText.append(typeFilter); queryText.append(typeFilter);
} }
if (! StringUtils.isEmpty(classficationFilter)) { if (! StringUtils.isEmpty(classificationFilter)) {
if (queryText.length() > 0) { if (queryText.length() > 0) {
queryText.append(" AND "); queryText.append(" AND ");
} }
queryText.append(classficationFilter); queryText.append(classificationFilter);
} }
return String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, queryText.toString()); return String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, queryText.toString());
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
...@@ -52,6 +52,7 @@ public abstract class SearchProcessor { ...@@ -52,6 +52,7 @@ public abstract class SearchProcessor {
public static final String BRACE_CLOSE_STR = " )"; public static final String BRACE_CLOSE_STR = " )";
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>(); private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final char[] OFFENDING_CHARS = {'@', '/', ' '}; // This can grow as we discover corner cases
static static
{ {
...@@ -60,9 +61,9 @@ public abstract class SearchProcessor { ...@@ -60,9 +61,9 @@ public abstract class SearchProcessor {
OPERATOR_MAP.put(SearchParameters.Operator.LTE,"v.\"%s\": [* TO %s]"); OPERATOR_MAP.put(SearchParameters.Operator.LTE,"v.\"%s\": [* TO %s]");
OPERATOR_MAP.put(SearchParameters.Operator.GTE,"v.\"%s\": [%s TO *]"); OPERATOR_MAP.put(SearchParameters.Operator.GTE,"v.\"%s\": [%s TO *]");
OPERATOR_MAP.put(SearchParameters.Operator.EQ,"v.\"%s\": %s"); OPERATOR_MAP.put(SearchParameters.Operator.EQ,"v.\"%s\": %s");
OPERATOR_MAP.put(SearchParameters.Operator.NEQ,"v.\"%s\": (NOT %s)"); OPERATOR_MAP.put(SearchParameters.Operator.NEQ,"-" + "v.\"%s\": %s");
OPERATOR_MAP.put(SearchParameters.Operator.IN, "v.\"%s\": (%s)"); OPERATOR_MAP.put(SearchParameters.Operator.IN, "v.\"%s\": (%s)"); // this should be a list of quoted strings
OPERATOR_MAP.put(SearchParameters.Operator.LIKE, "v.\"%s\": (%s)"); OPERATOR_MAP.put(SearchParameters.Operator.LIKE, "v.\"%s\": (%s)"); // this should be regex pattern
OPERATOR_MAP.put(SearchParameters.Operator.STARTS_WITH, "v.\"%s\": (%s*)"); OPERATOR_MAP.put(SearchParameters.Operator.STARTS_WITH, "v.\"%s\": (%s*)");
OPERATOR_MAP.put(SearchParameters.Operator.ENDS_WITH, "v.\"%s\": (*%s)"); OPERATOR_MAP.put(SearchParameters.Operator.ENDS_WITH, "v.\"%s\": (*%s)");
OPERATOR_MAP.put(SearchParameters.Operator.CONTAINS, "v.\"%s\": (*%s*)"); OPERATOR_MAP.put(SearchParameters.Operator.CONTAINS, "v.\"%s\": (*%s*)");
...@@ -184,7 +185,7 @@ public abstract class SearchProcessor { ...@@ -184,7 +185,7 @@ public abstract class SearchProcessor {
if (filterCriteria != null) { if (filterCriteria != null) {
LOG.debug("Processing Filters"); LOG.debug("Processing Filters");
String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes); String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes, 0);
if (StringUtils.isNotEmpty(filterQuery)) { if (StringUtils.isNotEmpty(filterQuery)) {
solrQuery.append(AND_STR).append(filterQuery); solrQuery.append(AND_STR).append(filterQuery);
...@@ -196,27 +197,31 @@ public abstract class SearchProcessor { ...@@ -196,27 +197,31 @@ public abstract class SearchProcessor {
} }
} }
private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes) { private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, int level) {
return toSolrQuery(type, criteria, solrAttributes, new StringBuilder()); return toSolrQuery(type, criteria, solrAttributes, new StringBuilder(), level);
} }
private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, StringBuilder sb) { private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, StringBuilder sb, int level) {
if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) { if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
StringBuilder nestedExpression = new StringBuilder(); StringBuilder nestedExpression = new StringBuilder();
for (FilterCriteria filterCriteria : criteria.getCriterion()) { for (FilterCriteria filterCriteria : criteria.getCriterion()) {
String nestedQuery = toSolrQuery(type, filterCriteria, solrAttributes); String nestedQuery = toSolrQuery(type, filterCriteria, solrAttributes, level + 1);
if (StringUtils.isNotEmpty(nestedQuery)) { if (StringUtils.isNotEmpty(nestedQuery)) {
if (nestedExpression.length() > 0) { if (nestedExpression.length() > 0) {
nestedExpression.append(SPACE_STRING).append(criteria.getCondition()).append(SPACE_STRING); nestedExpression.append(SPACE_STRING).append(criteria.getCondition()).append(SPACE_STRING);
} }
// todo: when a neq operation is nested and occurs in the beginning of the query, solr has issues
nestedExpression.append(nestedQuery); nestedExpression.append(nestedQuery);
} }
} }
return nestedExpression.length() > 0 ? sb.append(BRACE_OPEN_STR).append(nestedExpression.toString()).append(BRACE_CLOSE_STR).toString() : EMPTY_STRING; if (level == 0) {
return nestedExpression.length() > 0 ? sb.append(nestedExpression).toString() : EMPTY_STRING;
} else {
return nestedExpression.length() > 0 ? sb.append(BRACE_OPEN_STR).append(nestedExpression).append(BRACE_CLOSE_STR).toString() : EMPTY_STRING;
}
} else if (solrAttributes.contains(criteria.getAttributeName())){ } else if (solrAttributes.contains(criteria.getAttributeName())){
return toSolrExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue()); return toSolrExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
} else { } else {
...@@ -231,8 +236,13 @@ public abstract class SearchProcessor { ...@@ -231,8 +236,13 @@ public abstract class SearchProcessor {
String qualifiedName = type.getQualifiedAttributeName(attrName); String qualifiedName = type.getQualifiedAttributeName(attrName);
if (OPERATOR_MAP.get(op) != null) { if (OPERATOR_MAP.get(op) != null) {
if (hasOffendingChars(attrVal)) {
// FIXME: if attrVal has offending chars & op is contains, endsWith, startsWith, solr doesn't like it and results are skewed
ret = String.format(OPERATOR_MAP.get(op), qualifiedName, "\"" + attrVal + "\"");
} else {
ret = String.format(OPERATOR_MAP.get(op), qualifiedName, attrVal); ret = String.format(OPERATOR_MAP.get(op), qualifiedName, attrVal);
} }
}
} catch (AtlasBaseException ex) { } catch (AtlasBaseException ex) {
LOG.warn(ex.getMessage()); LOG.warn(ex.getMessage());
} }
...@@ -378,4 +388,8 @@ public abstract class SearchProcessor { ...@@ -378,4 +388,8 @@ public abstract class SearchProcessor {
return defaultValue; return defaultValue;
} }
private boolean hasOffendingChars(String str) {
return StringUtils.containsAny(str, OFFENDING_CHARS);
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment