Skip to content

A
atlas
Commit 9b72de98 by apoorvnaik Committed by Madhan Neethiraj
Options

ATLAS-1994: fix - basic-search ignores tag attribute filters when free-text query is present

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 6fb2a038
Showing with 227 additions and 28 deletions
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -17,17 +17,13 @@
*/
package org.apache.atlas.type;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
......@@ -41,6 +37,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF;
/**
* class that implements behaviour of a struct-type.
*/
......@@ -773,7 +773,7 @@ public class AtlasStructType extends AtlasType {
new String[] { "%", "_p" },
};
private static final char[] IDX_QRY_OFFENDING_CHARS = { '@', '/', ' ' };
private static final char[] IDX_QRY_OFFENDING_CHARS = { '@', '/', ' ', '-' };
private static final char BRACE_OPEN_CHAR = '(';
private static final char BRACE_CLOSE_CHAR = ')';
private static final char DOUBLE_QUOTE_CHAR = '"';
......
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
......@@ -17,18 +17,35 @@
*/
package org.apache.atlas.discovery;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class ClassificationSearchProcessor extends SearchProcessor {
......@@ -37,7 +54,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
private final AtlasIndexQuery indexQuery;
private final AtlasGraphQuery allGraphQuery;
private final AtlasGraphQuery filterGraphQuery;
private final String gremlinTagFilterQuery;
private final Map<String, Object> gremlinQueryBindings;
public ClassificationSearchProcessor(SearchContext context) {
super(context);
......@@ -56,6 +75,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
// for classification search, if any attribute can't be handled by Solr - switch to all Gremlin
boolean useSolrSearch = typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false);
AtlasGraph graph = context.getGraph();
if (useSolrSearch) {
StringBuilder solrQuery = new StringBuilder();
......@@ -67,18 +88,42 @@ public class ClassificationSearchProcessor extends SearchProcessor {
solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll("");
indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString);
indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, solrQueryString);
} else {
indexQuery = null;
}
AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
AtlasGraphQuery query = graph.query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
allGraphQuery = toGraphFilterQuery(classificationType, filterCriteria, allAttributes, query);
if (context.getSearchParameters().getTagFilters() != null) {
// Now filter on the tag attributes
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
allGraphQuery = toGremlinFilterQuery(classificationType, filterCriteria, allAttributes, query);
StringBuilder gremlinQuery = new StringBuilder();
gremlinQuery.append("g.V().has('__guid', T.in, guids)");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER));
gremlinQuery.append(".as('e').out()");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
query = context.getGraph().query().in(Constants.TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes);
constructGremlinFilterQuery(gremlinQuery, context.getClassificationType(), context.getSearchParameters().getTagFilters());
// After filtering on tags go back to e and output the list of entity vertices
gremlinQuery.append(".back('e').toList()");
filterGraphQuery = query; // TODO: filer based on tag attributes
gremlinTagFilterQuery = gremlinQuery.toString();
gremlinQueryBindings = new HashMap<>();
gremlinQueryBindings.put("traitNames", typeAndSubTypes);
gremlinQueryBindings.put("typeNames", typeAndSubTypes); // classification typeName
if (LOG.isDebugEnabled()) {
LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
}
} else {
gremlinTagFilterQuery = null;
gremlinQueryBindings = null;
}
}
@Override
......@@ -195,12 +240,28 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size());
}
AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices));
if (gremlinTagFilterQuery != null && gremlinQueryBindings != null) {
// Now filter on the tag attributes
Set<String> guids = getGuids(entityVertices);
gremlinQueryBindings.put("guids", guids);
try {
AtlasGraph graph = context.getGraph();
ScriptEngine gremlinScriptEngine = graph.getGremlinScriptEngine();
List<AtlasVertex> atlasVertices = (List<AtlasVertex>) graph.executeGremlinScript(gremlinScriptEngine, gremlinQueryBindings, gremlinTagFilterQuery, false);
// Clear prior results
entityVertices.clear();
query.addConditionsFrom(filterGraphQuery);
if (CollectionUtils.isNotEmpty(atlasVertices)) {
entityVertices.addAll(atlasVertices);
}
entityVertices.clear();
getVertices(query.vertices().iterator(), entityVertices);
} catch (AtlasBaseException | ScriptException e) {
LOG.warn(e.getMessage());
}
}
super.filter(entityVertices);
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
......@@ -19,7 +19,9 @@ package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.utils.AtlasPerfTracer;
......@@ -27,7 +29,11 @@ import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
public class EntitySearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class);
......@@ -95,7 +101,7 @@ public class EntitySearchProcessor extends SearchProcessor {
query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes());
}
graphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
graphQuery = toGraphFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery == null) {
graphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
......@@ -110,7 +116,7 @@ public class EntitySearchProcessor extends SearchProcessor {
query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes());
}
filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
filterGraphQuery = toGraphFilterQuery(entityType, filterCriteria, allAttributes, query);
if (context.getSearchParameters().getExcludeDeletedEntities()) {
filterGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
......@@ -23,11 +23,14 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
......@@ -205,6 +208,49 @@ public abstract class SearchProcessor {
}
}
protected void constructGremlinFilterQuery(StringBuilder tagFilterQuery, AtlasStructType structType, FilterCriteria filterCriteria) {
if (filterCriteria != null) {
FilterCriteria.Condition condition = filterCriteria.getCondition();
if (condition != null) {
StringBuilder orQuery = new StringBuilder();
List<FilterCriteria> criterion = filterCriteria.getCriterion();
for (int i = 0; i < criterion.size(); i++) {
FilterCriteria criteria = criterion.get(i);
if (condition == FilterCriteria.Condition.OR) {
StringBuilder nestedOrQuery = new StringBuilder("_()");
constructGremlinFilterQuery(nestedOrQuery, structType, criteria);
orQuery.append(i == 0 ? "" : ",").append(nestedOrQuery);
} else {
constructGremlinFilterQuery(tagFilterQuery, structType, criteria);
}
}
if (condition == FilterCriteria.Condition.OR) {
tagFilterQuery.append(".or(").append(orQuery).append(")");
}
} else {
String attributeName = filterCriteria.getAttributeName();
AtlasAttribute attribute = structType.getAttribute(attributeName);
if (attribute != null) {
SearchParameters.Operator operator = filterCriteria.getOperator();
String attributeValue = filterCriteria.getAttributeValue();
tagFilterQuery.append(toGremlinComparisonQuery(attribute, operator, attributeValue));
} else {
LOG.warn("Ignoring unknown attribute {}.{}", structType.getTypeName(), attributeName);
}
}
}
}
protected void constructStateTestQuery(StringBuilder solrQuery) {
if (solrQuery.length() > 0) {
solrQuery.append(AND_STR);
......@@ -261,12 +307,12 @@ public abstract class SearchProcessor {
return ret;
}
protected AtlasGraphQuery toGremlinFilterQuery(AtlasStructType type, FilterCriteria criteria, Set<String> gremlinAttributes, AtlasGraphQuery query) {
protected AtlasGraphQuery toGraphFilterQuery(AtlasStructType type, FilterCriteria criteria, Set<String> gremlinAttributes, AtlasGraphQuery query) {
if (criteria != null) {
if (criteria.getCondition() != null) {
if (criteria.getCondition() == Condition.AND) {
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
AtlasGraphQuery nestedQuery = toGremlinFilterQuery(type, filterCriteria, gremlinAttributes, context.getGraph().query());
AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, gremlinAttributes, context.getGraph().query());
query.addConditionsFrom(nestedQuery);
}
......@@ -274,7 +320,7 @@ public abstract class SearchProcessor {
List<AtlasGraphQuery> orConditions = new LinkedList<>();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
AtlasGraphQuery nestedQuery = toGremlinFilterQuery(type, filterCriteria, gremlinAttributes, context.getGraph().query());
AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, gremlinAttributes, context.getGraph().query());
orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery));
}
......@@ -336,6 +382,53 @@ public abstract class SearchProcessor {
return query;
}
private String toGremlinComparisonQuery(AtlasAttribute attribute, SearchParameters.Operator operator, String attrValue) {
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
String queryTemplate = null;
switch (operator) {
case LT:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_LT);
break;
case GT:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_GT);
break;
case LTE:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_LTE);
break;
case GTE:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_GTE);
break;
case EQ:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_EQ);
break;
case NEQ:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_NEQ);
break;
case LIKE:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_MATCHES);
break;
case STARTS_WITH:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_STARTS_WITH);
break;
case ENDS_WITH:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_ENDS_WITH);
break;
case CONTAINS:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_CONTAINS);
break;
}
if (org.apache.commons.lang3.StringUtils.isNotEmpty(queryTemplate)) {
if (StringUtils.equalsIgnoreCase(attribute.getAttributeType().getTypeName(), AtlasBaseTypeDef.ATLAS_TYPE_STRING)) {
attrValue = "'" + attrValue + "'";
}
return String.format(queryTemplate, attribute.getQualifiedName(), attrValue);
} else {
return EMPTY_STRING;
}
}
private String getContainsRegex(String attributeValue) {
return ".*" + attributeValue + ".*";
}
......
repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
......@@ -75,6 +75,26 @@ public class AtlasGremlin2QueryProvider extends AtlasGremlinQueryProvider {
return " [startIdx..<endIdx].toList()";
case GUID_PREFIX_FILTER:
return ".filter{it.'__guid'.matches(guid)}";
case COMPARE_LT:
return ".has('%s', T.lt, %s)";
case COMPARE_LTE:
return ".has('%s', T.lte, %s)";
case COMPARE_GT:
return ".has('%s', T.gt, %s)";
case COMPARE_GTE:
return ".has('%s', T.gte, %s)";
case COMPARE_EQ:
return ".has('%s', T.eq, %s)";
case COMPARE_NEQ:
return ".has('%s', T.neq, %s)";
case COMPARE_MATCHES:
return ".filter({it.getProperty('%s').matches(%s)})";
case COMPARE_STARTS_WITH:
return ".filter({it.getProperty('%s').startsWith(%s)})";
case COMPARE_ENDS_WITH:
return ".filter({it.getProperty('%s').endsWith(%s)})";
case COMPARE_CONTAINS:
return ".filter({it.getProperty('%s').contains(%s)})";
}
// Should never reach this point
return null;
......
repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java
......@@ -60,6 +60,18 @@ public abstract class AtlasGremlinQueryProvider {
BASIC_SEARCH_CLASSIFICATION_FILTER,
BASIC_SEARCH_STATE_FILTER,
TO_RANGE_LIST,
GUID_PREFIX_FILTER
GUID_PREFIX_FILTER,
// Comparison clauses
COMPARE_LT,
COMPARE_LTE,
COMPARE_GT,
COMPARE_GTE,
COMPARE_EQ,
COMPARE_NEQ,
COMPARE_MATCHES,
COMPARE_STARTS_WITH,
COMPARE_ENDS_WITH,
COMPARE_CONTAINS,
}
}
webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
......@@ -167,8 +167,15 @@ public class DiscoveryREST {
typeName + "," + classification + "," + limit + "," + offset + ")");
}
return atlasDiscoveryService.searchUsingBasicQuery(query, typeName, classification, null, null,
excludeDeletedEntities, limit, offset);
SearchParameters searchParameters = new SearchParameters();
searchParameters.setTypeName(typeName);
searchParameters.setClassification(classification);
searchParameters.setQuery(query);
searchParameters.setExcludeDeletedEntities(excludeDeletedEntities);
searchParameters.setLimit(limit);
searchParameters.setOffset(offset);
return atlasDiscoveryService.searchWithParameters(searchParameters);
} finally {
AtlasPerfTracer.log(perf);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment