Skip to content

A
atlas
Commit e0d2cdc2 by Madhan Neethiraj
Options

ATLAS-1981: Cache escaped type-query string to avoid repeated computation

parent f511f272
Showing with 131 additions and 102 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
...@@ -97,8 +97,8 @@ public final class Constants { ...@@ -97,8 +97,8 @@ public final class Constants {
public static final String QUALIFIED_NAME = "Referenceable.qualifiedName"; public static final String QUALIFIED_NAME = "Referenceable.qualifiedName";
public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName"; public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size";
public static final String INDEX_SEARCH_MAX_TYPES_COUNT = "atlas.graph.index.search.max-types-count"; public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length";
public static final String INDEX_SEARCH_MAX_TAGS_COUNT = "atlas.graph.index.search.max-tags-count"; public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.tags.max-query-str-length";
private Constants() { private Constants() {
} }
......
intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
...@@ -45,10 +45,11 @@ public class AtlasClassificationType extends AtlasStructType { ...@@ -45,10 +45,11 @@ public class AtlasClassificationType extends AtlasStructType {
private final AtlasClassificationDef classificationDef; private final AtlasClassificationDef classificationDef;
private List<AtlasClassificationType> superTypes = Collections.emptyList(); private List<AtlasClassificationType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet(); private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> allSubTypes = Collections.emptySet(); private Set<String> allSubTypes = Collections.emptySet();
private Set<String> typeAndAllSubTypes = Collections.emptySet(); private Set<String> typeAndAllSubTypes = Collections.emptySet();
private String typeAndAllSubTypesQryStr = "";
public AtlasClassificationType(AtlasClassificationDef classificationDef) { public AtlasClassificationType(AtlasClassificationDef classificationDef) {
super(classificationDef); super(classificationDef);
...@@ -108,6 +109,13 @@ public class AtlasClassificationType extends AtlasStructType { ...@@ -108,6 +109,13 @@ public class AtlasClassificationType extends AtlasStructType {
} }
} }
@Override
public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
allSubTypes = Collections.unmodifiableSet(allSubTypes);
typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes);
typeAndAllSubTypesQryStr = ""; // will be computed on next access
}
private void addSubType(AtlasClassificationType subType) { private void addSubType(AtlasClassificationType subType) {
allSubTypes.add(subType.getTypeName()); allSubTypes.add(subType.getTypeName());
typeAndAllSubTypes.add(subType.getTypeName()); typeAndAllSubTypes.add(subType.getTypeName());
...@@ -119,11 +127,17 @@ public class AtlasClassificationType extends AtlasStructType { ...@@ -119,11 +127,17 @@ public class AtlasClassificationType extends AtlasStructType {
public Set<String> getAllSuperTypes() { return allSuperTypes; } public Set<String> getAllSuperTypes() { return allSuperTypes; }
public Set<String> getAllSubTypes() { public Set<String> getAllSubTypes() { return allSubTypes; }
return Collections.unmodifiableSet(allSubTypes);
}
public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); } public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; }
public String getTypeAndAllSubTypesQryStr() {
if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) {
typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes);
}
return typeAndAllSubTypesQryStr;
}
public boolean isSuperTypeOf(AtlasClassificationType classificationType) { public boolean isSuperTypeOf(AtlasClassificationType classificationType) {
return classificationType != null && allSubTypes.contains(classificationType.getTypeName()); return classificationType != null && allSubTypes.contains(classificationType.getTypeName());
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
...@@ -55,6 +55,7 @@ public class AtlasEntityType extends AtlasStructType { ...@@ -55,6 +55,7 @@ public class AtlasEntityType extends AtlasStructType {
private Set<String> typeAndAllSuperTypes = Collections.emptySet(); private Set<String> typeAndAllSuperTypes = Collections.emptySet();
private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap(); private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap();
private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap(); private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap();
private String typeAndAllSubTypesQryStr = "";
public AtlasEntityType(AtlasEntityDef entityDef) { public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef); super(entityDef);
...@@ -150,7 +151,7 @@ public class AtlasEntityType extends AtlasStructType { ...@@ -150,7 +151,7 @@ public class AtlasEntityType extends AtlasStructType {
allSubTypes = Collections.unmodifiableSet(allSubTypes); allSubTypes = Collections.unmodifiableSet(allSubTypes);
typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes); typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes);
typeAndAllSuperTypes = Collections.unmodifiableSet(typeAndAllSuperTypes); typeAndAllSubTypesQryStr = ""; // will be computed on next access
relationshipAttributes = Collections.unmodifiableMap(relationshipAttributes); relationshipAttributes = Collections.unmodifiableMap(relationshipAttributes);
relationshipAttributesType = Collections.unmodifiableMap(relationshipAttributesType); relationshipAttributesType = Collections.unmodifiableMap(relationshipAttributesType);
} }
...@@ -218,6 +219,14 @@ public class AtlasEntityType extends AtlasStructType { ...@@ -218,6 +219,14 @@ public class AtlasEntityType extends AtlasStructType {
return relationshipAttributesType; return relationshipAttributesType;
} }
public String getTypeAndAllSubTypesQryStr() {
if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) {
typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes);
}
return typeAndAllSubTypesQryStr;
}
public boolean hasRelationshipAttribute(String attributeName) { public boolean hasRelationshipAttribute(String attributeName) {
return relationshipAttributes.containsKey(attributeName); return relationshipAttributes.containsKey(attributeName);
} }
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
...@@ -38,6 +38,7 @@ import java.util.Collections; ...@@ -38,6 +38,7 @@ import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
/** /**
* class that implements behaviour of a struct-type. * class that implements behaviour of a struct-type.
...@@ -713,6 +714,39 @@ public class AtlasStructType extends AtlasType { ...@@ -713,6 +714,39 @@ public class AtlasStructType extends AtlasType {
return key; return key;
} }
public static String escapeIndexQueryValue(Set<String> values) {
StringBuilder sb = new StringBuilder();
sb.append(BRACE_OPEN_CHAR);
for (String value : values) {
sb.append(escapeIndexQueryValue(value)).append(SPACE_CHAR);
}
sb.append(BRACE_CLOSE_CHAR);
return sb.toString();
}
public static String escapeIndexQueryValue(String value) {
String ret = value;
if (StringUtils.containsAny(value, IDX_QRY_OFFENDING_CHARS)) {
boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE_CHAR;
boolean isQuoteAtEnd = value.charAt(value.length() - 1) == DOUBLE_QUOTE_CHAR;
if (!isQuoteAtStart) {
if (!isQuoteAtEnd) {
ret = DOUBLE_QUOTE_CHAR + value + DOUBLE_QUOTE_CHAR;
} else {
ret = DOUBLE_QUOTE_CHAR + value;
}
} else if (!isQuoteAtEnd) {
ret = value + DOUBLE_QUOTE_CHAR;
}
}
return ret;
}
private String getRelationshipEdgeLabel(String relationshipLabel) { private String getRelationshipEdgeLabel(String relationshipLabel) {
return (relationshipLabel == null) ? getEdgeLabel(vertexPropertyName) : relationshipLabel; return (relationshipLabel == null) ? getEdgeLabel(vertexPropertyName) : relationshipLabel;
} }
...@@ -730,6 +764,12 @@ public class AtlasStructType extends AtlasType { ...@@ -730,6 +764,12 @@ public class AtlasStructType extends AtlasType {
new String[] { "%", "_p" }, new String[] { "%", "_p" },
}; };
private static final char[] IDX_QRY_OFFENDING_CHARS = { '@', '/', ' ' };
private static final char BRACE_OPEN_CHAR = '(';
private static final char BRACE_CLOSE_CHAR = ')';
private static final char DOUBLE_QUOTE_CHAR = '"';
private static final char SPACE_CHAR = ' ';
public enum AtlasRelationshipEdgeDirection { IN, OUT } public enum AtlasRelationshipEdgeDirection { IN, OUT }
} }
} }
intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
...@@ -365,8 +365,8 @@ public class AtlasTypeRegistry { ...@@ -365,8 +365,8 @@ public class AtlasTypeRegistry {
type.resolveReferencesPhase2(this); type.resolveReferencesPhase2(this);
} }
for (AtlasEntityType entityType : registryData.entityDefs.getAllTypes()) { for (AtlasType type : registryData.allTypes.getAllTypes()) {
entityType.resolveReferencesPhase3(this); type.resolveReferencesPhase3(this);
} }
} }
......
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
...@@ -42,23 +42,24 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -42,23 +42,24 @@ public class ClassificationSearchProcessor extends SearchProcessor {
public ClassificationSearchProcessor(SearchContext context) { public ClassificationSearchProcessor(SearchContext context) {
super(context); super(context);
AtlasClassificationType classificationType = context.getClassificationType(); final AtlasClassificationType classificationType = context.getClassificationType();
FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters(); final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes(); final Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes();
Set<String> solrAttributes = new HashSet<>(); final String typeAndSubTypesQryStr = classificationType.getTypeAndAllSubTypesQryStr();
Set<String> gremlinAttributes = new HashSet<>(); final Set<String> solrAttributes = new HashSet<>();
Set<String> allAttributes = new HashSet<>(); final Set<String> gremlinAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
processSearchAttributes(classificationType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes); processSearchAttributes(classificationType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
// for classification search, if any attribute can't be handled by Solr - switch to all Gremlin // for classification search, if any attribute can't be handled by Solr - switch to all Gremlin
boolean useSolrSearch = typeAndSubTypes.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false); boolean useSolrSearch = typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false);
if (useSolrSearch) { if (useSolrSearch) {
StringBuilder solrQuery = new StringBuilder(); StringBuilder solrQuery = new StringBuilder();
constructTypeTestQuery(solrQuery, typeAndSubTypes); constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr);
constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes); constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes);
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")"); String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
......
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
...@@ -87,8 +87,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -87,8 +87,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private final GraphBackedSearchIndexer indexer; private final GraphBackedSearchIndexer indexer;
private final SearchTracker searchTracker; private final SearchTracker searchTracker;
private final int maxResultSetSize; private final int maxResultSetSize;
private final int maxTypesCountInIdxQuery; private final int maxTypesLengthInIdxQuery;
private final int maxTagsCountInIdxQuery; private final int maxTagsLengthInIdxQuery;
@Inject @Inject
EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry, EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry,
...@@ -101,8 +101,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -101,8 +101,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
this.typeRegistry = typeRegistry; this.typeRegistry = typeRegistry;
this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150); this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
this.maxTypesCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10); this.maxTypesLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512);
this.maxTagsCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10); this.maxTagsLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512);
} }
@Override @Override
...@@ -490,8 +490,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -490,8 +490,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
} }
private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) { private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) {
String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery); String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesLengthInIdxQuery);
String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery); String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsLengthInIdxQuery);
StringBuilder queryText = new StringBuilder(); StringBuilder queryText = new StringBuilder();
...@@ -619,23 +619,23 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -619,23 +619,23 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED; return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED;
} }
private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesCountInIdxQuery) { private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesLengthInIdxQuery) {
AtlasClassificationType classification = typeRegistry.getClassificationTypeByName(classificationName); AtlasClassificationType type = typeRegistry.getClassificationTypeByName(classificationName);
Set<String> typeAndSubTypes = classification != null ? classification.getTypeAndAllSubTypes() : null; String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null;
if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) { if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
return String.format("(%s)", StringUtils.join(typeAndSubTypes, " ")); return typeAndSubTypesQryStr;
} }
return ""; return "";
} }
private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesCountInIdxQuery) { private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesLengthInIdxQuery) {
AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName); AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName);
Set<String> typeAndSubTypes = type != null ? type.getTypeAndAllSubTypes() : null; String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null;
if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) { if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
return String.format("(%s)", StringUtils.join(typeAndSubTypes, " ")); return typeAndSubTypesQryStr;
} }
return ""; return "";
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
...@@ -40,12 +40,13 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -40,12 +40,13 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) { public EntitySearchProcessor(SearchContext context) {
super(context); super(context);
final AtlasEntityType entityType = context.getEntityType(); final AtlasEntityType entityType = context.getEntityType();
final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters(); final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes(); final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
final Set<String> solrAttributes = new HashSet<>(); final String typeAndSubTypesQryStr = entityType.getTypeAndAllSubTypesQryStr();
final Set<String> gremlinAttributes = new HashSet<>(); final Set<String> solrAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>(); final Set<String> gremlinAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
final AtlasClassificationType classificationType = context.getClassificationType(); final AtlasClassificationType classificationType = context.getClassificationType();
final boolean filterClassification = classificationType != null && !context.needClassificationProcessor(); final boolean filterClassification = classificationType != null && !context.needClassificationProcessor();
...@@ -53,13 +54,13 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -53,13 +54,13 @@ public class EntitySearchProcessor extends SearchProcessor {
processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes); processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
final boolean typeSearchBySolr = !filterClassification && typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY; final boolean typeSearchBySolr = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES;
final boolean attrSearchBySolr = !filterClassification && CollectionUtils.isNotEmpty(solrAttributes) && canApplySolrFilter(entityType, filterCriteria, false); final boolean attrSearchBySolr = !filterClassification && CollectionUtils.isNotEmpty(solrAttributes) && canApplySolrFilter(entityType, filterCriteria, false);
StringBuilder solrQuery = new StringBuilder(); StringBuilder solrQuery = new StringBuilder();
if (typeSearchBySolr) { if (typeSearchBySolr) {
constructTypeTestQuery(solrQuery, typeAndSubTypes); constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr);
} }
if (attrSearchBySolr) { if (attrSearchBySolr) {
...@@ -169,6 +170,7 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -169,6 +170,7 @@ public class EntitySearchProcessor extends SearchProcessor {
guidQuery.addConditionsFrom(graphQuery); guidQuery.addConditionsFrom(graphQuery);
entityVertices.clear();
getVertices(guidQuery.vertices().iterator(), entityVertices); getVertices(guidQuery.vertices().iterator(), entityVertices);
} }
} else { } else {
......
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
...@@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory; ...@@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Set;
public class FullTextSearchProcessor extends SearchProcessor { public class FullTextSearchProcessor extends SearchProcessor {
...@@ -50,28 +49,26 @@ public class FullTextSearchProcessor extends SearchProcessor { ...@@ -50,28 +49,26 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary // if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor // processing (and rejection) by subsequent EntitySearchProcessor
if (context.getEntityType() != null) { if (context.getEntityType() != null) {
Set<String> typeAndSubTypeNames = context.getEntityType().getTypeAndAllSubTypes(); String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) { if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR); queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
appendIndexQueryValue(typeAndSubTypeNames, queryString);
} else { } else {
LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance", LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getEntityType().getTypeName(), typeAndSubTypeNames.size()); context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length());
} }
} }
// if search includes classification criteria, adding a filter here can help avoid unnecessary // if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if (context.getClassificationType() != null) { if (context.getClassificationType() != null) {
Set<String> typeAndSubTypeNames = context.getClassificationType().getTypeAndAllSubTypes(); String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNames.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) { if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
queryString.append(AND_STR); queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
appendIndexQueryValue(typeAndSubTypeNames, queryString);
} else { } else {
LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance", LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getClassificationType().getTypeName(), typeAndSubTypeNames.size()); context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length());
} }
} }
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
...@@ -39,21 +39,19 @@ import java.util.regex.Pattern; ...@@ -39,21 +39,19 @@ import java.util.regex.Pattern;
public abstract class SearchProcessor { public abstract class SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(SearchProcessor.class); private static final Logger LOG = LoggerFactory.getLogger(SearchProcessor.class);
public static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)"); public static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)");
public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)"); public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)");
public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)"); public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)");
public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150); public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
public static final int MAX_ENTITY_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10); public static final int MAX_QUERY_STR_LENGTH_TYPES = getApplicationProperty(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512);
public static final int MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10); public static final int MAX_QUERY_STR_LENGTH_TAGS = getApplicationProperty(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512);
public static final String AND_STR = " AND "; public static final String AND_STR = " AND ";
public static final String EMPTY_STRING = ""; public static final String EMPTY_STRING = "";
public static final String SPACE_STRING = " "; public static final String SPACE_STRING = " ";
public static final String BRACE_OPEN_STR = "("; public static final String BRACE_OPEN_STR = "(";
public static final String BRACE_CLOSE_STR = ")"; public static final String BRACE_CLOSE_STR = ")";
public static final char DOUBLE_QUOTE = '"';
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>(); private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final char[] OFFENDING_CHARS = { '@', '/', ' ' }; // This can grow as we discover corner cases
static static
{ {
...@@ -181,14 +179,13 @@ public abstract class SearchProcessor { ...@@ -181,14 +179,13 @@ public abstract class SearchProcessor {
return ret; return ret;
} }
protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) { protected void constructTypeTestQuery(StringBuilder solrQuery, String typeAndAllSubTypesQryStr) {
if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) { if (StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
if (solrQuery.length() > 0) { if (solrQuery.length() > 0) {
solrQuery.append(AND_STR); solrQuery.append(AND_STR);
} }
solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":"); solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr);
appendIndexQueryValue(typeAndAllSubTypes, solrQuery);
} }
} }
...@@ -255,7 +252,7 @@ public abstract class SearchProcessor { ...@@ -255,7 +252,7 @@ public abstract class SearchProcessor {
if (OPERATOR_MAP.get(op) != null) { if (OPERATOR_MAP.get(op) != null) {
String qualifiedName = type.getQualifiedAttributeName(attrName); String qualifiedName = type.getQualifiedAttributeName(attrName);
ret = String.format(OPERATOR_MAP.get(op), qualifiedName, escapeIndexQueryValue(attrVal)); ret = String.format(OPERATOR_MAP.get(op), qualifiedName, AtlasStructType.AtlasAttribute.escapeIndexQueryValue(attrVal));
} }
} catch (AtlasBaseException ex) { } catch (AtlasBaseException ex) {
LOG.warn(ex.getMessage()); LOG.warn(ex.getMessage());
...@@ -389,16 +386,6 @@ public abstract class SearchProcessor { ...@@ -389,16 +386,6 @@ public abstract class SearchProcessor {
return ret; return ret;
} }
protected String appendIndexQueryValue(Set<String> values, StringBuilder sb) {
sb.append(BRACE_OPEN_STR);
for (String value : values) {
sb.append(escapeIndexQueryValue(value)).append(SPACE_STRING);
}
sb.append(BRACE_CLOSE_STR);
return sb.toString();
}
private static int getApplicationProperty(String propertyName, int defaultValue) { private static int getApplicationProperty(String propertyName, int defaultValue) {
try { try {
return ApplicationProperties.get().getInt(propertyName, defaultValue); return ApplicationProperties.get().getInt(propertyName, defaultValue);
...@@ -408,25 +395,4 @@ public abstract class SearchProcessor { ...@@ -408,25 +395,4 @@ public abstract class SearchProcessor {
return defaultValue; return defaultValue;
} }
private String escapeIndexQueryValue(String value) {
String ret = value;
if (StringUtils.containsAny(value, OFFENDING_CHARS)) {
boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE;
boolean isQuoteAtEnd = value.charAt(value.length() - 1) == DOUBLE_QUOTE;
if (!isQuoteAtStart) {
if (!isQuoteAtEnd) {
ret = DOUBLE_QUOTE + value + DOUBLE_QUOTE;
} else {
ret = DOUBLE_QUOTE + value;
}
} else if (!isQuoteAtEnd) {
ret = value + DOUBLE_QUOTE;
}
}
return ret;
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment