Skip to content

A
atlas
Commit e0d2cdc2 by Madhan Neethiraj
Options

ATLAS-1981: Cache escaped type-query string to avoid repeated computation

parent f511f272
Showing with 115 additions and 86 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -97,8 +97,8 @@ public final class Constants {
public static final String QUALIFIED_NAME = "Referenceable.qualifiedName";
public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size";
public static final String INDEX_SEARCH_MAX_TYPES_COUNT = "atlas.graph.index.search.max-types-count";
public static final String INDEX_SEARCH_MAX_TAGS_COUNT = "atlas.graph.index.search.max-tags-count";
public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length";
public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.tags.max-query-str-length";
private Constants() {
}
......
intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
......@@ -49,6 +49,7 @@ public class AtlasClassificationType extends AtlasStructType {
private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> allSubTypes = Collections.emptySet();
private Set<String> typeAndAllSubTypes = Collections.emptySet();
private String typeAndAllSubTypesQryStr = "";
public AtlasClassificationType(AtlasClassificationDef classificationDef) {
super(classificationDef);
......@@ -108,6 +109,13 @@ public class AtlasClassificationType extends AtlasStructType {
}
}
@Override
public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
allSubTypes = Collections.unmodifiableSet(allSubTypes);
typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes);
typeAndAllSubTypesQryStr = ""; // will be computed on next access
}
private void addSubType(AtlasClassificationType subType) {
allSubTypes.add(subType.getTypeName());
typeAndAllSubTypes.add(subType.getTypeName());
......@@ -119,11 +127,17 @@ public class AtlasClassificationType extends AtlasStructType {
public Set<String> getAllSuperTypes() { return allSuperTypes; }
public Set<String> getAllSubTypes() {
return Collections.unmodifiableSet(allSubTypes);
public Set<String> getAllSubTypes() { return allSubTypes; }
public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; }
public String getTypeAndAllSubTypesQryStr() {
if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) {
typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes);
}
public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); }
return typeAndAllSubTypesQryStr;
}
public boolean isSuperTypeOf(AtlasClassificationType classificationType) {
return classificationType != null && allSubTypes.contains(classificationType.getTypeName());
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -55,6 +55,7 @@ public class AtlasEntityType extends AtlasStructType {
private Set<String> typeAndAllSuperTypes = Collections.emptySet();
private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap();
private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap();
private String typeAndAllSubTypesQryStr = "";
public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef);
......@@ -150,7 +151,7 @@ public class AtlasEntityType extends AtlasStructType {
allSubTypes = Collections.unmodifiableSet(allSubTypes);
typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes);
typeAndAllSuperTypes = Collections.unmodifiableSet(typeAndAllSuperTypes);
typeAndAllSubTypesQryStr = ""; // will be computed on next access
relationshipAttributes = Collections.unmodifiableMap(relationshipAttributes);
relationshipAttributesType = Collections.unmodifiableMap(relationshipAttributesType);
}
......@@ -218,6 +219,14 @@ public class AtlasEntityType extends AtlasStructType {
return relationshipAttributesType;
}
public String getTypeAndAllSubTypesQryStr() {
if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) {
typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes);
}
return typeAndAllSubTypesQryStr;
}
public boolean hasRelationshipAttribute(String attributeName) {
return relationshipAttributes.containsKey(attributeName);
}
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -38,6 +38,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* class that implements behaviour of a struct-type.
......@@ -713,6 +714,39 @@ public class AtlasStructType extends AtlasType {
return key;
}
public static String escapeIndexQueryValue(Set<String> values) {
StringBuilder sb = new StringBuilder();
sb.append(BRACE_OPEN_CHAR);
for (String value : values) {
sb.append(escapeIndexQueryValue(value)).append(SPACE_CHAR);
}
sb.append(BRACE_CLOSE_CHAR);
return sb.toString();
}
public static String escapeIndexQueryValue(String value) {
String ret = value;
if (StringUtils.containsAny(value, IDX_QRY_OFFENDING_CHARS)) {
boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE_CHAR;
boolean isQuoteAtEnd = value.charAt(value.length() - 1) == DOUBLE_QUOTE_CHAR;
if (!isQuoteAtStart) {
if (!isQuoteAtEnd) {
ret = DOUBLE_QUOTE_CHAR + value + DOUBLE_QUOTE_CHAR;
} else {
ret = DOUBLE_QUOTE_CHAR + value;
}
} else if (!isQuoteAtEnd) {
ret = value + DOUBLE_QUOTE_CHAR;
}
}
return ret;
}
private String getRelationshipEdgeLabel(String relationshipLabel) {
return (relationshipLabel == null) ? getEdgeLabel(vertexPropertyName) : relationshipLabel;
}
......@@ -730,6 +764,12 @@ public class AtlasStructType extends AtlasType {
new String[] { "%", "_p" },
};
private static final char[] IDX_QRY_OFFENDING_CHARS = { '@', '/', ' ' };
private static final char BRACE_OPEN_CHAR = '(';
private static final char BRACE_CLOSE_CHAR = ')';
private static final char DOUBLE_QUOTE_CHAR = '"';
private static final char SPACE_CHAR = ' ';
public enum AtlasRelationshipEdgeDirection { IN, OUT }
}
}
intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
......@@ -365,8 +365,8 @@ public class AtlasTypeRegistry {
type.resolveReferencesPhase2(this);
}
for (AtlasEntityType entityType : registryData.entityDefs.getAllTypes()) {
entityType.resolveReferencesPhase3(this);
for (AtlasType type : registryData.allTypes.getAllTypes()) {
type.resolveReferencesPhase3(this);
}
}
......
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
......@@ -42,23 +42,24 @@ public class ClassificationSearchProcessor extends SearchProcessor {
public ClassificationSearchProcessor(SearchContext context) {
super(context);
AtlasClassificationType classificationType = context.getClassificationType();
FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes();
Set<String> solrAttributes = new HashSet<>();
Set<String> gremlinAttributes = new HashSet<>();
Set<String> allAttributes = new HashSet<>();
final AtlasClassificationType classificationType = context.getClassificationType();
final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
final Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes();
final String typeAndSubTypesQryStr = classificationType.getTypeAndAllSubTypesQryStr();
final Set<String> solrAttributes = new HashSet<>();
final Set<String> gremlinAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
processSearchAttributes(classificationType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
// for classification search, if any attribute can't be handled by Solr - switch to all Gremlin
boolean useSolrSearch = typeAndSubTypes.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false);
boolean useSolrSearch = typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false);
if (useSolrSearch) {
StringBuilder solrQuery = new StringBuilder();
constructTypeTestQuery(solrQuery, typeAndSubTypes);
constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr);
constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes);
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
......
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
......@@ -87,8 +87,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private final GraphBackedSearchIndexer indexer;
private final SearchTracker searchTracker;
private final int maxResultSetSize;
private final int maxTypesCountInIdxQuery;
private final int maxTagsCountInIdxQuery;
private final int maxTypesLengthInIdxQuery;
private final int maxTagsLengthInIdxQuery;
@Inject
EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry,
......@@ -101,8 +101,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
this.typeRegistry = typeRegistry;
this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
this.maxTypesCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10);
this.maxTagsCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10);
this.maxTypesLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512);
this.maxTagsLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512);
}
@Override
......@@ -490,8 +490,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
}
private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) {
String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery);
String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery);
String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesLengthInIdxQuery);
String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsLengthInIdxQuery);
StringBuilder queryText = new StringBuilder();
......@@ -619,23 +619,23 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED;
}
private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesCountInIdxQuery) {
AtlasClassificationType classification = typeRegistry.getClassificationTypeByName(classificationName);
Set<String> typeAndSubTypes = classification != null ? classification.getTypeAndAllSubTypes() : null;
private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesLengthInIdxQuery) {
AtlasClassificationType type = typeRegistry.getClassificationTypeByName(classificationName);
String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null;
if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) {
return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
return typeAndSubTypesQryStr;
}
return "";
}
private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesCountInIdxQuery) {
private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesLengthInIdxQuery) {
AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName);
Set<String> typeAndSubTypes = type != null ? type.getTypeAndAllSubTypes() : null;
String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null;
if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) {
return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
return typeAndSubTypesQryStr;
}
return "";
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
......@@ -43,6 +43,7 @@ public class EntitySearchProcessor extends SearchProcessor {
final AtlasEntityType entityType = context.getEntityType();
final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
final String typeAndSubTypesQryStr = entityType.getTypeAndAllSubTypesQryStr();
final Set<String> solrAttributes = new HashSet<>();
final Set<String> gremlinAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
......@@ -53,13 +54,13 @@ public class EntitySearchProcessor extends SearchProcessor {
processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
final boolean typeSearchBySolr = !filterClassification && typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
final boolean typeSearchBySolr = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES;
final boolean attrSearchBySolr = !filterClassification && CollectionUtils.isNotEmpty(solrAttributes) && canApplySolrFilter(entityType, filterCriteria, false);
StringBuilder solrQuery = new StringBuilder();
if (typeSearchBySolr) {
constructTypeTestQuery(solrQuery, typeAndSubTypes);
constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr);
}
if (attrSearchBySolr) {
......@@ -169,6 +170,7 @@ public class EntitySearchProcessor extends SearchProcessor {
guidQuery.addConditionsFrom(graphQuery);
entityVertices.clear();
getVertices(guidQuery.vertices().iterator(), entityVertices);
}
} else {
......
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
......@@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
public class FullTextSearchProcessor extends SearchProcessor {
......@@ -50,28 +49,26 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
if (context.getEntityType() != null) {
Set<String> typeAndSubTypeNames = context.getEntityType().getTypeAndAllSubTypes();
String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) {
queryString.append(AND_STR);
appendIndexQueryValue(typeAndSubTypeNames, queryString);
if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
} else {
LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length());
}
}
// if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if (context.getClassificationType() != null) {
Set<String> typeAndSubTypeNames = context.getClassificationType().getTypeAndAllSubTypes();
String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNames.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) {
queryString.append(AND_STR);
appendIndexQueryValue(typeAndSubTypeNames, queryString);
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
} else {
LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
context.getClassificationType().getTypeName(), typeAndSubTypeNames.size());
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length());
}
}
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
......@@ -43,17 +43,15 @@ public abstract class SearchProcessor {
public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)");
public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)");
public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
public static final int MAX_ENTITY_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10);
public static final int MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10);
public static final int MAX_QUERY_STR_LENGTH_TYPES = getApplicationProperty(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512);
public static final int MAX_QUERY_STR_LENGTH_TAGS = getApplicationProperty(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512);
public static final String AND_STR = " AND ";
public static final String EMPTY_STRING = "";
public static final String SPACE_STRING = " ";
public static final String BRACE_OPEN_STR = "(";
public static final String BRACE_CLOSE_STR = ")";
public static final char DOUBLE_QUOTE = '"';
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final char[] OFFENDING_CHARS = { '@', '/', ' ' }; // This can grow as we discover corner cases
static
{
......@@ -181,14 +179,13 @@ public abstract class SearchProcessor {
return ret;
}
protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) {
protected void constructTypeTestQuery(StringBuilder solrQuery, String typeAndAllSubTypesQryStr) {
if (StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
if (solrQuery.length() > 0) {
solrQuery.append(AND_STR);
}
solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":");
appendIndexQueryValue(typeAndAllSubTypes, solrQuery);
solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr);
}
}
......@@ -255,7 +252,7 @@ public abstract class SearchProcessor {
if (OPERATOR_MAP.get(op) != null) {
String qualifiedName = type.getQualifiedAttributeName(attrName);
ret = String.format(OPERATOR_MAP.get(op), qualifiedName, escapeIndexQueryValue(attrVal));
ret = String.format(OPERATOR_MAP.get(op), qualifiedName, AtlasStructType.AtlasAttribute.escapeIndexQueryValue(attrVal));
}
} catch (AtlasBaseException ex) {
LOG.warn(ex.getMessage());
......@@ -389,16 +386,6 @@ public abstract class SearchProcessor {
return ret;
}
protected String appendIndexQueryValue(Set<String> values, StringBuilder sb) {
sb.append(BRACE_OPEN_STR);
for (String value : values) {
sb.append(escapeIndexQueryValue(value)).append(SPACE_STRING);
}
sb.append(BRACE_CLOSE_STR);
return sb.toString();
}
private static int getApplicationProperty(String propertyName, int defaultValue) {
try {
return ApplicationProperties.get().getInt(propertyName, defaultValue);
......@@ -408,25 +395,4 @@ public abstract class SearchProcessor {
return defaultValue;
}
private String escapeIndexQueryValue(String value) {
String ret = value;
if (StringUtils.containsAny(value, OFFENDING_CHARS)) {
boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE;
boolean isQuoteAtEnd = value.charAt(value.length() - 1) == DOUBLE_QUOTE;
if (!isQuoteAtStart) {
if (!isQuoteAtEnd) {
ret = DOUBLE_QUOTE + value + DOUBLE_QUOTE;
} else {
ret = DOUBLE_QUOTE + value;
}
} else if (!isQuoteAtEnd) {
ret = value + DOUBLE_QUOTE;
}
}
return ret;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment