Skip to content

A
atlas
Commit 9d432d5d by Le Ma Committed by Sarath Subramanian
Options

ATLAS-3257: Enhance Classification Basic Search

Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org>
parent f31db4c3
Showing with 198 additions and 57 deletions
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java
......@@ -18,8 +18,6 @@
package org.apache.atlas.repository.graphdb;
import java.util.Iterator;
/**
* Represents an index query parameter for use with AtlasGraph queries.
*/
......
graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
......@@ -370,6 +370,11 @@ public class Solr6Index implements IndexProvider {
String analyzer = ParameterType.STRING_ANALYZER.findParameter(information.getParameters(), null);
if (analyzer != null) {
//If the key have a tokenizer, we try to get it by reflection
// some referred classes might not be able to be found via SystemClassLoader
// since they might be associated with other classloader, in this situation
// ClassNotFound exception will be thrown. instead of using SystemClassLoader
// for all classes, we find its classloader first and then load the class, please
// call - instantiateUsingClassLoader()
try {
((Constructor<Tokenizer>) ClassLoader.getSystemClassLoader().loadClass(analyzer)
.getConstructor()).newInstance();
......@@ -389,6 +394,17 @@ public class Solr6Index implements IndexProvider {
}
}
private void instantiateUsingClassLoader(String analyzer) throws PermanentBackendException {
if (analyzer == null) return;
try {
Class analyzerClass = Class.forName(analyzer);
ClassLoader cl = analyzerClass.getClassLoader();
((Constructor<Tokenizer>) cl.loadClass(analyzer).getConstructor()).newInstance();
} catch (final ReflectiveOperationException e) {
throw new PermanentBackendException(e.getMessage(),e);
}
}
@Override
public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever information,
BaseTransaction tx) throws BackendException {
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
......@@ -33,10 +33,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
*
*/
public abstract class AbstractGraphDatabaseTest {
protected static final String WEIGHT_PROPERTY = "weight";
......@@ -112,13 +108,8 @@ public abstract class AbstractGraphDatabaseTest {
//ok
t.printStackTrace();
}
}
protected final <V, E> AtlasGraph<V, E> getGraph() {
if (graph == null) {
graph = new AtlasJanusGraph();
......@@ -184,5 +175,4 @@ public abstract class AbstractGraphDatabaseTest {
return exceptionThrown;
}
}
}
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
......@@ -544,6 +544,8 @@ public final class TestUtilsV2 {
public static final String COLUMN_TYPE = "column_type";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
public static final String FETL_CLASSIFICATION = "fetl" + CLASSIFICATION;
public static final String PII = "PII";
public static final String PHI = "PHI";
public static final String SUPER_TYPE_NAME = "Base";
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
......@@ -111,7 +111,7 @@ public class EntitySearchProcessor extends SearchProcessor {
StringBuilder indexQuery = new StringBuilder();
if (typeSearchByIndex) {
constructTypeTestQuery(indexQuery, typeAndSubTypesQryStr);
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(indexQuery, typeAndSubTypesQryStr);
// TypeName check to be done in-memory as well to address ATLAS-2121 (case sensitivity)
inMemoryPredicate = typeNamePredicate;
......@@ -131,9 +131,8 @@ public class EntitySearchProcessor extends SearchProcessor {
}
if (indexQuery.length() > 0) {
if (context.getSearchParameters().getExcludeDeletedEntities()) {
constructStateTestQuery(indexQuery);
}
graphIndexQueryBuilder.addActiveStateQueryFilter(indexQuery);
String indexQueryString = STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")");
......
repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.discovery.SearchProcessor.INDEX_SEARCH_PREFIX;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import org.apache.atlas.repository.Constants;
import org.apache.commons.lang3.StringUtils;
public class GraphIndexQueryBuilder {
SearchContext context;
GraphIndexQueryBuilder(SearchContext context) {
this.context = context;
}
void addClassificationTypeFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
String classificationName = context.getSearchParameters().getClassification();
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName)
.append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
.append('\"').append(':').append(classificationName).append(")");
}
}
void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
String classificationTypesQryStr = context.getClassificationTypesQryStr();
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
.append("\"").append(":" + classificationTypesQryStr).append(" OR ").append(INDEX_SEARCH_PREFIX)
.append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + classificationTypesQryStr).append(")");
}
}
void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) {
if (indexQuery != null && context.getClassificationType() != null) {
if (context.getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() == MATCH_ALL_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"")
.append(CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]")
.append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"")
.append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]").append(")");
} else if (context.getClassificationType() == MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append("-").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
.append("\"").append(":" + "[* TO *]").append(" AND ").append("-")
.append(INDEX_SEARCH_PREFIX).append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
.append("\"").append(":" + "[* TO *]").append(")");
}
}
}
void addActiveStateQueryFilter(StringBuilder indexQuery){
if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery != null) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(STATE_PROPERTY_KEY)
.append("\"").append(":" + "ACTIVE").append(")");
}
}
void addTypeAndSubTypesQueryFilter(StringBuilder indexQuery, String typeAndAllSubTypesQryStr) {
if (indexQuery != null && StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
if (indexQuery.length() > 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX + "\"").append(Constants.TYPE_NAME_PROPERTY_KEY)
.append("\":").append(typeAndAllSubTypesQryStr).append(")");
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
......@@ -63,6 +63,7 @@ public class SearchContext {
private final Set<String> entityAttributes;
private final AtlasEntityType entityType;
private final AtlasClassificationType classificationType;
private final String classificationName;
private SearchProcessor searchProcessor;
private boolean terminateSearch = false;
private final Set<String> typeAndSubTypes;
......@@ -74,10 +75,8 @@ public class SearchContext {
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
String classificationName = searchParameters.getClassification();
this.classificationName = searchParameters.getClassification();
this.searchParameters = searchParameters;
this.typeRegistry = typeRegistry;
this.graph = graph;
......@@ -92,7 +91,7 @@ public class SearchContext {
}
// Validate if the classification exists
if (StringUtils.isNotEmpty(classificationName) && classificationType == null) {
if ((StringUtils.isNotEmpty(classificationName) && classificationType == null && !classificationName.contains(WILDCARD_CLASSIFICATIONS))) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName);
}
......@@ -239,7 +238,21 @@ public class SearchContext {
}
boolean needClassificationProcessor() {
return classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()));
return (classificationType != null || isWildCardSearch());
}
boolean isBuiltInClassificationType() {
return getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION
|| getClassificationType() == MATCH_ALL_CLASSIFIED
|| getClassificationType() == MATCH_ALL_NOT_CLASSIFIED;
}
boolean isWildCardSearch () {
String classification = getSearchParameters().getClassification();
if (StringUtils.isNotEmpty(classification) && getClassificationType() == null) {
return classification.contains("*");
}
return false;
}
boolean needEntityProcessor() {
......@@ -263,7 +276,10 @@ public class SearchContext {
private void validateAttributes(final AtlasStructType structType, final String... attributeNames) throws AtlasBaseException {
for (String attributeName : attributeNames) {
if (StringUtils.isNotEmpty(attributeName) && structType.getAttributeType(attributeName) == null) {
if (StringUtils.isNotEmpty(attributeName) && (structType == null || structType.getAttributeType(attributeName) == null)) {
if (structType == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL");
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, structType.getTypeName());
}
}
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
......@@ -113,10 +113,11 @@ public abstract class SearchProcessor {
protected final SearchContext context;
protected SearchProcessor nextProcessor;
protected Predicate inMemoryPredicate;
protected GraphIndexQueryBuilder graphIndexQueryBuilder;
protected SearchProcessor(SearchContext context) {
this.context = context;
this.graphIndexQueryBuilder = new GraphIndexQueryBuilder(context);
}
public void addProcessor(SearchProcessor processor) {
......@@ -239,13 +240,30 @@ public abstract class SearchProcessor {
return ret;
}
protected void constructTypeTestQuery(StringBuilder indexQuery, String typeAndAllSubTypesQryStr) {
if (StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
if (indexQuery.length() > 0) {
indexQuery.append(AND_STR);
protected void filterWhiteSpaceClassification(List<AtlasVertex> entityVertices) {
if (CollectionUtils.isNotEmpty(entityVertices)) {
boolean hasExactMatch = false;
Iterator<AtlasVertex> it = entityVertices.iterator();
while (it.hasNext()) {
AtlasVertex entityVertex = it.next();
List<String> classificationNames = AtlasGraphUtilsV2.getClassificationNames(entityVertex);
if (CollectionUtils.isNotEmpty(classificationNames) && classificationNames.contains(context.getClassificationType().getTypeName())) {
hasExactMatch = true;
}
if (hasExactMatch) continue;
classificationNames = AtlasGraphUtilsV2.getPropagatedClassificationNames(entityVertex);
if (CollectionUtils.isNotEmpty(classificationNames) && classificationNames.contains(context.getClassificationType().getTypeName())) {
hasExactMatch = true;
}
indexQuery.append(INDEX_SEARCH_PREFIX + "\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr);
if (!hasExactMatch) {
it.remove();
}
}
}
}
......@@ -322,14 +340,6 @@ public abstract class SearchProcessor {
}
}
protected void constructStateTestQuery(StringBuilder indexQuery) {
if (indexQuery.length() > 0) {
indexQuery.append(AND_STR);
}
indexQuery.append(INDEX_SEARCH_PREFIX + "\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
}
private boolean isIndexSearchable(FilterCriteria filterCriteria, AtlasStructType structType) throws AtlasBaseException {
String qualifiedName = structType.getQualifiedAttributeName(filterCriteria.getAttributeName());
Set<String> indexedKeys = context.getIndexedKeys();
......@@ -747,7 +757,7 @@ public abstract class SearchProcessor {
return false;
}
protected List<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, List<AtlasVertex> vertices) {
protected Collection<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, Collection<AtlasVertex> vertices) {
if (idxQueryResult != null) {
while (idxQueryResult.hasNext()) {
AtlasVertex vertex = idxQueryResult.next().getVertex();
......@@ -759,7 +769,7 @@ public abstract class SearchProcessor {
return vertices;
}
protected List<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, List<AtlasVertex> vertices) {
protected Collection<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, Collection<AtlasVertex> vertices) {
if (iterator != null) {
while (iterator.hasNext()) {
AtlasVertex vertex = iterator.next();
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
......@@ -50,7 +50,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
......@@ -58,9 +58,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
......@@ -634,4 +636,21 @@ public class AtlasGraphUtilsV2 {
public static String getIndexSearchPrefix() {
return INDEX_SEARCH_PREFIX;
}
public static List<String> getClassificationNames(AtlasVertex entityVertex) {
return getClassificationNamesHelper(entityVertex, CLASSIFICATION_NAMES_KEY);
}
public static List<String> getPropagatedClassificationNames(AtlasVertex entityVertex) {
return getClassificationNamesHelper(entityVertex, PROPAGATED_CLASSIFICATION_NAMES_KEY);
}
private static List<String> getClassificationNamesHelper(AtlasVertex entityVertex, String propertyKey) {
List<String> classificationNames = null;
String classificationNamesString = entityVertex.getProperty(propertyKey, String.class);
if (StringUtils.isNotEmpty(classificationNamesString)) {
classificationNames = Arrays.asList(classificationNamesString.split("\\|"));
}
return classificationNames;
}
}
webapp/src/test/resources/json/search-parameters/tag-filters.json
......@@ -12,22 +12,5 @@
"classification":"fooTag"
},
"expectedCount": 1
},
{
"testDescription": "Search for exact Tag name order by",
"searchParameters": {
"entityFilters":null,
"tagFilters":null,
"attributes":null,
"query":null,
"excludeDeletedEntities":true,
"limit":25,
"typeName":null,
"sortBy": "name",
"sortOrder": "DESCENDING",
"classification":"fooTag"
},
"expectedCount": 1
}
]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment