Skip to content

A
atlas
Commit 9d432d5d by Le Ma Committed by Sarath Subramanian
Options

ATLAS-3257: Enhance Classification Basic Search

Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org>
parent f31db4c3
Showing with 524 additions and 213 deletions
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java
......@@ -18,8 +18,6 @@
package org.apache.atlas.repository.graphdb;
import java.util.Iterator;
/**
* Represents an index query parameter for use with AtlasGraph queries.
*/
......
graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
......@@ -370,6 +370,11 @@ public class Solr6Index implements IndexProvider {
String analyzer = ParameterType.STRING_ANALYZER.findParameter(information.getParameters(), null);
if (analyzer != null) {
//If the key have a tokenizer, we try to get it by reflection
// some referred classes might not be able to be found via SystemClassLoader
// since they might be associated with other classloader, in this situation
// ClassNotFound exception will be thrown. instead of using SystemClassLoader
// for all classes, we find its classloader first and then load the class, please
// call - instantiateUsingClassLoader()
try {
((Constructor<Tokenizer>) ClassLoader.getSystemClassLoader().loadClass(analyzer)
.getConstructor()).newInstance();
......@@ -389,6 +394,17 @@ public class Solr6Index implements IndexProvider {
}
}
private void instantiateUsingClassLoader(String analyzer) throws PermanentBackendException {
if (analyzer == null) return;
try {
Class analyzerClass = Class.forName(analyzer);
ClassLoader cl = analyzerClass.getClassLoader();
((Constructor<Tokenizer>) cl.loadClass(analyzer).getConstructor()).newInstance();
} catch (final ReflectiveOperationException e) {
throw new PermanentBackendException(e.getMessage(),e);
}
}
@Override
public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever information,
BaseTransaction tx) throws BackendException {
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
......@@ -33,10 +33,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
*
*/
public abstract class AbstractGraphDatabaseTest {
protected static final String WEIGHT_PROPERTY = "weight";
......@@ -112,13 +108,8 @@ public abstract class AbstractGraphDatabaseTest {
//ok
t.printStackTrace();
}
}
protected final <V, E> AtlasGraph<V, E> getGraph() {
if (graph == null) {
graph = new AtlasJanusGraph();
......@@ -184,5 +175,4 @@ public abstract class AbstractGraphDatabaseTest {
return exceptionThrown;
}
}
}
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
......@@ -544,6 +544,8 @@ public final class TestUtilsV2 {
public static final String COLUMN_TYPE = "column_type";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
public static final String FETL_CLASSIFICATION = "fetl" + CLASSIFICATION;
public static final String PII = "PII";
public static final String PHI = "PHI";
public static final String SUPER_TYPE_NAME = "Base";
......
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
......@@ -20,7 +20,6 @@ package org.apache.atlas.discovery;
import org.apache.atlas.SortOrder;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
......@@ -31,11 +30,10 @@ import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -45,7 +43,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -53,95 +50,139 @@ import java.util.Set;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC;
/**
* This class is needed when this is a registered classification type or wildcard search,
* registered classification includes special type as well. (tag filters will be ignored, and front-end should not enable
* tag-filter for special classification types, including wildcard search - classification name contains *)
*/
public class ClassificationSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(ClassificationSearchProcessor.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("ClassificationSearchProcessor");
private final AtlasIndexQuery indexQuery;
private final AtlasGraphQuery tagGraphQueryWithAttributes;
private final AtlasGraphQuery entityGraphQueryTraitNames;
private Predicate entityPredicateTraitNames;
private static final Logger LOG = LoggerFactory.getLogger(ClassificationSearchProcessor.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("ClassificationSearchProcessor");
private final AtlasIndexQuery indexQuery;
private final AtlasIndexQuery classificationIndexQuery;
private final AtlasGraphQuery tagGraphQueryWithAttributes;
private final Map<String, Object> gremlinQueryBindings;
private final String gremlinTagFilterQuery;
private final String gremlinTagFilterQuery;
private final Map<String, Object> gremlinQueryBindings;
// Some index engines may take space as a delimiter, when basic search is
// executed, unsatisfying results may be returned.
// eg, an entity A has classification "cls" and B has "cls 1"
// when user execute a exact search for "cls", only A should be returned
// but both A and B are returned. To avoid this, we should filter the res.
private boolean whiteSpaceFilter = false;
public ClassificationSearchProcessor(SearchContext context) {
super(context);
final AtlasClassificationType classificationType = context.getClassificationType();
final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
final AtlasClassificationType classificationType = context.getClassificationType();
final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getClassificationTypes();
final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
final String sortBy = context.getSearchParameters().getSortBy();
final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
final boolean isBuiltInType = context.isBuiltInClassificationType();
final boolean isWildcardSearch = context.isWildCardSearch();
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
// for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
boolean useIndexSearch = classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION &&
classificationType != MATCH_ALL_CLASSIFIED &&
classificationType != MATCH_ALL_NOT_CLASSIFIED &&
typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS &&
CollectionUtils.isEmpty(graphAttributes) &&
canApplyIndexFilter(classificationType, filterCriteria, false);
/* for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
There are four cases in the classification type :
1. unique classification type, including not classified, single wildcard (*), match all classified
2. wildcard search, including starting/ending/mid wildcard, like cls*, *c*, *ion.
3. registered classification type, like PII, PHI
4. classification is not present in the search parameter
each of above cases with either has empty/or not tagFilters
*/
final boolean useIndexSearchForEntity = (classificationType != null || isWildcardSearch) &&
filterCriteria == null &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
/* If classification's attributes can be applied index filter, we can use direct index
* to query classification index as well.
*/
final boolean useIndexSearchForClassification = (classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION &&
classificationType != MATCH_ALL_CLASSIFIED &&
classificationType != MATCH_ALL_NOT_CLASSIFIED && !isWildcardSearch) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
CollectionUtils.isEmpty(graphAttributes) &&
canApplyIndexFilter(classificationType, filterCriteria, false);
AtlasGraph graph = context.getGraph();
if (useIndexSearch) {
StringBuilder indexQuery = new StringBuilder();
// index query directly on entity
if (useIndexSearchForEntity) {
constructTypeTestQuery(indexQuery, typeAndSubTypesQryStr);
constructFilterQuery(indexQuery, classificationType, filterCriteria, indexAttributes);
StringBuilder queryString = new StringBuilder();
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
String indexQueryString = STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")");
if (isWildcardSearch) {
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
// tagFilters is not allowed in wildcard search
graphIndexQueryBuilder.addClassificationTypeFilter(queryString);
} else {
if (isBuiltInType) {
this.indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
// tagFilters is not allowed in unique classificationType search
graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString);
Predicate typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator()
.generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(typeNamePredicate, attributePredicate);
} else {
inMemoryPredicate = typeNamePredicate;
} else {
// only registered classification will search for subtypes
graphIndexQueryBuilder.addClassificationAndSubTypesQueryFilter(queryString);
whiteSpaceFilter = true;
}
}
String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
LOG.debug("Using query string '{}'.", indexQuery);
} else {
indexQuery = null;
}
if (context.getSearchParameters().getTagFilters() != null) {
// Now filter on the tag attributes
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
// index query directly on classification
if (useIndexSearchForClassification) {
StringBuilder queryString = new StringBuilder();
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString, typeAndSubTypesQryStr);
constructFilterQuery(queryString, classificationType, filterCriteria, indexAttributes);
String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
} else {
classificationIndexQuery = null;
}
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationType, filterCriteria, allAttributes, graph.query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes));
entityGraphQueryTraitNames = null;
entityPredicateTraitNames = null;
// only registered classification will search with tag filters
if (!isWildcardSearch && !isBuiltInType && !graphAttributes.isEmpty()) {
gremlinQueryBindings = new HashMap<>();
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationType, filterCriteria, allAttributes,
graph.query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes));
gremlinQueryBindings = new HashMap<>();
StringBuilder gremlinQuery = new StringBuilder();
gremlinQuery.append("g.V().has('__guid', T.in, guids)");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER));
gremlinQuery.append(".as('e').out()");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
// After filtering on tags go back to e and output the list of entity vertices
gremlinQuery.append(".back('e').toList()");
......@@ -154,50 +195,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
}
} else {
tagGraphQueryWithAttributes = null;
List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
orConditions.add(graph.query().createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.orPredicate(
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
orConditions.add(graph.query().createChildQuery().has(GUID_PROPERTY_KEY, NOT_EQUAL, null).has(ENTITY_TYPE_PROPERTY_KEY, NOT_EQUAL, null)
.has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null).has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.andPredicate(
SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else {
orConditions.add(graph.query().createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
orConditions.add(graph.query().createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
entityGraphQueryTraitNames = graph.query().or(orConditions);
entityPredicateTraitNames = PredicateUtils.orPredicate(
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class));
}
if (context.getSearchParameters().getExcludeDeletedEntities()) {
entityGraphQueryTraitNames.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator().generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
entityPredicateTraitNames = PredicateUtils.andPredicate(entityPredicateTraitNames, activePredicate);
}
if (sortBy != null && !sortBy.isEmpty()) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
entityGraphQueryTraitNames.orderBy(sortBy, qrySortOrder);
}
tagGraphQueryWithAttributes = null;
gremlinTagFilterQuery = null;
gremlinQueryBindings = null;
gremlinQueryBindings = null;
}
}
......@@ -218,7 +218,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
try {
final int startIdx = context.getSearchParameters().getOffset();
final int limit = context.getSearchParameters().getLimit();
final boolean activeOnly = context.getSearchParameters().getExcludeDeletedEntities();
// query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
// have been dropped: like non-active-entities or duplicate-entities (same entity pointed to by multiple
......@@ -232,6 +231,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
final List<AtlasVertex> entityVertices = new ArrayList<>();
final List<AtlasVertex> classificationVertices = new ArrayList<>();
final String sortBy = context.getSearchParameters().getSortBy();
final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
for (; ret.size() < limit; qryOffset += limit) {
entityVertices.clear();
classificationVertices.clear();
......@@ -242,34 +244,31 @@ public class ClassificationSearchProcessor extends SearchProcessor {
break;
}
final boolean isLastResultPage;
boolean isLastResultPage = true;
if (indexQuery != null) {
Iterator<AtlasIndexQuery.Result> queryResult = indexQuery.vertices(qryOffset, limit);
getVerticesFromIndexQueryResult(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
Iterator<AtlasIndexQuery.Result> queryResult;
if (StringUtils.isNotEmpty(sortBy)) {
Order qrySortOrder = sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc;
queryResult = indexQuery.vertices(qryOffset, limit, sortBy, qrySortOrder);
} else {
queryResult = indexQuery.vertices(qryOffset, limit);
}
// Do in-memory filtering before the graph query
CollectionUtils.filter(classificationVertices, inMemoryPredicate);
getVerticesFromIndexQueryResult(queryResult, entityVertices);
isLastResultPage = entityVertices.size() < limit;
} else {
if (context.getSearchParameters().getTagFilters() == null) {
// We can use single graph query to determine in this case
Iterator<AtlasVertex> queryResult = entityGraphQueryTraitNames.vertices(qryOffset, limit).iterator();
if (classificationIndexQuery != null) {
Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
getVertices(queryResult, entityVertices);
getVerticesFromIndexQueryResult(queryResult, classificationVertices);
isLastResultPage = entityVertices.size() < limit;
} else {
} else if (context.getSearchParameters().getTagFilters() != null) {
Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
getVertices(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
// Do in-memory filtering before the graph query
CollectionUtils.filter(classificationVertices, inMemoryPredicate);
}
}
......@@ -282,10 +281,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
for (AtlasEdge edge : edges) {
AtlasVertex entityVertex = edge.getOutVertex();
if (activeOnly && AtlasGraphUtilsV2.getState(entityVertex) != AtlasEntity.Status.ACTIVE) {
continue;
}
String guid = AtlasGraphUtilsV2.getIdFromVertex(entityVertex);
if (processedGuids.contains(guid)) {
......@@ -299,6 +294,10 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
}
if (whiteSpaceFilter) {
filterWhiteSpaceClassification(entityVertices);
}
super.filter(entityVertices);
resultIdx = collectResultVertices(ret, startIdx, limit, resultIdx, entityVertices);
......@@ -346,8 +345,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.warn(e.getMessage(), e);
}
}
} else if (entityPredicateTraitNames != null) {
CollectionUtils.filter(entityVertices, entityPredicateTraitNames);
}
super.filter(entityVertices);
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
......@@ -111,7 +111,7 @@ public class EntitySearchProcessor extends SearchProcessor {
StringBuilder indexQuery = new StringBuilder();
if (typeSearchByIndex) {
constructTypeTestQuery(indexQuery, typeAndSubTypesQryStr);
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(indexQuery, typeAndSubTypesQryStr);
// TypeName check to be done in-memory as well to address ATLAS-2121 (case sensitivity)
inMemoryPredicate = typeNamePredicate;
......@@ -131,9 +131,8 @@ public class EntitySearchProcessor extends SearchProcessor {
}
if (indexQuery.length() > 0) {
if (context.getSearchParameters().getExcludeDeletedEntities()) {
constructStateTestQuery(indexQuery);
}
graphIndexQueryBuilder.addActiveStateQueryFilter(indexQuery);
String indexQueryString = STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")");
......
repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.discovery.SearchProcessor.INDEX_SEARCH_PREFIX;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import org.apache.atlas.repository.Constants;
import org.apache.commons.lang3.StringUtils;
public class GraphIndexQueryBuilder {
SearchContext context;
GraphIndexQueryBuilder(SearchContext context) {
this.context = context;
}
void addClassificationTypeFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
String classificationName = context.getSearchParameters().getClassification();
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName)
.append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
.append('\"').append(':').append(classificationName).append(")");
}
}
void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) {
if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
String classificationTypesQryStr = context.getClassificationTypesQryStr();
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
.append("\"").append(":" + classificationTypesQryStr).append(" OR ").append(INDEX_SEARCH_PREFIX)
.append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + classificationTypesQryStr).append(")");
}
}
void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) {
if (indexQuery != null && context.getClassificationType() != null) {
if (context.getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() == MATCH_ALL_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"")
.append(CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]")
.append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"")
.append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]").append(")");
} else if (context.getClassificationType() == MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append("-").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
.append("\"").append(":" + "[* TO *]").append(" AND ").append("-")
.append(INDEX_SEARCH_PREFIX).append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
.append("\"").append(":" + "[* TO *]").append(")");
}
}
}
void addActiveStateQueryFilter(StringBuilder indexQuery){
if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery != null) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(STATE_PROPERTY_KEY)
.append("\"").append(":" + "ACTIVE").append(")");
}
}
void addTypeAndSubTypesQueryFilter(StringBuilder indexQuery, String typeAndAllSubTypesQryStr) {
if (indexQuery != null && StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
if (indexQuery.length() > 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX + "\"").append(Constants.TYPE_NAME_PROPERTY_KEY)
.append("\":").append(typeAndAllSubTypesQryStr).append(")");
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
......@@ -63,6 +63,7 @@ public class SearchContext {
private final Set<String> entityAttributes;
private final AtlasEntityType entityType;
private final AtlasClassificationType classificationType;
private final String classificationName;
private SearchProcessor searchProcessor;
private boolean terminateSearch = false;
private final Set<String> typeAndSubTypes;
......@@ -74,10 +75,8 @@ public class SearchContext {
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
String classificationName = searchParameters.getClassification();
this.classificationName = searchParameters.getClassification();
this.searchParameters = searchParameters;
this.typeRegistry = typeRegistry;
this.graph = graph;
......@@ -92,7 +91,7 @@ public class SearchContext {
}
// Validate if the classification exists
if (StringUtils.isNotEmpty(classificationName) && classificationType == null) {
if ((StringUtils.isNotEmpty(classificationName) && classificationType == null && !classificationName.contains(WILDCARD_CLASSIFICATIONS))) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName);
}
......@@ -239,7 +238,21 @@ public class SearchContext {
}
boolean needClassificationProcessor() {
return classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()));
return (classificationType != null || isWildCardSearch());
}
boolean isBuiltInClassificationType() {
return getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION
|| getClassificationType() == MATCH_ALL_CLASSIFIED
|| getClassificationType() == MATCH_ALL_NOT_CLASSIFIED;
}
boolean isWildCardSearch () {
String classification = getSearchParameters().getClassification();
if (StringUtils.isNotEmpty(classification) && getClassificationType() == null) {
return classification.contains("*");
}
return false;
}
boolean needEntityProcessor() {
......@@ -263,7 +276,10 @@ public class SearchContext {
private void validateAttributes(final AtlasStructType structType, final String... attributeNames) throws AtlasBaseException {
for (String attributeName : attributeNames) {
if (StringUtils.isNotEmpty(attributeName) && structType.getAttributeType(attributeName) == null) {
if (StringUtils.isNotEmpty(attributeName) && (structType == null || structType.getAttributeType(attributeName) == null)) {
if (structType == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL");
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, structType.getTypeName());
}
}
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
......@@ -110,13 +110,14 @@ public abstract class SearchProcessor {
OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_NULL, getNotNullPredicateGenerator());
}
protected final SearchContext context;
protected SearchProcessor nextProcessor;
protected Predicate inMemoryPredicate;
protected final SearchContext context;
protected SearchProcessor nextProcessor;
protected Predicate inMemoryPredicate;
protected GraphIndexQueryBuilder graphIndexQueryBuilder;
protected SearchProcessor(SearchContext context) {
this.context = context;
this.graphIndexQueryBuilder = new GraphIndexQueryBuilder(context);
}
public void addProcessor(SearchProcessor processor) {
......@@ -239,13 +240,30 @@ public abstract class SearchProcessor {
return ret;
}
protected void constructTypeTestQuery(StringBuilder indexQuery, String typeAndAllSubTypesQryStr) {
if (StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
if (indexQuery.length() > 0) {
indexQuery.append(AND_STR);
}
protected void filterWhiteSpaceClassification(List<AtlasVertex> entityVertices) {
if (CollectionUtils.isNotEmpty(entityVertices)) {
boolean hasExactMatch = false;
Iterator<AtlasVertex> it = entityVertices.iterator();
while (it.hasNext()) {
AtlasVertex entityVertex = it.next();
List<String> classificationNames = AtlasGraphUtilsV2.getClassificationNames(entityVertex);
if (CollectionUtils.isNotEmpty(classificationNames) && classificationNames.contains(context.getClassificationType().getTypeName())) {
hasExactMatch = true;
}
if (hasExactMatch) continue;
classificationNames = AtlasGraphUtilsV2.getPropagatedClassificationNames(entityVertex);
if (CollectionUtils.isNotEmpty(classificationNames) && classificationNames.contains(context.getClassificationType().getTypeName())) {
hasExactMatch = true;
}
indexQuery.append(INDEX_SEARCH_PREFIX + "\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr);
if (!hasExactMatch) {
it.remove();
}
}
}
}
......@@ -322,14 +340,6 @@ public abstract class SearchProcessor {
}
}
protected void constructStateTestQuery(StringBuilder indexQuery) {
if (indexQuery.length() > 0) {
indexQuery.append(AND_STR);
}
indexQuery.append(INDEX_SEARCH_PREFIX + "\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
}
private boolean isIndexSearchable(FilterCriteria filterCriteria, AtlasStructType structType) throws AtlasBaseException {
String qualifiedName = structType.getQualifiedAttributeName(filterCriteria.getAttributeName());
Set<String> indexedKeys = context.getIndexedKeys();
......@@ -747,7 +757,7 @@ public abstract class SearchProcessor {
return false;
}
protected List<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, List<AtlasVertex> vertices) {
protected Collection<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, Collection<AtlasVertex> vertices) {
if (idxQueryResult != null) {
while (idxQueryResult.hasNext()) {
AtlasVertex vertex = idxQueryResult.next().getVertex();
......@@ -759,7 +769,7 @@ public abstract class SearchProcessor {
return vertices;
}
protected List<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, List<AtlasVertex> vertices) {
protected Collection<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, Collection<AtlasVertex> vertices) {
if (iterator != null) {
while (iterator.hasNext()) {
AtlasVertex vertex = iterator.next();
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
......@@ -50,7 +50,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
......@@ -58,9 +58,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
......@@ -634,4 +636,21 @@ public class AtlasGraphUtilsV2 {
public static String getIndexSearchPrefix() {
return INDEX_SEARCH_PREFIX;
}
public static List<String> getClassificationNames(AtlasVertex entityVertex) {
return getClassificationNamesHelper(entityVertex, CLASSIFICATION_NAMES_KEY);
}
public static List<String> getPropagatedClassificationNames(AtlasVertex entityVertex) {
return getClassificationNamesHelper(entityVertex, PROPAGATED_CLASSIFICATION_NAMES_KEY);
}
private static List<String> getClassificationNamesHelper(AtlasVertex entityVertex, String propertyKey) {
List<String> classificationNames = null;
String classificationNamesString = entityVertex.getProperty(propertyKey, String.class);
if (StringUtils.isNotEmpty(classificationNamesString)) {
classificationNames = Arrays.asList(classificationNamesString.split("\\|"));
}
return classificationNames;
}
}
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
......@@ -17,10 +17,16 @@
*/
package org.apache.atlas.web.adapters;
import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE;
import static org.apache.atlas.TestUtilsV2.DATABASE_TYPE;
import static org.apache.atlas.TestUtilsV2.TABLE_TYPE;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
......@@ -36,6 +42,7 @@ import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.web.rest.DiscoveryREST;
import org.apache.atlas.web.rest.EntityREST;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -53,28 +60,27 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Guice(modules = {TestModules.TestOnlyModule.class})
public class TestEntitiesREST {
private static final Logger LOG = LoggerFactory.getLogger(TestEntitiesREST.class);
@Inject
AtlasTypeRegistry typeRegistry;
private AtlasTypeRegistry typeRegistry;
@Inject
private AtlasTypeDefStore typeStore;
@Inject
private EntityREST entityREST;
private List<String> createdGuids = new ArrayList<>();
private AtlasEntity dbEntity;
private AtlasEntity tableEntity;
private DiscoveryREST discoveryREST;
@Inject
private EntityREST entityREST;
private List<AtlasEntity> columns;
private AtlasEntity dbEntity;
private AtlasEntity tableEntity;
private AtlasEntity tableEntity2;
private List<AtlasEntity> columns;
private List<AtlasEntity> columns2;
private SearchParameters searchParameters = new SearchParameters();
private Map<String, List<String>> createdGuids = new HashMap<>();
@BeforeClass
public void setUp() throws Exception {
......@@ -88,12 +94,17 @@ public class TestEntitiesREST {
}
}
dbEntity = TestUtilsV2.createDBEntity();
tableEntity = TestUtilsV2.createTableEntity(dbEntity);
dbEntity = TestUtilsV2.createDBEntity();
tableEntity = TestUtilsV2.createTableEntity(dbEntity);
tableEntity2 = TestUtilsV2.createTableEntity(dbEntity);
final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity);
final AtlasEntity colEntity2 = TestUtilsV2.createColumnEntity(tableEntity2);
columns = new ArrayList<AtlasEntity>() {{ add(colEntity); }};
columns2 = new ArrayList<AtlasEntity>() {{ add(colEntity2); }};
final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity);
columns = new ArrayList<AtlasEntity>() {{ add(colEntity); }};
tableEntity.setAttribute("columns", getObjIdList(columns));
tableEntity2.setAttribute("columns", getObjIdList(columns2));
}
@AfterMethod
......@@ -107,35 +118,196 @@ public class TestEntitiesREST {
entities.addEntity(dbEntity);
entities.addEntity(tableEntity);
entities.addEntity(tableEntity2);
for (AtlasEntity column : columns) {
entities.addReferredEntity(column);
}
for (AtlasEntity column : columns2) {
entities.addReferredEntity(column);
}
EntityMutationResponse response = entityREST.createOrUpdate(entities);
List<AtlasEntityHeader> guids = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 3);
Assert.assertEquals(guids.size(), 5);
for (AtlasEntityHeader header : guids) {
createdGuids.add(header.getGuid());
if (!createdGuids.containsKey(header.getTypeName())) {
createdGuids.put(header.getTypeName(), new ArrayList<>());
}
createdGuids.get(header.getTypeName()).add(header.getGuid());
}
}
@Test(dependsOnMethods = "testCreateOrUpdateEntities")
public void testTagToMultipleEntities() throws Exception{
AtlasClassification tag = new AtlasClassification(TestUtilsV2.CLASSIFICATION, new HashMap<String, Object>() {{ put("tag", "tagName"); }});
ClassificationAssociateRequest classificationAssociateRequest = new ClassificationAssociateRequest(createdGuids, tag);
// tag with table entities, leave rest for comparison
ClassificationAssociateRequest classificationAssociateRequest = new ClassificationAssociateRequest(createdGuids.get(TABLE_TYPE), tag);
entityREST.addClassification(classificationAssociateRequest);
for (String guid : createdGuids) {
final AtlasClassification result_tag = entityREST.getClassification(guid, TestUtilsV2.CLASSIFICATION);
for (int i = 0; i < createdGuids.get(TABLE_TYPE).size() - 1; i++) {
final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(TABLE_TYPE).get(i), TestUtilsV2.CLASSIFICATION);
Assert.assertNotNull(result_tag);
Assert.assertEquals(result_tag, tag);
}
}
@Test
@Test(dependsOnMethods = "testTagToMultipleEntities")
public void testBasicSearchWithSub() throws Exception {
// search entities with classification named classification
searchParameters = new SearchParameters();
searchParameters.setIncludeSubClassifications(true);
searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
}
@Test(dependsOnMethods = "testTagToMultipleEntities")
public void testWildCardBasicSearch() throws Exception {
searchParameters = new SearchParameters();
searchParameters.setClassification("*");
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
searchParameters.setClassification("_CLASSIFIED");
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
// Test wildcard usage of basic search
searchParameters.setClassification("cl*");
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
searchParameters.setClassification("*ion");
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
searchParameters.setClassification("*l*");
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
searchParameters.setClassification("_NOT_CLASSIFIED");
searchParameters.setTypeName(DATABASE_TYPE);
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
}
@Test(dependsOnMethods = "testWildCardBasicSearch")
public void testBasicSearchAddCls() throws Exception {
AtlasClassification cls = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
put("stringAttr", "sample_string");
put("booleanAttr", true);
put("integerAttr", 100);
}});
ClassificationAssociateRequest clsAssRequest = new ClassificationAssociateRequest(createdGuids.get(DATABASE_TYPE), cls);
entityREST.addClassification(clsAssRequest);
final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(DATABASE_TYPE).get(0), TestUtilsV2.PHI);
Assert.assertNotNull(result_tag);
// search entities associated with phi
searchParameters = new SearchParameters();
searchParameters.setClassification(TestUtilsV2.PHI);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
}
private void addPHICls() throws Exception {
AtlasClassification clsPHI = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
put("stringAttr", "string");
put("booleanAttr", true);
put("integerAttr", 100);
}});
// add clsPHI to col entities
ClassificationAssociateRequest clsAssRequest = new ClassificationAssociateRequest(createdGuids.get(COLUMN_TYPE), clsPHI);
entityREST.addClassification(clsAssRequest);
final AtlasClassification result_PHI = entityREST.getClassification(createdGuids.get(COLUMN_TYPE).get(0), TestUtilsV2.PHI);
Assert.assertNotNull(result_PHI);
}
@Test(dependsOnMethods = "testBasicSearchAddCls")
public void testBasicSearch() throws Exception{
searchParameters = new SearchParameters();
searchParameters.setClassification("PH*");
searchParameters.setTypeName(DATABASE_TYPE);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
addPHICls();
// basic search with tag filterCriteria
searchParameters = new SearchParameters();
searchParameters.setClassification("PHI");
SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria();
filterCriteria.setAttributeName("stringAttr");
filterCriteria.setOperator(SearchParameters.Operator.CONTAINS);
filterCriteria.setAttributeValue("str");
searchParameters.setTagFilters(filterCriteria);
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 3);
filterCriteria.setAttributeName("stringAttr");
filterCriteria.setOperator(SearchParameters.Operator.EQ);
filterCriteria.setAttributeValue("string");
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
}
@Test(dependsOnMethods = "testWildCardBasicSearch")
public void testBasicSearchWithSubTypes() throws Exception{
AtlasClassification fetlCls = new AtlasClassification(TestUtilsV2.FETL_CLASSIFICATION, new HashMap<String, Object>() {{
put("tag", "sample_tag");
}});
ClassificationAssociateRequest clsAssRequest = new ClassificationAssociateRequest(createdGuids.get(DATABASE_TYPE), fetlCls);
entityREST.addClassification(clsAssRequest);
final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(DATABASE_TYPE).get(0), TestUtilsV2.PHI);
Assert.assertNotNull(result_tag);
// basic search with subtypes
searchParameters = new SearchParameters();
searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
searchParameters.setIncludeSubTypes(true);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 3);
}
@Test(dependsOnMethods = "testBasicSearchWithSubTypes")
public void testUpdateWithSerializedEntities() throws Exception {
//Check with serialization and deserialization of entity attributes for the case
// where attributes which are de-serialized into a map
AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
......@@ -165,11 +337,11 @@ public class TestEntitiesREST {
@Test(dependsOnMethods = "testCreateOrUpdateEntities")
public void testGetEntities() throws Exception {
final AtlasEntitiesWithExtInfo response = entityREST.getByGuids(createdGuids, false, false);
final AtlasEntitiesWithExtInfo response = entityREST.getByGuids(createdGuids.get(DATABASE_TYPE), false, false);
final List<AtlasEntity> entities = response.getEntities();
Assert.assertNotNull(entities);
Assert.assertEquals(entities.size(), 3);
Assert.assertEquals(entities.size(), 1);
verifyAttributes(entities);
}
......@@ -192,11 +364,11 @@ public class TestEntitiesREST {
AtlasEntity retrievedTableEntity = null;
AtlasEntity retrievedColumnEntity = null;
for (AtlasEntity entity: retrievedEntities ) {
if ( entity.getTypeName().equals(TestUtilsV2.DATABASE_TYPE)) {
if ( entity.getTypeName().equals(DATABASE_TYPE)) {
retrievedDBEntity = entity;
}
if ( entity.getTypeName().equals(TestUtilsV2.TABLE_TYPE)) {
if ( entity.getTypeName().equals(TABLE_TYPE)) {
retrievedTableEntity = entity;
}
......@@ -261,4 +433,4 @@ public class TestEntitiesREST {
return ret;
}
}
}
\ No newline at end of file
webapp/src/test/resources/json/search-parameters/tag-filters.json
......@@ -12,22 +12,5 @@
"classification":"fooTag"
},
"expectedCount": 1
},
{
"testDescription": "Search for exact Tag name order by",
"searchParameters": {
"entityFilters":null,
"tagFilters":null,
"attributes":null,
"query":null,
"excludeDeletedEntities":true,
"limit":25,
"typeName":null,
"sortBy": "name",
"sortOrder": "DESCENDING",
"classification":"fooTag"
},
"expectedCount": 1
}
]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment