Commit 9a067d38 by Pinal Shah Committed by nixonrodrigues

ATLAS-3782 : Support NOT_CONTAINS operator in basic search

parent ec314fde
......@@ -195,11 +195,15 @@ public class AtlasSolrQueryBuilder {
if (!indexAttributes.contains(indexAttributeName)) {
StringBuilder sb = new StringBuilder();
if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works
if (operator.equals(Operator.CONTAINS)) {
operator = Operator.EQ;
} else if (operator.equals(Operator.NOT_CONTAINS)) {
operator = Operator.NEQ;
}
attributeValue = getIndexQueryAttributeValue(attributeValue);
}
......@@ -261,6 +265,9 @@ public class AtlasSolrQueryBuilder {
case CONTAINS:
withContains(queryBuilder, indexFieldName, attributeValue);
break;
case NOT_CONTAINS:
withNotContains(queryBuilder, indexFieldName, attributeValue);
break;
case IS_NULL:
withIsNull(queryBuilder, indexFieldName);
break;
......@@ -388,6 +395,10 @@ public class AtlasSolrQueryBuilder {
queryBuilder.append("+").append(indexFieldName).append(":*").append(attributeValue).append("* ");
}
private void withNotContains(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
queryBuilder.append("*:* -").append(indexFieldName).append(":*").append(attributeValue).append("* ");
}
private void withIsNull(StringBuilder queryBuilder, String indexFieldName) {
queryBuilder.append("-").append(indexFieldName).append(":*").append(" ");
}
......
......@@ -18,13 +18,12 @@
package org.apache.atlas.discovery;
import org.apache.atlas.SortOrder;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
......@@ -35,8 +34,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.util.*;
/**
......@@ -52,10 +49,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
private final AtlasIndexQuery indexQuery;
private final AtlasIndexQuery classificationIndexQuery;
private final AtlasGraphQuery tagGraphQueryWithAttributes;
private final Map<String, Object> gremlinQueryBindings;
private final String gremlinTagFilterQuery;
private final Predicate traitPredicate;
private final Predicate isEntityPredicate;
private Predicate activePredicate;
// Some index engines may take space as a delimiter, when basic search is
// executed, unsatisfying results may be returned.
......@@ -107,6 +103,14 @@ public class ClassificationSearchProcessor extends SearchProcessor {
traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
if (context.getSearchParameters().getExcludeDeletedEntities()) {
activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name(), String.class);
}
Predicate attributePredicate = null;
Predicate typeNamePredicate = null;
AtlasGraph graph = context.getGraph();
// index query directly on entity
......@@ -156,19 +160,12 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
Predicate typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
if (typeNamePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
}
Predicate attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
if (attributePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
typeNamePredicate = isClassificationRootType() ? null :
SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
} else {
classificationIndexQuery = null;
}
......@@ -176,7 +173,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
// only registered classification will search with tag filters
if (useGraphSearchForClassification) {
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
AtlasGraphQuery query = graph.query();
if (!isClassificationRootType()) {
......@@ -184,31 +180,20 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
gremlinQueryBindings = new HashMap<>();
StringBuilder gremlinQuery = new StringBuilder();
gremlinQuery.append("g.V().has('__guid', within(guids))");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER));
gremlinQuery.append(".as('e').filter(out()");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
// constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
// After filtering on tags go back to e and output the list of entity vertices
gremlinQuery.append(").toList()");
typeNamePredicate = isClassificationRootType() ? null :
SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, allAttributes);
gremlinQueryBindings.put("traitNames", typeAndSubTypes);
gremlinQueryBindings.put("typeNames", typeAndSubTypes); // classification typeName
gremlinTagFilterQuery = gremlinQuery.toString();
if (LOG.isDebugEnabled()) {
LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
}
} else {
tagGraphQueryWithAttributes = null;
gremlinTagFilterQuery = null;
gremlinQueryBindings = null;
}
if (typeNamePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
}
if (attributePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
}
......@@ -274,23 +259,23 @@ public class ClassificationSearchProcessor extends SearchProcessor {
CollectionUtils.filter(entityVertices, isEntityPredicate);
} else {
if (tagGraphQueryWithAttributes != null) {
if (classificationIndexQuery != null) {
Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
getVertices(queryResult, classificationVertices);
getVerticesFromIndexQueryResult(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
} else if (classificationIndexQuery != null){
CollectionUtils.filter(classificationVertices, inMemoryPredicate);
} else if (tagGraphQueryWithAttributes != null) {
Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
getVerticesFromIndexQueryResult(queryResult, classificationVertices);
getVertices(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
// Do in-memory filtering before the graph query
CollectionUtils.filter(classificationVertices, inMemoryPredicate);
}
}
......@@ -322,6 +307,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
// Do in-memory filtering
CollectionUtils.filter(entityVertices, isEntityPredicate);
if (activePredicate != null) {
CollectionUtils.filter(entityVertices, activePredicate);
}
super.filter(entityVertices);
......@@ -347,30 +335,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size());
}
//in case of classification type + graph attributes
if (gremlinTagFilterQuery != null && gremlinQueryBindings != null) {
// Now filter on the tag attributes
Set<String> guids = getGuids(entityVertices);
// Clear prior results
entityVertices.clear();
if (CollectionUtils.isNotEmpty(guids)) {
gremlinQueryBindings.put("guids", guids);
try {
AtlasGraph graph = context.getGraph();
ScriptEngine gremlinScriptEngine = graph.getGremlinScriptEngine();
List<AtlasVertex> atlasVertices = (List<AtlasVertex>) graph.executeGremlinScript(gremlinScriptEngine, gremlinQueryBindings, gremlinTagFilterQuery, false);
if (CollectionUtils.isNotEmpty(atlasVertices)) {
entityVertices.addAll(atlasVertices);
}
} catch (AtlasBaseException | ScriptException e) {
LOG.warn(e.getMessage(), e);
}
}
} else if (inMemoryPredicate != null) {
if (inMemoryPredicate != null) {
//in case of classification type + index attributes
CollectionUtils.filter(entityVertices, traitPredicate);
......
......@@ -464,9 +464,9 @@ public abstract class SearchProcessor {
AtlasType attributeType = structType.getAttributeType(filterCriteria.getAttributeName());
if (AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeType.getTypeName())) {
if (filterCriteria.getOperator() == SearchParameters.Operator.NEQ) {
if (filterCriteria.getOperator() == SearchParameters.Operator.NEQ || filterCriteria.getOperator() == SearchParameters.Operator.NOT_CONTAINS) {
if (LOG.isDebugEnabled()) {
LOG.debug("NEQ operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", qualifiedName);
LOG.debug("{} operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", filterCriteria.getOperator(), qualifiedName);
}
ret = false;
......@@ -633,6 +633,7 @@ public abstract class SearchProcessor {
op = SearchParameters.Operator.NOT_CONTAINS;
break;
case CONTAINS:
case NOT_CONTAINS:
if (attrName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
attrVal = getCustomAttributeIndexQueryValue(attrVal, true);
}
......@@ -849,6 +850,8 @@ public abstract class SearchProcessor {
case NOT_NULL:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
break;
case NOT_CONTAINS:
break;
default:
LOG.warn("{}: unsupported operator. Ignored", operator);
break;
......
......@@ -19,10 +19,12 @@ package org.apache.atlas;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.glossary.GlossaryService;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.glossary.AtlasGlossary;
import org.apache.atlas.model.glossary.AtlasGlossaryTerm;
import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader;
import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.typedef.*;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
......@@ -42,6 +44,7 @@ import static org.testng.Assert.fail;
public abstract class BasicTestSetup {
// Entity type //
protected static final String DATABASE_TYPE = "hive_db";
protected static final String HIVE_TABLE_TYPE = "hive_table";
private static final String COLUMN_TYPE = "hive_column";
......@@ -50,6 +53,7 @@ public abstract class BasicTestSetup {
private static final String VIEW_TYPE = "hive_process";
protected static final String DATASET_SUBTYPE = "Asset";
//Classification type //
public static final String DIMENSION_CLASSIFICATION = "Dimension";
public static final String FACT_CLASSIFICATION = "Fact";
public static final String PII_CLASSIFICATION = "PII";
......@@ -59,14 +63,21 @@ public abstract class BasicTestSetup {
public static final String LOGDATA_CLASSIFICATION = "Log Data";
public static final String DIMENSIONAL_CLASSIFICATION = "Dimensional";
// Glossary type //
public static final String SALES_GLOSSARY = "salesGlossary";
public static final String SALES_TERM = "salesTerm";
@Inject
protected AtlasTypeRegistry typeRegistry;
@Inject
protected AtlasTypeDefStore typeDefStore;
@Inject
protected AtlasEntityStore entityStore;
@Inject
protected GlossaryService glossaryService;
private boolean baseLoaded = false;
private EntityMutationResponse hiveEntities;
protected void setupTestData() {
loadBaseModels();
......@@ -77,6 +88,7 @@ public abstract class BasicTestSetup {
private void loadBaseModels() {
try {
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("0000-Area0/0011-glossary_model.json", typeDefStore, typeRegistry);
baseLoaded = true;
} catch (IOException | AtlasBaseException e) {
fail("Base model setup is required for test to run!");
......@@ -97,7 +109,7 @@ public abstract class BasicTestSetup {
AtlasEntity.AtlasEntitiesWithExtInfo hiveTestEntities = hiveTestEntities();
try {
entityStore.createOrUpdate(new AtlasEntityStream(hiveTestEntities), false);
hiveEntities = entityStore.createOrUpdate(new AtlasEntityStream(hiveTestEntities), false);
} catch (AtlasBaseException e) {
fail("Hive entities need to be created for test to run!");
}
......@@ -450,12 +462,13 @@ public abstract class BasicTestSetup {
return datasetSubType;
}
public void createDummyEntity(String name, String type, String... traitNames) throws AtlasBaseException {
public EntityMutationResponse createDummyEntity(String name, String type, String... traitNames) throws AtlasBaseException {
AtlasEntity entity = new AtlasEntity(type);
entity.setAttribute("name", name);
entity.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
entity.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList()));
entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false);
EntityMutationResponse resp = entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false);
return resp;
}
public SearchParameters.FilterCriteria getSingleFilterCondition(String attName, SearchParameters.Operator op, String attrValue) {
......@@ -472,4 +485,22 @@ public abstract class BasicTestSetup {
return filterCriteria;
}
public void assignGlossary() throws AtlasBaseException {
AtlasGlossary glossary = new AtlasGlossary();
glossary.setName(SALES_GLOSSARY);
glossary = glossaryService.createGlossary(glossary);
AtlasGlossaryTerm term = new AtlasGlossaryTerm();
term.setAnchor(new AtlasGlossaryHeader(glossary.getGuid()));
term.setName(SALES_TERM);
term = glossaryService.createTerm(term);
List<AtlasRelatedObjectId> guids = hiveEntities.getCreatedEntities().stream().filter(e -> e.getTypeName().equals(HIVE_TABLE_TYPE))
.map(p -> {AtlasRelatedObjectId obj = new AtlasRelatedObjectId();
obj.setGuid(p.getGuid());
obj.setTypeName(p.getTypeName()); return obj;}).collect(Collectors.toList());
glossaryService.assignTermToEntities(term.getGuid(), guids);
}
}
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.BasicTestSetup;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.commons.collections.CollectionUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import static org.apache.atlas.model.discovery.SearchParameters.*;
import static org.testng.Assert.assertEquals;
@Guice(modules = TestModules.TestOnlyModule.class)
public class AtlasDiscoveryServiceTest extends BasicTestSetup {
@Inject
private AtlasDiscoveryService discoveryService;
@BeforeClass
public void setup() throws AtlasException, AtlasBaseException {
ApplicationProperties.get().setProperty(ApplicationProperties.ENABLE_FREETEXT_SEARCH_CONF, true);
setupTestData();
createDimensionalTaggedEntity("sales");
assignGlossary();
}
/* TermSearchProcessor(TSP),
FreeTextSearchProcessor(FSP),
ClassificationSearchProcessor(CSP),
EntitySearchProcessor(ESP) */
@Test
public void term() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 10);
}
// TSP execute and CSP,ESP filter
@Test
public void term_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
params.setClassification(METRIC_CLASSIFICATION);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
for(AtlasEntityHeader e : entityHeaders){
System.out.println(e.toString());
}
assertEquals(entityHeaders.size(), 4);
}
@Test
public void term_entity() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
params.setTypeName(HIVE_TABLE_TYPE);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 10);
}
@Test
public void term_entity_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
params.setTypeName(HIVE_TABLE_TYPE);
params.setClassification(DIMENSIONAL_CLASSIFICATION);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isEmpty(entityHeaders));
}
//FSP execute and CSP,ESP filter
@Test
public void query_ALLTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(ALL_CLASSIFICATION_TYPES);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 5);
}
@Test
public void query_ALLTag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(ALL_CLASSIFICATION_TYPES);
//typeName will check for only classification name not propogated classification
SearchParameters.FilterCriteria fc = getSingleFilterCondition("__typeName", Operator.NOT_CONTAINS, METRIC_CLASSIFICATION);
params.setTagFilters(fc);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 4);
}
@Test
public void query_NOTCLASSIFIEDTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(NO_CLASSIFICATIONS);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 1);
}
@Test
public void query_ALLWildcardTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification("*");
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 5);
}
@Test
public void query_wildcardTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification("Dimen*on");
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 2);
}
@Test
public void query_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(METRIC_CLASSIFICATION);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 3);
}
@Test
public void query_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(METRIC_CLASSIFICATION);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(fc);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 3);
}
@Test
public void query_entity() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 4);
}
@Test
public void query_entity_entityFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("tableType", Operator.NOT_NULL, "null");
params.setEntityFilters(fc);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 3);
}
@Test
public void query_entity_entityFilter_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("tableType", Operator.IS_NULL, "null");
params.setEntityFilters(fc);
params.setClassification(DIMENSIONAL_CLASSIFICATION);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 1);
}
@Test
public void query_entity_entityFilter_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria fcE = getSingleFilterCondition("tableType", Operator.IS_NULL, "null");
params.setEntityFilters(fcE);
params.setClassification(DIMENSIONAL_CLASSIFICATION);
params.setQuery("sales");
SearchParameters.FilterCriteria fcC = getSingleFilterCondition("attr1", Operator.EQ, "value1");
params.setTagFilters(fcC);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 1);
}
@Test
public void query_entity_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setClassification(METRIC_CLASSIFICATION);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(fc);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 2);
}
@Test
public void query_entity_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setClassification(METRIC_CLASSIFICATION);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 2);
}
// CSP Execute and ESP filter
@Test
public void entity_entityFilter_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria fcE = getSingleFilterCondition("tableType", Operator.EQ, "Managed");
params.setEntityFilters(fcE);
params.setClassification(METRIC_CLASSIFICATION);
SearchParameters.FilterCriteria fcC = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(fcC);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 4);
}
@Test
public void entity_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setClassification(METRIC_CLASSIFICATION);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(fc);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 4);
}
private void createDimensionalTaggedEntity(String name) throws AtlasBaseException {
EntityMutationResponse resp = createDummyEntity(name, HIVE_TABLE_TYPE);
AtlasEntityHeader entityHeader = resp.getCreatedEntities().get(0);
String guid = entityHeader.getGuid();
HashMap<String,Object> attr = new HashMap<>();
attr.put("attr1","value1");
entityStore.addClassification(Arrays.asList(guid), new AtlasClassification(DIMENSIONAL_CLASSIFICATION, attr));
}
@AfterClass
public void teardown() {
AtlasGraphProvider.cleanup();
}
}
......@@ -26,9 +26,15 @@ import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.commons.collections.CollectionUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -37,19 +43,23 @@ import javax.inject.Inject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;
import static org.apache.atlas.model.discovery.SearchParameters.*;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;
@Guice(modules = TestModules.TestOnlyModule.class)
public class BasicSearchClassificationTest extends BasicTestSetup {
public class ClassificationSearchProcessorTest extends BasicTestSetup {
@Inject
private AtlasDiscoveryService discoveryService;
private AtlasGraph graph;
@Inject
public GraphBackedSearchIndexer indexer;
@Inject
private EntityGraphRetriever entityRetriever;
private int totalEntities = 0;
private int totalClassifiedEntities = 0;
private int getTotalClassifiedEntitiesHistorical = 0;
private int dimensionTagEntities = 10;
private String dimensionTagDeleteGuid;
private String dimensionalTagGuid;
......@@ -65,14 +75,14 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
public void searchByALLTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(ALL_CLASSIFICATION_TYPES);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
totalEntities = getEntityCount();
totalClassifiedEntities = entityHeaders.size();
getTotalClassifiedEntitiesHistorical = getEntityWithTagCountHistorical();
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
totalClassifiedEntities = vertices.size();
}
@Test
......@@ -81,10 +91,14 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setClassification(ALL_CLASSIFICATION_TYPES);
FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(filterCriteria);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), totalClassifiedEntities);
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), totalClassifiedEntities);
}
@Test
......@@ -95,29 +109,40 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setTagFilters(filterCriteria);
params.setLimit(totalClassifiedEntities - 2);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), totalClassifiedEntities - 2);
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), totalClassifiedEntities - 2);
}
@Test
//@Test
public void searchByNOTCLASSIFIED() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(NO_CLASSIFICATIONS);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), totalEntities - totalClassifiedEntities);
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), 20);
}
@Test
public void searchByTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(DIMENSION_CLASSIFICATION);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), dimensionTagEntities);
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), dimensionTagEntities);
}
@Test
......@@ -126,11 +151,23 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setClassification(DIMENSIONAL_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.EQ, "Test");
params.setTagFilters(filterCriteria);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
assertEquals(entityHeaders.size(), 1);
assertEquals(entityHeaders.get(0).getGuid(), dimensionalTagGuid);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), 1);
List<String> guids = vertices.stream().map(g -> {
try {
return entityRetriever.toAtlasEntityHeader(g).getGuid();
} catch (AtlasBaseException e) {
fail("Failure in mapping vertex to AtlasEntityHeader");
}
return "";
}).collect(Collectors.toList());
Assert.assertTrue(guids.contains(dimensionalTagGuid));
}
......@@ -141,35 +178,86 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setClassification(DIMENSION_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(filterCriteria);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), dimensionTagEntities);
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), dimensionTagEntities);
}
@Test
public void searchByWildcardTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification("Dimension*");
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), dimensionTagEntities + 1);
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), dimensionTagEntities + 1);
}
//@Test
@Test
public void searchByALLWildcardTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification("*");
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(),20);
}
@Test
public void searchWithNotContains() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(DIMENSIONAL_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.NOT_CONTAINS, "Test");
params.setTagFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
Assert.assertTrue(CollectionUtils.isEmpty(vertices));
}
@Test
public void searchByTagAndGraphSysFilters() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(DIMENSION_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("__entityStatus", Operator.EQ, "DELETED");
params.setTagFilters(filterCriteria);
params.setExcludeDeletedEntities(false);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
assertEquals(entityHeaders.size(), 1);
assertEquals(entityHeaders.get(0).getGuid(), dimensionTagDeleteGuid);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), 1);
List<String> guids = vertices.stream().map(g -> {
try {
return entityRetriever.toAtlasEntityHeader(g).getGuid();
} catch (AtlasBaseException e) {
fail("Failure in mapping vertex to AtlasEntityHeader");
}
return "";
}).collect(Collectors.toList());
Assert.assertTrue(guids.contains(dimensionTagDeleteGuid));
}
......@@ -209,21 +297,8 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
}
private int getEntityCount() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(ALL_ENTITY_TYPES);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
return entityHeaders.size();
@AfterClass
public void teardown() {
AtlasGraphProvider.cleanup();
}
private int getEntityWithTagCountHistorical() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(ALL_CLASSIFICATION_TYPES);
params.setExcludeDeletedEntities(false);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
return entityHeaders.size();
}
}
......@@ -23,11 +23,13 @@ import org.apache.atlas.SortOrder;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -60,6 +62,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
@Inject
public GraphBackedSearchIndexer indexer;
private String expectedEntityName = "hive_Table_Null_tableType";
@Test
public void searchTablesByClassification() throws AtlasBaseException {
......@@ -132,7 +135,6 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
@Test(priority = -1)
public void searchWithNEQ_stringAttr() throws AtlasBaseException {
String expectedEntityName = "hive_Table_Null_tableType";
createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
......@@ -154,7 +156,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertTrue(nameList.contains(expectedEntityName));
}
@Test(dependsOnMethods = "searchWithNEQ_stringAttr")
@Test
public void searchWithNEQ_pipeSeperatedAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
......@@ -173,10 +175,10 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
}
assertTrue(nameList.contains("hive_Table_Null_tableType"));
assertTrue(nameList.contains(expectedEntityName));
}
@Test(dependsOnMethods = "searchWithNEQ_stringAttr")
@Test
public void searchWithNEQ_doubleAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
......@@ -309,4 +311,53 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertEquals(processor.execute().size(), 2);
}
@Test
public void searchWithNotContains_stringAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.NOT_CONTAINS, "Managed");
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(vertices.size(), 3);
List<String> nameList = new ArrayList<>();
for (AtlasVertex vertex : vertices) {
nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
}
assertTrue(nameList.contains(expectedEntityName));
}
@Test
public void searchWithNotContains_pipeSeperatedAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("__classificationNames", SearchParameters.Operator.NOT_CONTAINS, METRIC_CLASSIFICATION);
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(vertices.size(), 7);
List<String> nameList = new ArrayList<>();
for (AtlasVertex vertex : vertices) {
nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
}
assertTrue(nameList.contains(expectedEntityName));
}
@AfterClass
public void teardown() {
AtlasGraphProvider.cleanup();
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment