Skip to content

A
atlas
Commit 9a067d38 by Pinal Shah Committed by nixonrodrigues
Options

ATLAS-3782 : Support NOT_CONTAINS operator in basic search

Signed-off-by: 's avatarnixonrodrigues <nixon@apache.org>
parent ec314fde
Showing with 608 additions and 133 deletions
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
...@@ -195,11 +195,15 @@ public class AtlasSolrQueryBuilder { ...@@ -195,11 +195,15 @@ public class AtlasSolrQueryBuilder {
if (!indexAttributes.contains(indexAttributeName)) { if (!indexAttributes.contains(indexAttributeName)) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) { if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair, // CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2 // for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works // surrounding inverted commas to attributeValue works
operator = Operator.EQ; if (operator.equals(Operator.CONTAINS)) {
operator = Operator.EQ;
} else if (operator.equals(Operator.NOT_CONTAINS)) {
operator = Operator.NEQ;
}
attributeValue = getIndexQueryAttributeValue(attributeValue); attributeValue = getIndexQueryAttributeValue(attributeValue);
} }
...@@ -261,6 +265,9 @@ public class AtlasSolrQueryBuilder { ...@@ -261,6 +265,9 @@ public class AtlasSolrQueryBuilder {
case CONTAINS: case CONTAINS:
withContains(queryBuilder, indexFieldName, attributeValue); withContains(queryBuilder, indexFieldName, attributeValue);
break; break;
case NOT_CONTAINS:
withNotContains(queryBuilder, indexFieldName, attributeValue);
break;
case IS_NULL: case IS_NULL:
withIsNull(queryBuilder, indexFieldName); withIsNull(queryBuilder, indexFieldName);
break; break;
...@@ -388,6 +395,10 @@ public class AtlasSolrQueryBuilder { ...@@ -388,6 +395,10 @@ public class AtlasSolrQueryBuilder {
queryBuilder.append("+").append(indexFieldName).append(":*").append(attributeValue).append("* "); queryBuilder.append("+").append(indexFieldName).append(":*").append(attributeValue).append("* ");
} }
private void withNotContains(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
queryBuilder.append("*:* -").append(indexFieldName).append(":*").append(attributeValue).append("* ");
}
private void withIsNull(StringBuilder queryBuilder, String indexFieldName) { private void withIsNull(StringBuilder queryBuilder, String indexFieldName) {
queryBuilder.append("-").append(indexFieldName).append(":*").append(" "); queryBuilder.append("-").append(indexFieldName).append(":*").append(" ");
} }
......
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
...@@ -18,13 +18,12 @@ ...@@ -18,13 +18,12 @@
package org.apache.atlas.discovery; package org.apache.atlas.discovery;
import org.apache.atlas.SortOrder; import org.apache.atlas.SortOrder;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria; import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasClassificationType; import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.SearchPredicateUtil; import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
...@@ -35,8 +34,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order; ...@@ -35,8 +34,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.util.*; import java.util.*;
/** /**
...@@ -52,10 +49,9 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -52,10 +49,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
private final AtlasIndexQuery indexQuery; private final AtlasIndexQuery indexQuery;
private final AtlasIndexQuery classificationIndexQuery; private final AtlasIndexQuery classificationIndexQuery;
private final AtlasGraphQuery tagGraphQueryWithAttributes; private final AtlasGraphQuery tagGraphQueryWithAttributes;
private final Map<String, Object> gremlinQueryBindings;
private final String gremlinTagFilterQuery;
private final Predicate traitPredicate; private final Predicate traitPredicate;
private final Predicate isEntityPredicate; private final Predicate isEntityPredicate;
private Predicate activePredicate;
// Some index engines may take space as a delimiter, when basic search is // Some index engines may take space as a delimiter, when basic search is
// executed, unsatisfying results may be returned. // executed, unsatisfying results may be returned.
...@@ -107,6 +103,14 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -107,6 +103,14 @@ public class ClassificationSearchProcessor extends SearchProcessor {
traitPredicate = buildTraitPredict(classificationTypes); traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry()); isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
if (context.getSearchParameters().getExcludeDeletedEntities()) {
activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name(), String.class);
}
Predicate attributePredicate = null;
Predicate typeNamePredicate = null;
AtlasGraph graph = context.getGraph(); AtlasGraph graph = context.getGraph();
// index query directly on entity // index query directly on entity
...@@ -156,19 +160,12 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -156,19 +160,12 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")"); indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll(""); indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
Predicate typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class); this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
if (typeNamePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
}
Predicate attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) { typeNamePredicate = isClassificationRootType() ? null :
inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate); SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
} attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
} else { } else {
classificationIndexQuery = null; classificationIndexQuery = null;
} }
...@@ -176,7 +173,6 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -176,7 +173,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
// only registered classification will search with tag filters // only registered classification will search with tag filters
if (useGraphSearchForClassification) { if (useGraphSearchForClassification) {
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
AtlasGraphQuery query = graph.query(); AtlasGraphQuery query = graph.query();
if (!isClassificationRootType()) { if (!isClassificationRootType()) {
...@@ -184,31 +180,20 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -184,31 +180,20 @@ public class ClassificationSearchProcessor extends SearchProcessor {
} }
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query); tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
gremlinQueryBindings = new HashMap<>();
StringBuilder gremlinQuery = new StringBuilder();
gremlinQuery.append("g.V().has('__guid', within(guids))"); typeNamePredicate = isClassificationRootType() ? null :
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER)); SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
gremlinQuery.append(".as('e').filter(out()"); attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, allAttributes);
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
// constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
// After filtering on tags go back to e and output the list of entity vertices
gremlinQuery.append(").toList()");
gremlinQueryBindings.put("traitNames", typeAndSubTypes);
gremlinQueryBindings.put("typeNames", typeAndSubTypes); // classification typeName
gremlinTagFilterQuery = gremlinQuery.toString();
if (LOG.isDebugEnabled()) {
LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
}
} else { } else {
tagGraphQueryWithAttributes = null; tagGraphQueryWithAttributes = null;
gremlinTagFilterQuery = null; }
gremlinQueryBindings = null;
if (typeNamePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
}
if (attributePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
} }
} }
...@@ -274,23 +259,23 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -274,23 +259,23 @@ public class ClassificationSearchProcessor extends SearchProcessor {
CollectionUtils.filter(entityVertices, isEntityPredicate); CollectionUtils.filter(entityVertices, isEntityPredicate);
} else { } else {
if (tagGraphQueryWithAttributes != null) { if (classificationIndexQuery != null) {
Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator(); Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
getVertices(queryResult, classificationVertices); getVerticesFromIndexQueryResult(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit; isLastResultPage = classificationVertices.size() < limit;
} else if (classificationIndexQuery != null){ CollectionUtils.filter(classificationVertices, inMemoryPredicate);
} else if (tagGraphQueryWithAttributes != null) {
Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit); Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
getVerticesFromIndexQueryResult(queryResult, classificationVertices); getVertices(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit; isLastResultPage = classificationVertices.size() < limit;
// Do in-memory filtering before the graph query
CollectionUtils.filter(classificationVertices, inMemoryPredicate); CollectionUtils.filter(classificationVertices, inMemoryPredicate);
} }
} }
...@@ -322,6 +307,9 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -322,6 +307,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
} }
// Do in-memory filtering // Do in-memory filtering
CollectionUtils.filter(entityVertices, isEntityPredicate); CollectionUtils.filter(entityVertices, isEntityPredicate);
if (activePredicate != null) {
CollectionUtils.filter(entityVertices, activePredicate);
}
super.filter(entityVertices); super.filter(entityVertices);
...@@ -347,30 +335,8 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -347,30 +335,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size()); LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size());
} }
//in case of classification type + graph attributes
if (gremlinTagFilterQuery != null && gremlinQueryBindings != null) {
// Now filter on the tag attributes
Set<String> guids = getGuids(entityVertices);
// Clear prior results
entityVertices.clear();
if (CollectionUtils.isNotEmpty(guids)) {
gremlinQueryBindings.put("guids", guids);
try { if (inMemoryPredicate != null) {
AtlasGraph graph = context.getGraph();
ScriptEngine gremlinScriptEngine = graph.getGremlinScriptEngine();
List<AtlasVertex> atlasVertices = (List<AtlasVertex>) graph.executeGremlinScript(gremlinScriptEngine, gremlinQueryBindings, gremlinTagFilterQuery, false);
if (CollectionUtils.isNotEmpty(atlasVertices)) {
entityVertices.addAll(atlasVertices);
}
} catch (AtlasBaseException | ScriptException e) {
LOG.warn(e.getMessage(), e);
}
}
} else if (inMemoryPredicate != null) {
//in case of classification type + index attributes //in case of classification type + index attributes
CollectionUtils.filter(entityVertices, traitPredicate); CollectionUtils.filter(entityVertices, traitPredicate);
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
...@@ -464,9 +464,9 @@ public abstract class SearchProcessor { ...@@ -464,9 +464,9 @@ public abstract class SearchProcessor {
AtlasType attributeType = structType.getAttributeType(filterCriteria.getAttributeName()); AtlasType attributeType = structType.getAttributeType(filterCriteria.getAttributeName());
if (AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeType.getTypeName())) { if (AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeType.getTypeName())) {
if (filterCriteria.getOperator() == SearchParameters.Operator.NEQ) { if (filterCriteria.getOperator() == SearchParameters.Operator.NEQ || filterCriteria.getOperator() == SearchParameters.Operator.NOT_CONTAINS) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("NEQ operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", qualifiedName); LOG.debug("{} operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", filterCriteria.getOperator(), qualifiedName);
} }
ret = false; ret = false;
...@@ -633,6 +633,7 @@ public abstract class SearchProcessor { ...@@ -633,6 +633,7 @@ public abstract class SearchProcessor {
op = SearchParameters.Operator.NOT_CONTAINS; op = SearchParameters.Operator.NOT_CONTAINS;
break; break;
case CONTAINS: case CONTAINS:
case NOT_CONTAINS:
if (attrName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) { if (attrName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
attrVal = getCustomAttributeIndexQueryValue(attrVal, true); attrVal = getCustomAttributeIndexQueryValue(attrVal, true);
} }
...@@ -849,6 +850,8 @@ public abstract class SearchProcessor { ...@@ -849,6 +850,8 @@ public abstract class SearchProcessor {
case NOT_NULL: case NOT_NULL:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null); innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
break; break;
case NOT_CONTAINS:
break;
default: default:
LOG.warn("{}: unsupported operator. Ignored", operator); LOG.warn("{}: unsupported operator. Ignored", operator);
break; break;
......
repository/src/test/java/org/apache/atlas/BasicTestSetup.java
...@@ -19,10 +19,12 @@ package org.apache.atlas; ...@@ -19,10 +19,12 @@ package org.apache.atlas;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.glossary.GlossaryService;
import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.glossary.AtlasGlossary;
import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.glossary.AtlasGlossaryTerm;
import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader;
import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.typedef.*; import org.apache.atlas.model.typedef.*;
import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
...@@ -42,6 +44,7 @@ import static org.testng.Assert.fail; ...@@ -42,6 +44,7 @@ import static org.testng.Assert.fail;
public abstract class BasicTestSetup { public abstract class BasicTestSetup {
// Entity type //
protected static final String DATABASE_TYPE = "hive_db"; protected static final String DATABASE_TYPE = "hive_db";
protected static final String HIVE_TABLE_TYPE = "hive_table"; protected static final String HIVE_TABLE_TYPE = "hive_table";
private static final String COLUMN_TYPE = "hive_column"; private static final String COLUMN_TYPE = "hive_column";
...@@ -50,6 +53,7 @@ public abstract class BasicTestSetup { ...@@ -50,6 +53,7 @@ public abstract class BasicTestSetup {
private static final String VIEW_TYPE = "hive_process"; private static final String VIEW_TYPE = "hive_process";
protected static final String DATASET_SUBTYPE = "Asset"; protected static final String DATASET_SUBTYPE = "Asset";
//Classification type //
public static final String DIMENSION_CLASSIFICATION = "Dimension"; public static final String DIMENSION_CLASSIFICATION = "Dimension";
public static final String FACT_CLASSIFICATION = "Fact"; public static final String FACT_CLASSIFICATION = "Fact";
public static final String PII_CLASSIFICATION = "PII"; public static final String PII_CLASSIFICATION = "PII";
...@@ -59,14 +63,21 @@ public abstract class BasicTestSetup { ...@@ -59,14 +63,21 @@ public abstract class BasicTestSetup {
public static final String LOGDATA_CLASSIFICATION = "Log Data"; public static final String LOGDATA_CLASSIFICATION = "Log Data";
public static final String DIMENSIONAL_CLASSIFICATION = "Dimensional"; public static final String DIMENSIONAL_CLASSIFICATION = "Dimensional";
// Glossary type //
public static final String SALES_GLOSSARY = "salesGlossary";
public static final String SALES_TERM = "salesTerm";
@Inject @Inject
protected AtlasTypeRegistry typeRegistry; protected AtlasTypeRegistry typeRegistry;
@Inject @Inject
protected AtlasTypeDefStore typeDefStore; protected AtlasTypeDefStore typeDefStore;
@Inject @Inject
protected AtlasEntityStore entityStore; protected AtlasEntityStore entityStore;
@Inject
protected GlossaryService glossaryService;
private boolean baseLoaded = false; private boolean baseLoaded = false;
private EntityMutationResponse hiveEntities;
protected void setupTestData() { protected void setupTestData() {
loadBaseModels(); loadBaseModels();
...@@ -77,6 +88,7 @@ public abstract class BasicTestSetup { ...@@ -77,6 +88,7 @@ public abstract class BasicTestSetup {
private void loadBaseModels() { private void loadBaseModels() {
try { try {
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry); loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("0000-Area0/0011-glossary_model.json", typeDefStore, typeRegistry);
baseLoaded = true; baseLoaded = true;
} catch (IOException | AtlasBaseException e) { } catch (IOException | AtlasBaseException e) {
fail("Base model setup is required for test to run!"); fail("Base model setup is required for test to run!");
...@@ -97,7 +109,7 @@ public abstract class BasicTestSetup { ...@@ -97,7 +109,7 @@ public abstract class BasicTestSetup {
AtlasEntity.AtlasEntitiesWithExtInfo hiveTestEntities = hiveTestEntities(); AtlasEntity.AtlasEntitiesWithExtInfo hiveTestEntities = hiveTestEntities();
try { try {
entityStore.createOrUpdate(new AtlasEntityStream(hiveTestEntities), false); hiveEntities = entityStore.createOrUpdate(new AtlasEntityStream(hiveTestEntities), false);
} catch (AtlasBaseException e) { } catch (AtlasBaseException e) {
fail("Hive entities need to be created for test to run!"); fail("Hive entities need to be created for test to run!");
} }
...@@ -450,12 +462,13 @@ public abstract class BasicTestSetup { ...@@ -450,12 +462,13 @@ public abstract class BasicTestSetup {
return datasetSubType; return datasetSubType;
} }
public void createDummyEntity(String name, String type, String... traitNames) throws AtlasBaseException { public EntityMutationResponse createDummyEntity(String name, String type, String... traitNames) throws AtlasBaseException {
AtlasEntity entity = new AtlasEntity(type); AtlasEntity entity = new AtlasEntity(type);
entity.setAttribute("name", name); entity.setAttribute("name", name);
entity.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); entity.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
entity.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList())); entity.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList()));
entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false); EntityMutationResponse resp = entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false);
return resp;
} }
public SearchParameters.FilterCriteria getSingleFilterCondition(String attName, SearchParameters.Operator op, String attrValue) { public SearchParameters.FilterCriteria getSingleFilterCondition(String attName, SearchParameters.Operator op, String attrValue) {
...@@ -472,4 +485,22 @@ public abstract class BasicTestSetup { ...@@ -472,4 +485,22 @@ public abstract class BasicTestSetup {
return filterCriteria; return filterCriteria;
} }
public void assignGlossary() throws AtlasBaseException {
AtlasGlossary glossary = new AtlasGlossary();
glossary.setName(SALES_GLOSSARY);
glossary = glossaryService.createGlossary(glossary);
AtlasGlossaryTerm term = new AtlasGlossaryTerm();
term.setAnchor(new AtlasGlossaryHeader(glossary.getGuid()));
term.setName(SALES_TERM);
term = glossaryService.createTerm(term);
List<AtlasRelatedObjectId> guids = hiveEntities.getCreatedEntities().stream().filter(e -> e.getTypeName().equals(HIVE_TABLE_TYPE))
.map(p -> {AtlasRelatedObjectId obj = new AtlasRelatedObjectId();
obj.setGuid(p.getGuid());
obj.setTypeName(p.getTypeName()); return obj;}).collect(Collectors.toList());
glossaryService.assignTermToEntities(term.getGuid(), guids);
}
} }
repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.BasicTestSetup;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.commons.collections.CollectionUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import static org.apache.atlas.model.discovery.SearchParameters.*;
import static org.testng.Assert.assertEquals;
@Guice(modules = TestModules.TestOnlyModule.class)
public class AtlasDiscoveryServiceTest extends BasicTestSetup {
@Inject
private AtlasDiscoveryService discoveryService;
@BeforeClass
public void setup() throws AtlasException, AtlasBaseException {
ApplicationProperties.get().setProperty(ApplicationProperties.ENABLE_FREETEXT_SEARCH_CONF, true);
setupTestData();
createDimensionalTaggedEntity("sales");
assignGlossary();
}
/* TermSearchProcessor(TSP),
FreeTextSearchProcessor(FSP),
ClassificationSearchProcessor(CSP),
EntitySearchProcessor(ESP) */
@Test
public void term() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 10);
}
// TSP execute and CSP,ESP filter
@Test
public void term_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
params.setClassification(METRIC_CLASSIFICATION);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
for(AtlasEntityHeader e : entityHeaders){
System.out.println(e.toString());
}
assertEquals(entityHeaders.size(), 4);
}
@Test
public void term_entity() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
params.setTypeName(HIVE_TABLE_TYPE);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 10);
}
@Test
public void term_entity_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
params.setTypeName(HIVE_TABLE_TYPE);
params.setClassification(DIMENSIONAL_CLASSIFICATION);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isEmpty(entityHeaders));
}
//FSP execute and CSP,ESP filter
@Test
public void query_ALLTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(ALL_CLASSIFICATION_TYPES);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 5);
}
@Test
public void query_ALLTag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(ALL_CLASSIFICATION_TYPES);
//typeName will check for only classification name not propogated classification
SearchParameters.FilterCriteria fc = getSingleFilterCondition("__typeName", Operator.NOT_CONTAINS, METRIC_CLASSIFICATION);
params.setTagFilters(fc);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 4);
}
@Test
public void query_NOTCLASSIFIEDTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(NO_CLASSIFICATIONS);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 1);
}
@Test
public void query_ALLWildcardTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification("*");
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 5);
}
@Test
public void query_wildcardTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification("Dimen*on");
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 2);
}
@Test
public void query_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(METRIC_CLASSIFICATION);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 3);
}
@Test
public void query_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(METRIC_CLASSIFICATION);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(fc);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 3);
}
@Test
public void query_entity() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 4);
}
@Test
public void query_entity_entityFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("tableType", Operator.NOT_NULL, "null");
params.setEntityFilters(fc);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 3);
}
@Test
public void query_entity_entityFilter_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("tableType", Operator.IS_NULL, "null");
params.setEntityFilters(fc);
params.setClassification(DIMENSIONAL_CLASSIFICATION);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 1);
}
@Test
public void query_entity_entityFilter_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria fcE = getSingleFilterCondition("tableType", Operator.IS_NULL, "null");
params.setEntityFilters(fcE);
params.setClassification(DIMENSIONAL_CLASSIFICATION);
params.setQuery("sales");
SearchParameters.FilterCriteria fcC = getSingleFilterCondition("attr1", Operator.EQ, "value1");
params.setTagFilters(fcC);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 1);
}
@Test
public void query_entity_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setClassification(METRIC_CLASSIFICATION);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(fc);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 2);
}
@Test
public void query_entity_tag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setClassification(METRIC_CLASSIFICATION);
params.setQuery("sales");
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 2);
}
// CSP Execute and ESP filter
@Test
public void entity_entityFilter_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria fcE = getSingleFilterCondition("tableType", Operator.EQ, "Managed");
params.setEntityFilters(fcE);
params.setClassification(METRIC_CLASSIFICATION);
SearchParameters.FilterCriteria fcC = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(fcC);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 4);
}
@Test
public void entity_tag_tagFilter() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setClassification(METRIC_CLASSIFICATION);
SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(fc);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
assertEquals(entityHeaders.size(), 4);
}
private void createDimensionalTaggedEntity(String name) throws AtlasBaseException {
EntityMutationResponse resp = createDummyEntity(name, HIVE_TABLE_TYPE);
AtlasEntityHeader entityHeader = resp.getCreatedEntities().get(0);
String guid = entityHeader.getGuid();
HashMap<String,Object> attr = new HashMap<>();
attr.put("attr1","value1");
entityStore.addClassification(Arrays.asList(guid), new AtlasClassification(DIMENSIONAL_CLASSIFICATION, attr));
}
@AfterClass
public void teardown() {
AtlasGraphProvider.cleanup();
}
}
repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java → repository/src/test/java/org/apache/atlas/discovery/ClassificationSearchProcessorTest.java
...@@ -26,9 +26,15 @@ import org.apache.atlas.model.instance.AtlasClassification; ...@@ -26,9 +26,15 @@ import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
...@@ -37,19 +43,23 @@ import javax.inject.Inject; ...@@ -37,19 +43,23 @@ import javax.inject.Inject;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
import static org.apache.atlas.model.discovery.SearchParameters.*; import static org.apache.atlas.model.discovery.SearchParameters.*;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;
@Guice(modules = TestModules.TestOnlyModule.class) @Guice(modules = TestModules.TestOnlyModule.class)
public class BasicSearchClassificationTest extends BasicTestSetup { public class ClassificationSearchProcessorTest extends BasicTestSetup {
@Inject @Inject
private AtlasDiscoveryService discoveryService; private AtlasGraph graph;
@Inject
public GraphBackedSearchIndexer indexer;
@Inject
private EntityGraphRetriever entityRetriever;
private int totalEntities = 0;
private int totalClassifiedEntities = 0; private int totalClassifiedEntities = 0;
private int getTotalClassifiedEntitiesHistorical = 0;
private int dimensionTagEntities = 10; private int dimensionTagEntities = 10;
private String dimensionTagDeleteGuid; private String dimensionTagDeleteGuid;
private String dimensionalTagGuid; private String dimensionalTagGuid;
...@@ -65,14 +75,14 @@ public class BasicSearchClassificationTest extends BasicTestSetup { ...@@ -65,14 +75,14 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
public void searchByALLTag() throws AtlasBaseException { public void searchByALLTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters(); SearchParameters params = new SearchParameters();
params.setClassification(ALL_CLASSIFICATION_TYPES); params.setClassification(ALL_CLASSIFICATION_TYPES);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities(); SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders)); List<AtlasVertex> vertices = processor.execute();
totalEntities = getEntityCount(); Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
totalClassifiedEntities = entityHeaders.size(); totalClassifiedEntities = vertices.size();
getTotalClassifiedEntitiesHistorical = getEntityWithTagCountHistorical();
} }
@Test @Test
...@@ -81,10 +91,14 @@ public class BasicSearchClassificationTest extends BasicTestSetup { ...@@ -81,10 +91,14 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setClassification(ALL_CLASSIFICATION_TYPES); params.setClassification(ALL_CLASSIFICATION_TYPES);
FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis())); FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(filterCriteria); params.setTagFilters(filterCriteria);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities(); SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), totalClassifiedEntities); Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), totalClassifiedEntities);
} }
@Test @Test
...@@ -95,29 +109,40 @@ public class BasicSearchClassificationTest extends BasicTestSetup { ...@@ -95,29 +109,40 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setTagFilters(filterCriteria); params.setTagFilters(filterCriteria);
params.setLimit(totalClassifiedEntities - 2); params.setLimit(totalClassifiedEntities - 2);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities(); SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), totalClassifiedEntities - 2); Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), totalClassifiedEntities - 2);
} }
@Test //@Test
public void searchByNOTCLASSIFIED() throws AtlasBaseException { public void searchByNOTCLASSIFIED() throws AtlasBaseException {
SearchParameters params = new SearchParameters(); SearchParameters params = new SearchParameters();
params.setClassification(NO_CLASSIFICATIONS); params.setClassification(NO_CLASSIFICATIONS);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities(); SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), totalEntities - totalClassifiedEntities); Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), 20);
} }
@Test @Test
public void searchByTag() throws AtlasBaseException { public void searchByTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters(); SearchParameters params = new SearchParameters();
params.setClassification(DIMENSION_CLASSIFICATION); params.setClassification(DIMENSION_CLASSIFICATION);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities(); SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), dimensionTagEntities); Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), dimensionTagEntities);
} }
@Test @Test
...@@ -126,11 +151,23 @@ public class BasicSearchClassificationTest extends BasicTestSetup { ...@@ -126,11 +151,23 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setClassification(DIMENSIONAL_CLASSIFICATION); params.setClassification(DIMENSIONAL_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.EQ, "Test"); FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.EQ, "Test");
params.setTagFilters(filterCriteria); params.setTagFilters(filterCriteria);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
assertEquals(entityHeaders.size(), 1); ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
assertEquals(entityHeaders.get(0).getGuid(), dimensionalTagGuid); List<AtlasVertex> vertices = processor.execute();
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), 1);
List<String> guids = vertices.stream().map(g -> {
try {
return entityRetriever.toAtlasEntityHeader(g).getGuid();
} catch (AtlasBaseException e) {
fail("Failure in mapping vertex to AtlasEntityHeader");
}
return "";
}).collect(Collectors.toList());
Assert.assertTrue(guids.contains(dimensionalTagGuid));
} }
...@@ -141,35 +178,86 @@ public class BasicSearchClassificationTest extends BasicTestSetup { ...@@ -141,35 +178,86 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setClassification(DIMENSION_CLASSIFICATION); params.setClassification(DIMENSION_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis())); FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(filterCriteria); params.setTagFilters(filterCriteria);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities(); SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), dimensionTagEntities); Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), dimensionTagEntities);
} }
@Test @Test
public void searchByWildcardTag() throws AtlasBaseException { public void searchByWildcardTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters(); SearchParameters params = new SearchParameters();
params.setClassification("Dimension*"); params.setClassification("Dimension*");
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities(); SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(entityHeaders.size(), dimensionTagEntities + 1); Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), dimensionTagEntities + 1);
} }
//@Test @Test
public void searchByALLWildcardTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification("*");
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(),20);
}
@Test
public void searchWithNotContains() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(DIMENSIONAL_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.NOT_CONTAINS, "Test");
params.setTagFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
Assert.assertTrue(CollectionUtils.isEmpty(vertices));
}
@Test
public void searchByTagAndGraphSysFilters() throws AtlasBaseException { public void searchByTagAndGraphSysFilters() throws AtlasBaseException {
SearchParameters params = new SearchParameters(); SearchParameters params = new SearchParameters();
params.setClassification(DIMENSION_CLASSIFICATION); params.setClassification(DIMENSION_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("__entityStatus", Operator.EQ, "DELETED"); FilterCriteria filterCriteria = getSingleFilterCondition("__entityStatus", Operator.EQ, "DELETED");
params.setTagFilters(filterCriteria); params.setTagFilters(filterCriteria);
params.setExcludeDeletedEntities(false); params.setExcludeDeletedEntities(false);
params.setLimit(20);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
assertEquals(entityHeaders.size(), 1); ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
assertEquals(entityHeaders.get(0).getGuid(), dimensionTagDeleteGuid); List<AtlasVertex> vertices = processor.execute();
Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
assertEquals(vertices.size(), 1);
List<String> guids = vertices.stream().map(g -> {
try {
return entityRetriever.toAtlasEntityHeader(g).getGuid();
} catch (AtlasBaseException e) {
fail("Failure in mapping vertex to AtlasEntityHeader");
}
return "";
}).collect(Collectors.toList());
Assert.assertTrue(guids.contains(dimensionTagDeleteGuid));
} }
...@@ -209,21 +297,8 @@ public class BasicSearchClassificationTest extends BasicTestSetup { ...@@ -209,21 +297,8 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
} }
private int getEntityCount() throws AtlasBaseException { @AfterClass
SearchParameters params = new SearchParameters(); public void teardown() {
params.setTypeName(ALL_ENTITY_TYPES); AtlasGraphProvider.cleanup();
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
return entityHeaders.size();
} }
private int getEntityWithTagCountHistorical() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(ALL_CLASSIFICATION_TYPES);
params.setExcludeDeletedEntities(false);
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
return entityHeaders.size();
}
} }
repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
...@@ -23,11 +23,13 @@ import org.apache.atlas.SortOrder; ...@@ -23,11 +23,13 @@ import org.apache.atlas.SortOrder;
import org.apache.atlas.TestModules; import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
...@@ -60,6 +62,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup { ...@@ -60,6 +62,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
@Inject @Inject
public GraphBackedSearchIndexer indexer; public GraphBackedSearchIndexer indexer;
private String expectedEntityName = "hive_Table_Null_tableType";
@Test @Test
public void searchTablesByClassification() throws AtlasBaseException { public void searchTablesByClassification() throws AtlasBaseException {
...@@ -132,7 +135,6 @@ public class EntitySearchProcessorTest extends BasicTestSetup { ...@@ -132,7 +135,6 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
@Test(priority = -1) @Test(priority = -1)
public void searchWithNEQ_stringAttr() throws AtlasBaseException { public void searchWithNEQ_stringAttr() throws AtlasBaseException {
String expectedEntityName = "hive_Table_Null_tableType";
createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE); createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
SearchParameters params = new SearchParameters(); SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE); params.setTypeName(HIVE_TABLE_TYPE);
...@@ -154,7 +156,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup { ...@@ -154,7 +156,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertTrue(nameList.contains(expectedEntityName)); assertTrue(nameList.contains(expectedEntityName));
} }
@Test(dependsOnMethods = "searchWithNEQ_stringAttr") @Test
public void searchWithNEQ_pipeSeperatedAttr() throws AtlasBaseException { public void searchWithNEQ_pipeSeperatedAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters(); SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE); params.setTypeName(HIVE_TABLE_TYPE);
...@@ -173,10 +175,10 @@ public class EntitySearchProcessorTest extends BasicTestSetup { ...@@ -173,10 +175,10 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name")); nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
} }
assertTrue(nameList.contains("hive_Table_Null_tableType")); assertTrue(nameList.contains(expectedEntityName));
} }
@Test(dependsOnMethods = "searchWithNEQ_stringAttr") @Test
public void searchWithNEQ_doubleAttr() throws AtlasBaseException { public void searchWithNEQ_doubleAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters(); SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE); params.setTypeName(HIVE_TABLE_TYPE);
...@@ -309,4 +311,53 @@ public class EntitySearchProcessorTest extends BasicTestSetup { ...@@ -309,4 +311,53 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertEquals(processor.execute().size(), 2); assertEquals(processor.execute().size(), 2);
} }
@Test
public void searchWithNotContains_stringAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.NOT_CONTAINS, "Managed");
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(vertices.size(), 3);
List<String> nameList = new ArrayList<>();
for (AtlasVertex vertex : vertices) {
nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
}
assertTrue(nameList.contains(expectedEntityName));
}
@Test
public void searchWithNotContains_pipeSeperatedAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("__classificationNames", SearchParameters.Operator.NOT_CONTAINS, METRIC_CLASSIFICATION);
params.setEntityFilters(filterCriteria);
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
EntitySearchProcessor processor = new EntitySearchProcessor(context);
List<AtlasVertex> vertices = processor.execute();
assertEquals(vertices.size(), 7);
List<String> nameList = new ArrayList<>();
for (AtlasVertex vertex : vertices) {
nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
}
assertTrue(nameList.contains(expectedEntityName));
}
@AfterClass
public void teardown() {
AtlasGraphProvider.cleanup();
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment