From 9a067d38e27fc2255647133373b952a6a64ed1a4 Mon Sep 17 00:00:00 2001
From: Pinal Shah <pinal.shah@freestoneinfotech.com>
Date: Tue, 30 Jun 2020 14:53:45 +0530
Subject: [PATCH] ATLAS-3782 : Support NOT_CONTAINS operator in basic search
Signed-off-by: nixonrodrigues <nixon@apache.org>
---
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java | 15 +++++++++++++--
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java | 104 +++++++++++++++++++++++++++++++++++---------------------------------------------------------------------
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java | 7 +++++--
repository/src/test/java/org/apache/atlas/BasicTestSetup.java | 43 +++++++++++++++++++++++++++++++++++++------
repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java | 338 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java | 229 -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
repository/src/test/java/org/apache/atlas/discovery/ClassificationSearchProcessorTest.java | 304 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++----
8 files changed, 787 insertions(+), 312 deletions(-)
create mode 100644 repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
delete mode 100644 repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java
create mode 100644 repository/src/test/java/org/apache/atlas/discovery/ClassificationSearchProcessorTest.java
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
index 6c06a3c..43114e9 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
@@ -195,11 +195,15 @@ public class AtlasSolrQueryBuilder {
if (!indexAttributes.contains(indexAttributeName)) {
StringBuilder sb = new StringBuilder();
- if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
+ if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works
- operator = Operator.EQ;
+ if (operator.equals(Operator.CONTAINS)) {
+ operator = Operator.EQ;
+ } else if (operator.equals(Operator.NOT_CONTAINS)) {
+ operator = Operator.NEQ;
+ }
attributeValue = getIndexQueryAttributeValue(attributeValue);
}
@@ -261,6 +265,9 @@ public class AtlasSolrQueryBuilder {
case CONTAINS:
withContains(queryBuilder, indexFieldName, attributeValue);
break;
+ case NOT_CONTAINS:
+ withNotContains(queryBuilder, indexFieldName, attributeValue);
+ break;
case IS_NULL:
withIsNull(queryBuilder, indexFieldName);
break;
@@ -388,6 +395,10 @@ public class AtlasSolrQueryBuilder {
queryBuilder.append("+").append(indexFieldName).append(":*").append(attributeValue).append("* ");
}
+ private void withNotContains(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ queryBuilder.append("*:* -").append(indexFieldName).append(":*").append(attributeValue).append("* ");
+ }
+
private void withIsNull(StringBuilder queryBuilder, String indexFieldName) {
queryBuilder.append("-").append(indexFieldName).append(":*").append(" ");
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 9c72cd4..647ff9c 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -18,13 +18,12 @@
package org.apache.atlas.discovery;
import org.apache.atlas.SortOrder;
-import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasClassificationType;
-import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
@@ -35,8 +34,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.script.ScriptEngine;
-import javax.script.ScriptException;
import java.util.*;
/**
@@ -52,10 +49,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
private final AtlasIndexQuery indexQuery;
private final AtlasIndexQuery classificationIndexQuery;
private final AtlasGraphQuery tagGraphQueryWithAttributes;
- private final Map<String, Object> gremlinQueryBindings;
- private final String gremlinTagFilterQuery;
private final Predicate traitPredicate;
private final Predicate isEntityPredicate;
+ private Predicate activePredicate;
// Some index engines may take space as a delimiter, when basic search is
// executed, unsatisfying results may be returned.
@@ -107,6 +103,14 @@ public class ClassificationSearchProcessor extends SearchProcessor {
traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
+ if (context.getSearchParameters().getExcludeDeletedEntities()) {
+ activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
+ .generatePredicate(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name(), String.class);
+ }
+
+ Predicate attributePredicate = null;
+ Predicate typeNamePredicate = null;
+
AtlasGraph graph = context.getGraph();
// index query directly on entity
@@ -156,19 +160,12 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
- Predicate typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
-
- if (typeNamePredicate != null) {
- inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
- }
-
- Predicate attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
+ this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
- if (attributePredicate != null) {
- inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
- }
+ typeNamePredicate = isClassificationRootType() ? null :
+ SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+ attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
- this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
} else {
classificationIndexQuery = null;
}
@@ -176,7 +173,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
// only registered classification will search with tag filters
if (useGraphSearchForClassification) {
- AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
AtlasGraphQuery query = graph.query();
if (!isClassificationRootType()) {
@@ -184,31 +180,20 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
- gremlinQueryBindings = new HashMap<>();
- StringBuilder gremlinQuery = new StringBuilder();
- gremlinQuery.append("g.V().has('__guid', within(guids))");
- gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER));
- gremlinQuery.append(".as('e').filter(out()");
- gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
+ typeNamePredicate = isClassificationRootType() ? null :
+ SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+ attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, allAttributes);
- // constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
-
- // After filtering on tags go back to e and output the list of entity vertices
- gremlinQuery.append(").toList()");
-
- gremlinQueryBindings.put("traitNames", typeAndSubTypes);
- gremlinQueryBindings.put("typeNames", typeAndSubTypes); // classification typeName
-
- gremlinTagFilterQuery = gremlinQuery.toString();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
- }
} else {
tagGraphQueryWithAttributes = null;
- gremlinTagFilterQuery = null;
- gremlinQueryBindings = null;
+ }
+
+ if (typeNamePredicate != null) {
+ inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
+ }
+ if (attributePredicate != null) {
+ inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
}
@@ -274,23 +259,23 @@ public class ClassificationSearchProcessor extends SearchProcessor {
CollectionUtils.filter(entityVertices, isEntityPredicate);
} else {
- if (tagGraphQueryWithAttributes != null) {
+ if (classificationIndexQuery != null) {
- Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
+ Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
- getVertices(queryResult, classificationVertices);
+ getVerticesFromIndexQueryResult(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
- } else if (classificationIndexQuery != null){
+ CollectionUtils.filter(classificationVertices, inMemoryPredicate);
+ } else if (tagGraphQueryWithAttributes != null) {
- Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
+ Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
- getVerticesFromIndexQueryResult(queryResult, classificationVertices);
+ getVertices(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
- // Do in-memory filtering before the graph query
CollectionUtils.filter(classificationVertices, inMemoryPredicate);
}
}
@@ -322,6 +307,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
// Do in-memory filtering
CollectionUtils.filter(entityVertices, isEntityPredicate);
+ if (activePredicate != null) {
+ CollectionUtils.filter(entityVertices, activePredicate);
+ }
super.filter(entityVertices);
@@ -347,30 +335,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size());
}
- //in case of classification type + graph attributes
- if (gremlinTagFilterQuery != null && gremlinQueryBindings != null) {
- // Now filter on the tag attributes
- Set<String> guids = getGuids(entityVertices);
-
- // Clear prior results
- entityVertices.clear();
-
- if (CollectionUtils.isNotEmpty(guids)) {
- gremlinQueryBindings.put("guids", guids);
- try {
- AtlasGraph graph = context.getGraph();
- ScriptEngine gremlinScriptEngine = graph.getGremlinScriptEngine();
- List<AtlasVertex> atlasVertices = (List<AtlasVertex>) graph.executeGremlinScript(gremlinScriptEngine, gremlinQueryBindings, gremlinTagFilterQuery, false);
-
- if (CollectionUtils.isNotEmpty(atlasVertices)) {
- entityVertices.addAll(atlasVertices);
- }
- } catch (AtlasBaseException | ScriptException e) {
- LOG.warn(e.getMessage(), e);
- }
- }
- } else if (inMemoryPredicate != null) {
+ if (inMemoryPredicate != null) {
//in case of classification type + index attributes
CollectionUtils.filter(entityVertices, traitPredicate);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index c9a6053..044179d 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -464,9 +464,9 @@ public abstract class SearchProcessor {
AtlasType attributeType = structType.getAttributeType(filterCriteria.getAttributeName());
if (AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeType.getTypeName())) {
- if (filterCriteria.getOperator() == SearchParameters.Operator.NEQ) {
+ if (filterCriteria.getOperator() == SearchParameters.Operator.NEQ || filterCriteria.getOperator() == SearchParameters.Operator.NOT_CONTAINS) {
if (LOG.isDebugEnabled()) {
- LOG.debug("NEQ operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", qualifiedName);
+ LOG.debug("{} operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", filterCriteria.getOperator(), qualifiedName);
}
ret = false;
@@ -633,6 +633,7 @@ public abstract class SearchProcessor {
op = SearchParameters.Operator.NOT_CONTAINS;
break;
case CONTAINS:
+ case NOT_CONTAINS:
if (attrName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
attrVal = getCustomAttributeIndexQueryValue(attrVal, true);
}
@@ -849,6 +850,8 @@ public abstract class SearchProcessor {
case NOT_NULL:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
break;
+ case NOT_CONTAINS:
+ break;
default:
LOG.warn("{}: unsupported operator. Ignored", operator);
break;
diff --git a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
index 8b98b39..270051f 100644
--- a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
+++ b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
@@ -19,10 +19,12 @@ package org.apache.atlas;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.glossary.GlossaryService;
import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.instance.AtlasClassification;
-import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.glossary.AtlasGlossary;
+import org.apache.atlas.model.glossary.AtlasGlossaryTerm;
+import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader;
+import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.typedef.*;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
@@ -42,6 +44,7 @@ import static org.testng.Assert.fail;
public abstract class BasicTestSetup {
+ // Entity type //
protected static final String DATABASE_TYPE = "hive_db";
protected static final String HIVE_TABLE_TYPE = "hive_table";
private static final String COLUMN_TYPE = "hive_column";
@@ -50,6 +53,7 @@ public abstract class BasicTestSetup {
private static final String VIEW_TYPE = "hive_process";
protected static final String DATASET_SUBTYPE = "Asset";
+ //Classification type //
public static final String DIMENSION_CLASSIFICATION = "Dimension";
public static final String FACT_CLASSIFICATION = "Fact";
public static final String PII_CLASSIFICATION = "PII";
@@ -59,14 +63,21 @@ public abstract class BasicTestSetup {
public static final String LOGDATA_CLASSIFICATION = "Log Data";
public static final String DIMENSIONAL_CLASSIFICATION = "Dimensional";
+ // Glossary type //
+ public static final String SALES_GLOSSARY = "salesGlossary";
+ public static final String SALES_TERM = "salesTerm";
+
@Inject
protected AtlasTypeRegistry typeRegistry;
@Inject
protected AtlasTypeDefStore typeDefStore;
@Inject
protected AtlasEntityStore entityStore;
+ @Inject
+ protected GlossaryService glossaryService;
private boolean baseLoaded = false;
+ private EntityMutationResponse hiveEntities;
protected void setupTestData() {
loadBaseModels();
@@ -77,6 +88,7 @@ public abstract class BasicTestSetup {
private void loadBaseModels() {
try {
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("0000-Area0/0011-glossary_model.json", typeDefStore, typeRegistry);
baseLoaded = true;
} catch (IOException | AtlasBaseException e) {
fail("Base model setup is required for test to run!");
@@ -97,7 +109,7 @@ public abstract class BasicTestSetup {
AtlasEntity.AtlasEntitiesWithExtInfo hiveTestEntities = hiveTestEntities();
try {
- entityStore.createOrUpdate(new AtlasEntityStream(hiveTestEntities), false);
+ hiveEntities = entityStore.createOrUpdate(new AtlasEntityStream(hiveTestEntities), false);
} catch (AtlasBaseException e) {
fail("Hive entities need to be created for test to run!");
}
@@ -450,12 +462,13 @@ public abstract class BasicTestSetup {
return datasetSubType;
}
- public void createDummyEntity(String name, String type, String... traitNames) throws AtlasBaseException {
+ public EntityMutationResponse createDummyEntity(String name, String type, String... traitNames) throws AtlasBaseException {
AtlasEntity entity = new AtlasEntity(type);
entity.setAttribute("name", name);
entity.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
entity.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList()));
- entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false);
+ EntityMutationResponse resp = entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false);
+ return resp;
}
public SearchParameters.FilterCriteria getSingleFilterCondition(String attName, SearchParameters.Operator op, String attrValue) {
@@ -472,4 +485,22 @@ public abstract class BasicTestSetup {
return filterCriteria;
}
+ public void assignGlossary() throws AtlasBaseException {
+ AtlasGlossary glossary = new AtlasGlossary();
+ glossary.setName(SALES_GLOSSARY);
+ glossary = glossaryService.createGlossary(glossary);
+
+ AtlasGlossaryTerm term = new AtlasGlossaryTerm();
+ term.setAnchor(new AtlasGlossaryHeader(glossary.getGuid()));
+ term.setName(SALES_TERM);
+ term = glossaryService.createTerm(term);
+
+ List<AtlasRelatedObjectId> guids = hiveEntities.getCreatedEntities().stream().filter(e -> e.getTypeName().equals(HIVE_TABLE_TYPE))
+ .map(p -> {AtlasRelatedObjectId obj = new AtlasRelatedObjectId();
+ obj.setGuid(p.getGuid());
+ obj.setTypeName(p.getTypeName()); return obj;}).collect(Collectors.toList());
+
+ glossaryService.assignTermToEntities(term.getGuid(), guids);
+ }
+
}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
new file mode 100644
index 0000000..d8c1546
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
@@ -0,0 +1,338 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.BasicTestSetup;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.commons.collections.CollectionUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.apache.atlas.model.discovery.SearchParameters.*;
+import static org.testng.Assert.assertEquals;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class AtlasDiscoveryServiceTest extends BasicTestSetup {
+
+ @Inject
+ private AtlasDiscoveryService discoveryService;
+
+ @BeforeClass
+ public void setup() throws AtlasException, AtlasBaseException {
+ ApplicationProperties.get().setProperty(ApplicationProperties.ENABLE_FREETEXT_SEARCH_CONF, true);
+ setupTestData();
+ createDimensionalTaggedEntity("sales");
+ assignGlossary();
+ }
+
+ /* TermSearchProcessor(TSP),
+ FreeTextSearchProcessor(FSP),
+ ClassificationSearchProcessor(CSP),
+ EntitySearchProcessor(ESP) */
+
+ @Test
+ public void term() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 10);
+ }
+
+ // TSP execute and CSP,ESP filter
+ @Test
+ public void term_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
+ params.setClassification(METRIC_CLASSIFICATION);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ for(AtlasEntityHeader e : entityHeaders){
+ System.out.println(e.toString());
+ }
+ assertEquals(entityHeaders.size(), 4);
+ }
+
+ @Test
+ public void term_entity() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
+ params.setTypeName(HIVE_TABLE_TYPE);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 10);
+ }
+
+ @Test
+ public void term_entity_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setClassification(DIMENSIONAL_CLASSIFICATION);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isEmpty(entityHeaders));
+ }
+
+ //FSP execute and CSP,ESP filter
+ @Test
+ public void query_ALLTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(ALL_CLASSIFICATION_TYPES);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 5);
+ }
+
+ @Test
+ public void query_ALLTag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(ALL_CLASSIFICATION_TYPES);
+ //typeName will check for only classification name not propogated classification
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("__typeName", Operator.NOT_CONTAINS, METRIC_CLASSIFICATION);
+ params.setTagFilters(fc);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 4);
+ }
+
+ @Test
+ public void query_NOTCLASSIFIEDTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(NO_CLASSIFICATIONS);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 1);
+ }
+
+
+ @Test
+ public void query_ALLWildcardTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification("*");
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 5);
+ }
+
+ @Test
+ public void query_wildcardTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification("Dimen*on");
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 2);
+ }
+
+ @Test
+ public void query_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(METRIC_CLASSIFICATION);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 3);
+ }
+
+ @Test
+ public void query_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(METRIC_CLASSIFICATION);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(fc);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 3);
+ }
+
+ @Test
+ public void query_entity() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 4);
+ }
+
+ @Test
+ public void query_entity_entityFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("tableType", Operator.NOT_NULL, "null");
+ params.setEntityFilters(fc);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 3);
+ }
+
+ @Test
+ public void query_entity_entityFilter_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("tableType", Operator.IS_NULL, "null");
+ params.setEntityFilters(fc);
+ params.setClassification(DIMENSIONAL_CLASSIFICATION);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 1);
+ }
+
+ @Test
+ public void query_entity_entityFilter_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria fcE = getSingleFilterCondition("tableType", Operator.IS_NULL, "null");
+ params.setEntityFilters(fcE);
+ params.setClassification(DIMENSIONAL_CLASSIFICATION);
+ params.setQuery("sales");
+ SearchParameters.FilterCriteria fcC = getSingleFilterCondition("attr1", Operator.EQ, "value1");
+ params.setTagFilters(fcC);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 1);
+ }
+
+ @Test
+ public void query_entity_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setClassification(METRIC_CLASSIFICATION);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(fc);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 2);
+
+ }
+
+ @Test
+ public void query_entity_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setClassification(METRIC_CLASSIFICATION);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 2);
+ }
+
+ // CSP Execute and ESP filter
+ @Test
+ public void entity_entityFilter_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria fcE = getSingleFilterCondition("tableType", Operator.EQ, "Managed");
+ params.setEntityFilters(fcE);
+ params.setClassification(METRIC_CLASSIFICATION);
+ SearchParameters.FilterCriteria fcC = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(fcC);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 4);
+
+ }
+
+ @Test
+ public void entity_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setClassification(METRIC_CLASSIFICATION);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(fc);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 4);
+ }
+
+ private void createDimensionalTaggedEntity(String name) throws AtlasBaseException {
+ EntityMutationResponse resp = createDummyEntity(name, HIVE_TABLE_TYPE);
+ AtlasEntityHeader entityHeader = resp.getCreatedEntities().get(0);
+ String guid = entityHeader.getGuid();
+ HashMap<String,Object> attr = new HashMap<>();
+ attr.put("attr1","value1");
+ entityStore.addClassification(Arrays.asList(guid), new AtlasClassification(DIMENSIONAL_CLASSIFICATION, attr));
+ }
+
+ @AfterClass
+ public void teardown() {
+ AtlasGraphProvider.cleanup();
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java b/repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java
deleted file mode 100644
index 9b16e91..0000000
--- a/repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.BasicTestSetup;
-import org.apache.atlas.TestModules;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.instance.AtlasClassification;
-import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.model.instance.AtlasEntityHeader;
-import org.apache.atlas.model.instance.EntityMutationResponse;
-import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
-import org.apache.commons.collections.CollectionUtils;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import static org.apache.atlas.model.discovery.SearchParameters.*;
-import static org.testng.Assert.assertEquals;
-
-@Guice(modules = TestModules.TestOnlyModule.class)
-public class BasicSearchClassificationTest extends BasicTestSetup {
-
- @Inject
- private AtlasDiscoveryService discoveryService;
-
- private int totalEntities = 0;
- private int totalClassifiedEntities = 0;
- private int getTotalClassifiedEntitiesHistorical = 0;
- private int dimensionTagEntities = 10;
- private String dimensionTagDeleteGuid;
- private String dimensionalTagGuid;
-
- @BeforeClass
- public void setup() throws AtlasBaseException {
- setupTestData();
- createDimensionTaggedEntityAndDelete();
- createDimensionalTaggedEntityWithAttr();
- }
-
- @Test(priority = -1)
- public void searchByALLTag() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification(ALL_CLASSIFICATION_TYPES);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
-
- totalEntities = getEntityCount();
- totalClassifiedEntities = entityHeaders.size();
- getTotalClassifiedEntitiesHistorical = getEntityWithTagCountHistorical();
- }
-
- @Test
- public void searchByALLTagAndIndexSysFilters() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification(ALL_CLASSIFICATION_TYPES);
- FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
- params.setTagFilters(filterCriteria);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), totalClassifiedEntities);
- }
-
- @Test
- public void searchByALLTagAndIndexSysFiltersToTestLimit() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification(ALL_CLASSIFICATION_TYPES);
- FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
- params.setTagFilters(filterCriteria);
- params.setLimit(totalClassifiedEntities - 2);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), totalClassifiedEntities - 2);
- }
-
- @Test
- public void searchByNOTCLASSIFIED() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification(NO_CLASSIFICATIONS);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), totalEntities - totalClassifiedEntities);
- }
-
- @Test
- public void searchByTag() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification(DIMENSION_CLASSIFICATION);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), dimensionTagEntities);
- }
-
- @Test
- public void searchByTagAndTagFilters() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification(DIMENSIONAL_CLASSIFICATION);
- FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.EQ, "Test");
- params.setTagFilters(filterCriteria);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), 1);
- assertEquals(entityHeaders.get(0).getGuid(), dimensionalTagGuid);
-
- }
-
- @Test
- public void searchByTagAndIndexSysFilters() throws AtlasBaseException {
-
- SearchParameters params = new SearchParameters();
- params.setClassification(DIMENSION_CLASSIFICATION);
- FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
- params.setTagFilters(filterCriteria);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), dimensionTagEntities);
- }
-
- @Test
- public void searchByWildcardTag() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification("Dimension*");
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), dimensionTagEntities + 1);
-
- }
-
- //@Test
- public void searchByTagAndGraphSysFilters() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification(DIMENSION_CLASSIFICATION);
- FilterCriteria filterCriteria = getSingleFilterCondition("__entityStatus", Operator.EQ, "DELETED");
- params.setTagFilters(filterCriteria);
- params.setExcludeDeletedEntities(false);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), 1);
- assertEquals(entityHeaders.get(0).getGuid(), dimensionTagDeleteGuid);
-
- }
-
- private void createDimensionTaggedEntityAndDelete() throws AtlasBaseException {
- AtlasEntity entityToDelete = new AtlasEntity(HIVE_TABLE_TYPE);
- entityToDelete.setAttribute("name", "entity to be deleted");
- entityToDelete.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "entity.tobedeleted");
-
- List<AtlasClassification> cls = new ArrayList<>();
- cls.add(new AtlasClassification(DIMENSION_CLASSIFICATION));
- entityToDelete.setClassifications(cls);
-
- //create entity
- EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entityToDelete)), false);
- AtlasEntityHeader entityHeader = response.getCreatedEntities().get(0);
- dimensionTagDeleteGuid = entityHeader.getGuid();
-
- //delete entity
- entityStore.deleteById(dimensionTagDeleteGuid);
- }
-
- private void createDimensionalTaggedEntityWithAttr() throws AtlasBaseException {
- AtlasEntity entityToDelete = new AtlasEntity(HIVE_TABLE_TYPE);
- entityToDelete.setAttribute("name", "Entity1");
- entityToDelete.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "entity.one");
-
- List<AtlasClassification> cls = new ArrayList<>();
- cls.add(new AtlasClassification(DIMENSIONAL_CLASSIFICATION, new HashMap<String, Object>() {{
- put("attr1", "Test");
- }}));
- entityToDelete.setClassifications(cls);
-
- //create entity
- final EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entityToDelete)), false);
- AtlasEntityHeader entityHeader = response.getCreatedEntities().get(0);
- dimensionalTagGuid = entityHeader.getGuid();
-
- }
-
- private int getEntityCount() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setTypeName(ALL_ENTITY_TYPES);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
- return entityHeaders.size();
- }
-
- private int getEntityWithTagCountHistorical() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification(ALL_CLASSIFICATION_TYPES);
- params.setExcludeDeletedEntities(false);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
- return entityHeaders.size();
- }
-
-}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/ClassificationSearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/ClassificationSearchProcessorTest.java
new file mode 100644
index 0000000..8693459
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/ClassificationSearchProcessorTest.java
@@ -0,0 +1,304 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.AtlasClient;
+import org.apache.atlas.BasicTestSetup;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
+import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
+import org.apache.commons.collections.CollectionUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.apache.atlas.model.discovery.SearchParameters.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class ClassificationSearchProcessorTest extends BasicTestSetup {
+
+ @Inject
+ private AtlasGraph graph;
+ @Inject
+ public GraphBackedSearchIndexer indexer;
+ @Inject
+ private EntityGraphRetriever entityRetriever;
+
+ private int totalClassifiedEntities = 0;
+ private int dimensionTagEntities = 10;
+ private String dimensionTagDeleteGuid;
+ private String dimensionalTagGuid;
+
+ @BeforeClass
+ public void setup() throws AtlasBaseException {
+ setupTestData();
+ createDimensionTaggedEntityAndDelete();
+ createDimensionalTaggedEntityWithAttr();
+ }
+
+ @Test(priority = -1)
+ public void searchByALLTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(ALL_CLASSIFICATION_TYPES);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ totalClassifiedEntities = vertices.size();
+ }
+
+ @Test
+ public void searchByALLTagAndIndexSysFilters() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(ALL_CLASSIFICATION_TYPES);
+ FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), totalClassifiedEntities);
+ }
+
+ @Test
+ public void searchByALLTagAndIndexSysFiltersToTestLimit() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(ALL_CLASSIFICATION_TYPES);
+ FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(filterCriteria);
+ params.setLimit(totalClassifiedEntities - 2);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), totalClassifiedEntities - 2);
+ }
+
+ //@Test
+ public void searchByNOTCLASSIFIED() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(NO_CLASSIFICATIONS);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), 20);
+ }
+
+ @Test
+ public void searchByTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(DIMENSION_CLASSIFICATION);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), dimensionTagEntities);
+ }
+
+ @Test
+ public void searchByTagAndTagFilters() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(DIMENSIONAL_CLASSIFICATION);
+ FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.EQ, "Test");
+ params.setTagFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), 1);
+ List<String> guids = vertices.stream().map(g -> {
+ try {
+ return entityRetriever.toAtlasEntityHeader(g).getGuid();
+ } catch (AtlasBaseException e) {
+ fail("Failure in mapping vertex to AtlasEntityHeader");
+ }
+ return "";
+ }).collect(Collectors.toList());
+ Assert.assertTrue(guids.contains(dimensionalTagGuid));
+
+ }
+
+ @Test
+ public void searchByTagAndIndexSysFilters() throws AtlasBaseException {
+
+ SearchParameters params = new SearchParameters();
+ params.setClassification(DIMENSION_CLASSIFICATION);
+ FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), dimensionTagEntities);
+ }
+
+ @Test
+ public void searchByWildcardTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification("Dimension*");
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), dimensionTagEntities + 1);
+
+ }
+
+ @Test
+ public void searchByALLWildcardTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification("*");
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(),20);
+
+ }
+
+ @Test
+ public void searchWithNotContains() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(DIMENSIONAL_CLASSIFICATION);
+ FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.NOT_CONTAINS, "Test");
+ params.setTagFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isEmpty(vertices));
+ }
+
+
+ @Test
+ public void searchByTagAndGraphSysFilters() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(DIMENSION_CLASSIFICATION);
+ FilterCriteria filterCriteria = getSingleFilterCondition("__entityStatus", Operator.EQ, "DELETED");
+ params.setTagFilters(filterCriteria);
+ params.setExcludeDeletedEntities(false);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), 1);
+ List<String> guids = vertices.stream().map(g -> {
+ try {
+ return entityRetriever.toAtlasEntityHeader(g).getGuid();
+ } catch (AtlasBaseException e) {
+ fail("Failure in mapping vertex to AtlasEntityHeader");
+ }
+ return "";
+ }).collect(Collectors.toList());
+ Assert.assertTrue(guids.contains(dimensionTagDeleteGuid));
+
+ }
+
+ private void createDimensionTaggedEntityAndDelete() throws AtlasBaseException {
+ AtlasEntity entityToDelete = new AtlasEntity(HIVE_TABLE_TYPE);
+ entityToDelete.setAttribute("name", "entity to be deleted");
+ entityToDelete.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "entity.tobedeleted");
+
+ List<AtlasClassification> cls = new ArrayList<>();
+ cls.add(new AtlasClassification(DIMENSION_CLASSIFICATION));
+ entityToDelete.setClassifications(cls);
+
+ //create entity
+ EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entityToDelete)), false);
+ AtlasEntityHeader entityHeader = response.getCreatedEntities().get(0);
+ dimensionTagDeleteGuid = entityHeader.getGuid();
+
+ //delete entity
+ entityStore.deleteById(dimensionTagDeleteGuid);
+ }
+
+ private void createDimensionalTaggedEntityWithAttr() throws AtlasBaseException {
+ AtlasEntity entityToDelete = new AtlasEntity(HIVE_TABLE_TYPE);
+ entityToDelete.setAttribute("name", "Entity1");
+ entityToDelete.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "entity.one");
+
+ List<AtlasClassification> cls = new ArrayList<>();
+ cls.add(new AtlasClassification(DIMENSIONAL_CLASSIFICATION, new HashMap<String, Object>() {{
+ put("attr1", "Test");
+ }}));
+ entityToDelete.setClassifications(cls);
+
+ //create entity
+ final EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entityToDelete)), false);
+ AtlasEntityHeader entityHeader = response.getCreatedEntities().get(0);
+ dimensionalTagGuid = entityHeader.getGuid();
+
+ }
+
+ @AfterClass
+ public void teardown() {
+ AtlasGraphProvider.cleanup();
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
index 8e42d17..7b40c21 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
@@ -23,11 +23,13 @@ import org.apache.atlas.SortOrder;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@@ -60,6 +62,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
@Inject
public GraphBackedSearchIndexer indexer;
+ private String expectedEntityName = "hive_Table_Null_tableType";
@Test
public void searchTablesByClassification() throws AtlasBaseException {
@@ -132,7 +135,6 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
@Test(priority = -1)
public void searchWithNEQ_stringAttr() throws AtlasBaseException {
- String expectedEntityName = "hive_Table_Null_tableType";
createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
@@ -154,7 +156,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertTrue(nameList.contains(expectedEntityName));
}
- @Test(dependsOnMethods = "searchWithNEQ_stringAttr")
+ @Test
public void searchWithNEQ_pipeSeperatedAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
@@ -173,10 +175,10 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
}
- assertTrue(nameList.contains("hive_Table_Null_tableType"));
+ assertTrue(nameList.contains(expectedEntityName));
}
- @Test(dependsOnMethods = "searchWithNEQ_stringAttr")
+ @Test
public void searchWithNEQ_doubleAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
@@ -309,4 +311,53 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertEquals(processor.execute().size(), 2);
}
+ @Test
+ public void searchWithNotContains_stringAttr() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.NOT_CONTAINS, "Managed");
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ assertEquals(vertices.size(), 3);
+
+ List<String> nameList = new ArrayList<>();
+ for (AtlasVertex vertex : vertices) {
+ nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
+ }
+
+ assertTrue(nameList.contains(expectedEntityName));
+ }
+
+ @Test
+ public void searchWithNotContains_pipeSeperatedAttr() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("__classificationNames", SearchParameters.Operator.NOT_CONTAINS, METRIC_CLASSIFICATION);
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ assertEquals(vertices.size(), 7);
+
+ List<String> nameList = new ArrayList<>();
+ for (AtlasVertex vertex : vertices) {
+ nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
+ }
+
+ assertTrue(nameList.contains(expectedEntityName));
+ }
+
+ @AfterClass
+ public void teardown() {
+ AtlasGraphProvider.cleanup();
+ }
+
}
--
libgit2 0.27.1