ATLAS-3533: search with term and tag doesn't return right results

intg/src/test/java/org/apache/atlas/TestUtilsV2.java

@@ -548,7 +548,7 @@ public final class TestUtilsV2 {
 
     public static final String PII = "PII";
     public static final String PHI = "PHI";
-    public static final String SUPER_TYPE_NAME = "Base";
+    public static final String SUPER_TYPE_NAME = "Referenceable";
     public static final String STORAGE_DESC_TYPE = "hive_storagedesc";
     public static final String PARTITION_STRUCT_TYPE = "partition_struct_type";
     public static final String PARTITION_CLASS_TYPE = "partition_class_type";

repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java

@@ -69,6 +69,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
     private final AtlasGraphQuery        tagGraphQueryWithAttributes;
     private final Map<String, Object>    gremlinQueryBindings;
     private final String                 gremlinTagFilterQuery;
+    private final Predicate              traitPredicate;
 
     // Some index engines may take space as a delimiter, when basic search is
     // executed, unsatisfying results may be returned.
@@ -146,8 +147,13 @@ public class ClassificationSearchProcessor extends SearchProcessor {
             indexQuery              = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
 
             LOG.debug("Using query string  '{}'.", indexQuery);
+
+            traitPredicate    = buildTraitPredict(classificationType);
+            inMemoryPredicate = inMemoryPredicate == null ? traitPredicate : PredicateUtils.andPredicate(inMemoryPredicate, traitPredicate);
+
         } else {
             indexQuery     = null;
+            traitPredicate = null;
         }
 
         // index query directly on classification
@@ -165,12 +171,15 @@ public class ClassificationSearchProcessor extends SearchProcessor {
             indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
 
             Predicate typeNamePredicate  = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+
+            if (typeNamePredicate != null) {
+                inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
+            }
+
             Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
 
             if (attributePredicate != null) {
-                inMemoryPredicate = typeNamePredicate == null ? attributePredicate : PredicateUtils.andPredicate(typeNamePredicate, attributePredicate);
-            } else {
-                inMemoryPredicate = typeNamePredicate;
+                inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
             }
 
             this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
@@ -360,6 +369,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
                     LOG.warn(e.getMessage(), e);
                 }
             }
+        } else if (traitPredicate != null) {
+            CollectionUtils.filter(entityVertices, traitPredicate);
         }
 
         super.filter(entityVertices);

repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java

@@ -88,20 +88,10 @@ public class EntitySearchProcessor extends SearchProcessor {
         }
 
         final Predicate typeNamePredicate;
-        final Predicate traitPredicate;
+        final Predicate traitPredicate    = buildTraitPredict(classificationType);
         final Predicate activePredicate   = SearchPredicateUtil.getEQPredicateGenerator()
                                                                .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
 
-        if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
-            traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
-                                                        SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
-        } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
-            traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
-                                                         SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
-        } else {
-            traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class),
-                                                        SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class));
-        }
 
         if (!isEntityRootType()) {
             typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator().generatePredicate(TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);

repository/src/main/java/org/apache/atlas/discovery/SearchContext.java

@@ -243,7 +243,7 @@ public class SearchContext {
     }
 
     boolean needClassificationProcessor() {
-        return (classificationType != null || isWildCardSearch());
+        return (classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
     }
 
     boolean isBuiltInClassificationType() {

repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java

@@ -32,6 +32,7 @@ import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
 import org.apache.atlas.type.*;
 import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
 import org.apache.atlas.util.AtlasGremlinQueryProvider;
+import org.apache.atlas.util.SearchPredicateUtil;
 import org.apache.atlas.util.SearchPredicateUtil.*;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.Predicate;
@@ -45,10 +46,15 @@ import java.math.BigInteger;
 import java.util.*;
 import java.util.regex.Pattern;
 
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
 import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
 import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
 import static org.apache.atlas.util.SearchPredicateUtil.*;
 
 public abstract class SearchProcessor {
@@ -179,6 +185,21 @@ public abstract class SearchProcessor {
         }
     }
 
+    protected Predicate buildTraitPredict(AtlasClassificationType classificationType) {
+        Predicate traitPredicate;
+        if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || context.isWildCardSearch()) {
+            traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
+                SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+        } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
+            traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
+                SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+        } else {
+            traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class),
+                SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class));
+        }
+        return traitPredicate;
+    }
+
 
     protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) {
         if (structType == null || filterCriteria == null) {

repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java

@@ -294,7 +294,7 @@ public class ZipFileResourceTestUtils {
         createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
     }
 
-    private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+    public static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
         if(typesFromJson == null) {
             return;
         }

webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java

@@ -30,19 +30,26 @@ import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPE
 import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createTypesAsNeeded;
 
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.RequestContext;
 import org.apache.atlas.TestModules;
 import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.glossary.GlossaryService;
 import org.apache.atlas.model.discovery.AtlasSearchResult;
 import org.apache.atlas.model.discovery.SearchParameters;
 import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.glossary.AtlasGlossary;
+import org.apache.atlas.model.glossary.AtlasGlossaryTerm;
+import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader;
 import org.apache.atlas.model.instance.AtlasClassification;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.instance.AtlasRelatedObjectId;
 import org.apache.atlas.model.instance.AtlasStruct;
 import org.apache.atlas.model.instance.ClassificationAssociateRequest;
 import org.apache.atlas.model.instance.EntityMutationResponse;
@@ -55,6 +62,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.type.AtlasTypeUtil;
 import org.apache.atlas.web.rest.DiscoveryREST;
 import org.apache.atlas.web.rest.EntityREST;
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
@@ -64,6 +72,8 @@ import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
 import javax.inject.Inject;
+
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -81,12 +91,16 @@ public class TestEntitiesREST {
     @Inject
     private AtlasTypeRegistry typeRegistry;
     @Inject
+    private GlossaryService   glossaryService;
+    @Inject
     private AtlasTypeDefStore typeStore;
     @Inject
     private DiscoveryREST     discoveryREST;
     @Inject
     private EntityREST        entityREST;
 
+    private AtlasGlossary                     glossary;
+    private AtlasGlossaryTerm                 term1;
     private AtlasEntity                       dbEntity;
     private AtlasEntity                       tableEntity;
     private AtlasEntity                       tableEntity2;
@@ -108,8 +122,14 @@ public class TestEntitiesREST {
             }
         }
 
+        loadGlossaryType();
+
         createEntities();
 
+        createGlossary();
+
+        createTerms();
+
         initTagMap();
 
         registerEntities();
@@ -372,6 +392,33 @@ public class TestEntitiesREST {
     }
 
     @Test(dependsOnMethods = "testSearchByMultiTags")
+    public void testSearchByTerms() throws Exception {
+
+        // database - phi, felt_classification
+        // table1 - phi, classification, term: term1 | table2 - classification, term:term2
+        // column - phi
+        assignTermTo(term1, tableEntity);
+        assignTermTo(term1, tableEntity2);
+
+        searchParameters = new SearchParameters();
+        searchParameters.setTermName(term1.getName() + "@testSearchGlossary");
+        searchParameters.setClassification(CLASSIFICATION);
+
+        AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
+        Assert.assertNotNull(res.getEntities());
+        Assert.assertEquals(res.getEntities().size(), 2);
+
+        searchParameters.setClassification(PII);
+        res = discoveryREST.searchWithParameters(searchParameters);
+        Assert.assertNull(res.getEntities());
+
+        searchParameters.setClassification(PHI);
+        res = discoveryREST.searchWithParameters(searchParameters);
+        Assert.assertNotNull(res.getEntities());
+        Assert.assertEquals(res.getEntities().size(), 1);
+    }
+
+    @Test(dependsOnMethods = "testSearchByMultiTags")
     public void testSearchByOtherSystemAttributes() throws Exception {
 
         // database - phi, felt_classification
@@ -506,6 +553,14 @@ public class TestEntitiesREST {
         AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
         Assert.assertNotNull(res.getEntities());
         Assert.assertEquals(res.getEntities().size(), 1);
+
+        searchParameters = new SearchParameters();
+        searchParameters.setQuery("classification");
+        searchParameters.setClassification(PHI);
+
+        res = discoveryREST.searchWithParameters(searchParameters);
+        Assert.assertNotNull(res.getEntities());
+        Assert.assertEquals(res.getEntities().size(), 1);
     }
 
     @Test(dependsOnMethods = "testSearchBySystemAttributesWithQuery")
@@ -536,7 +591,7 @@ public class TestEntitiesREST {
         AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
 
         Assert.assertNotNull(res.getEntities());
-        Assert.assertEquals(res.getEntities().size(), 5);
+        Assert.assertEquals(res.getEntities().size(), 7);
     }
 
     @Test(dependsOnMethods = "testTagSearchBySystemAttributes")
@@ -582,6 +637,59 @@ public class TestEntitiesREST {
 	*
 	*/
 
+    private void loadGlossaryType() throws IOException, AtlasBaseException {
+
+        registerAtlasTypesDef("/addons/models/0000-Area0/0010-base_model.json");
+        registerAtlasTypesDef("/addons/models/0000-Area0/0011-glossary_model.json");
+    }
+
+    private void registerAtlasTypesDef (String path) throws IOException, AtlasBaseException {
+        String projectBaseDirectory = System.getProperty("projectBaseDir");
+
+        String baseModel = projectBaseDirectory + path;
+        File f = new File(baseModel);
+        String s = FileUtils.readFileToString(f);
+        createTypesAsNeeded(AtlasType.fromJson(s, AtlasTypesDef.class), typeStore, typeRegistry);
+    }
+
+	   private void createGlossary() throws AtlasBaseException {
+        glossary = new AtlasGlossary();
+        glossary.setQualifiedName("testSearchGlossary");
+        glossary.setName("Search glossary");
+        glossary.setShortDescription("Short description");
+        glossary.setLongDescription("Long description");
+        glossary.setUsage("N/A");
+        glossary.setLanguage("en-US");
+
+        AtlasGlossary created = glossaryService.createGlossary(glossary);
+        glossary.setGuid(created.getGuid());
+    }
+
+	   private void assignTermTo(AtlasGlossaryTerm term, AtlasEntity entity) throws AtlasBaseException {
+        AtlasRelatedObjectId relatedObjectId = new AtlasRelatedObjectId();
+        relatedObjectId.setGuid(entity.getGuid());
+        relatedObjectId.setTypeName(entity.getTypeName());
+
+        glossaryService.assignTermToEntities(term.getGuid(), Arrays.asList(relatedObjectId));
+    }
+
+    private void createTerms() throws AtlasBaseException {
+        term1 = new AtlasGlossaryTerm();
+        // Glossary anchor
+        AtlasGlossaryHeader glossaryId = new AtlasGlossaryHeader();
+        glossaryId.setGlossaryGuid(glossary.getGuid());
+
+        term1.setName("term1");
+        term1.setShortDescription("Short description");
+        term1.setLongDescription("Long description");
+        term1.setAbbreviation("CHK");
+        term1.setExamples(Arrays.asList("Personal", "Joint"));
+        term1.setUsage("N/A");
+        term1.setAnchor(glossaryId);
+	       AtlasGlossaryTerm created1 = glossaryService.createTerm(term1);
+	       term1.setGuid(created1.getGuid());
+    }
+
     private void createEntities() {
         dbEntity     = TestUtilsV2.createDBEntity();
         tableEntity  = TestUtilsV2.createTableEntity(dbEntity);