diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index ff79994..6fe5063 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -548,7 +548,7 @@ public final class TestUtilsV2 { public static final String PII = "PII"; public static final String PHI = "PHI"; - public static final String SUPER_TYPE_NAME = "Base"; + public static final String SUPER_TYPE_NAME = "Referenceable"; public static final String STORAGE_DESC_TYPE = "hive_storagedesc"; public static final String PARTITION_STRUCT_TYPE = "partition_struct_type"; public static final String PARTITION_CLASS_TYPE = "partition_class_type"; diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java index 63880f9..c0a5a46 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java @@ -69,13 +69,14 @@ public class ClassificationSearchProcessor extends SearchProcessor { private final AtlasGraphQuery tagGraphQueryWithAttributes; private final Map<String, Object> gremlinQueryBindings; private final String gremlinTagFilterQuery; + private final Predicate traitPredicate; // Some index engines may take space as a delimiter, when basic search is // executed, unsatisfying results may be returned. // eg, an entity A has classification "cls" and B has "cls 1" // when user execute a exact search for "cls", only A should be returned // but both A and B are returned. To avoid this, we should filter the res. - private boolean whiteSpaceFilter = false; + private boolean whiteSpaceFilter = false; public ClassificationSearchProcessor(SearchContext context) { super(context); @@ -146,8 +147,13 @@ public class ClassificationSearchProcessor extends SearchProcessor { indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString); LOG.debug("Using query string '{}'.", indexQuery); + + traitPredicate = buildTraitPredict(classificationType); + inMemoryPredicate = inMemoryPredicate == null ? traitPredicate : PredicateUtils.andPredicate(inMemoryPredicate, traitPredicate); + } else { - indexQuery = null; + indexQuery = null; + traitPredicate = null; } // index query directly on classification @@ -165,12 +171,15 @@ public class ClassificationSearchProcessor extends SearchProcessor { indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll(""); Predicate typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class); + + if (typeNamePredicate != null) { + inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate); + } + Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes); if (attributePredicate != null) { - inMemoryPredicate = typeNamePredicate == null ? attributePredicate : PredicateUtils.andPredicate(typeNamePredicate, attributePredicate); - } else { - inMemoryPredicate = typeNamePredicate; + inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate); } this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString); @@ -360,6 +369,8 @@ public class ClassificationSearchProcessor extends SearchProcessor { LOG.warn(e.getMessage(), e); } } + } else if (traitPredicate != null) { + CollectionUtils.filter(entityVertices, traitPredicate); } super.filter(entityVertices); diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java index 93dbe87..b5606d0 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java @@ -88,20 +88,10 @@ public class EntitySearchProcessor extends SearchProcessor { } final Predicate typeNamePredicate; - final Predicate traitPredicate; + final Predicate traitPredicate = buildTraitPredict(classificationType); final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator() .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class); - if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) { - traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), - SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); - } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { - traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), - SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); - } else { - traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class), - SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class)); - } if (!isEntityRootType()) { typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator().generatePredicate(TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class); diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java index ef50d8d..3534113 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java @@ -243,7 +243,7 @@ public class SearchContext { } boolean needClassificationProcessor() { - return (classificationType != null || isWildCardSearch()); + return (classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ; } boolean isBuiltInClassificationType() { diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java index b04021f..76d5a01 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java +++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java @@ -32,6 +32,7 @@ import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.type.*; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.util.AtlasGremlinQueryProvider; +import org.apache.atlas.util.SearchPredicateUtil; import org.apache.atlas.util.SearchPredicateUtil.*; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Predicate; @@ -45,10 +46,15 @@ import java.math.BigInteger; import java.util.*; import java.util.regex.Pattern; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION; import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY; import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY; +import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY; import static org.apache.atlas.util.SearchPredicateUtil.*; public abstract class SearchProcessor { @@ -179,6 +185,21 @@ public abstract class SearchProcessor { } } + protected Predicate buildTraitPredict(AtlasClassificationType classificationType) { + Predicate traitPredicate; + if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || context.isWildCardSearch()) { + traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), + SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); + } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) { + traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class), + SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class)); + } else { + traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class), + SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class)); + } + return traitPredicate; + } + protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) { if (structType == null || filterCriteria == null) { diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java index f4e84b2..0ffc3d5 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java @@ -294,7 +294,7 @@ public class ZipFileResourceTestUtils { createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry); } - private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + public static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException { if(typesFromJson == null) { return; } diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java index e2906a2..afc4c50 100644 --- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java +++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java @@ -30,19 +30,26 @@ import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPE import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createTypesAsNeeded; import org.apache.atlas.AtlasClient; import org.apache.atlas.RequestContext; import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.glossary.GlossaryService; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria; +import org.apache.atlas.model.glossary.AtlasGlossary; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.ClassificationAssociateRequest; import org.apache.atlas.model.instance.EntityMutationResponse; @@ -55,6 +62,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.web.rest.DiscoveryREST; import org.apache.atlas.web.rest.EntityREST; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -64,6 +72,8 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; + +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -81,12 +91,16 @@ public class TestEntitiesREST { @Inject private AtlasTypeRegistry typeRegistry; @Inject + private GlossaryService glossaryService; + @Inject private AtlasTypeDefStore typeStore; @Inject private DiscoveryREST discoveryREST; @Inject private EntityREST entityREST; + private AtlasGlossary glossary; + private AtlasGlossaryTerm term1; private AtlasEntity dbEntity; private AtlasEntity tableEntity; private AtlasEntity tableEntity2; @@ -108,8 +122,14 @@ public class TestEntitiesREST { } } + loadGlossaryType(); + createEntities(); + createGlossary(); + + createTerms(); + initTagMap(); registerEntities(); @@ -372,6 +392,33 @@ public class TestEntitiesREST { } @Test(dependsOnMethods = "testSearchByMultiTags") + public void testSearchByTerms() throws Exception { + + // database - phi, felt_classification + // table1 - phi, classification, term: term1 | table2 - classification, term:term2 + // column - phi + assignTermTo(term1, tableEntity); + assignTermTo(term1, tableEntity2); + + searchParameters = new SearchParameters(); + searchParameters.setTermName(term1.getName() + "@testSearchGlossary"); + searchParameters.setClassification(CLASSIFICATION); + + AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters); + Assert.assertNotNull(res.getEntities()); + Assert.assertEquals(res.getEntities().size(), 2); + + searchParameters.setClassification(PII); + res = discoveryREST.searchWithParameters(searchParameters); + Assert.assertNull(res.getEntities()); + + searchParameters.setClassification(PHI); + res = discoveryREST.searchWithParameters(searchParameters); + Assert.assertNotNull(res.getEntities()); + Assert.assertEquals(res.getEntities().size(), 1); + } + + @Test(dependsOnMethods = "testSearchByMultiTags") public void testSearchByOtherSystemAttributes() throws Exception { // database - phi, felt_classification @@ -506,6 +553,14 @@ public class TestEntitiesREST { AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters); Assert.assertNotNull(res.getEntities()); Assert.assertEquals(res.getEntities().size(), 1); + + searchParameters = new SearchParameters(); + searchParameters.setQuery("classification"); + searchParameters.setClassification(PHI); + + res = discoveryREST.searchWithParameters(searchParameters); + Assert.assertNotNull(res.getEntities()); + Assert.assertEquals(res.getEntities().size(), 1); } @Test(dependsOnMethods = "testSearchBySystemAttributesWithQuery") @@ -536,7 +591,7 @@ public class TestEntitiesREST { AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters); Assert.assertNotNull(res.getEntities()); - Assert.assertEquals(res.getEntities().size(), 5); + Assert.assertEquals(res.getEntities().size(), 7); } @Test(dependsOnMethods = "testTagSearchBySystemAttributes") @@ -582,6 +637,59 @@ public class TestEntitiesREST { * */ + private void loadGlossaryType() throws IOException, AtlasBaseException { + + registerAtlasTypesDef("/addons/models/0000-Area0/0010-base_model.json"); + registerAtlasTypesDef("/addons/models/0000-Area0/0011-glossary_model.json"); + } + + private void registerAtlasTypesDef (String path) throws IOException, AtlasBaseException { + String projectBaseDirectory = System.getProperty("projectBaseDir"); + + String baseModel = projectBaseDirectory + path; + File f = new File(baseModel); + String s = FileUtils.readFileToString(f); + createTypesAsNeeded(AtlasType.fromJson(s, AtlasTypesDef.class), typeStore, typeRegistry); + } + + private void createGlossary() throws AtlasBaseException { + glossary = new AtlasGlossary(); + glossary.setQualifiedName("testSearchGlossary"); + glossary.setName("Search glossary"); + glossary.setShortDescription("Short description"); + glossary.setLongDescription("Long description"); + glossary.setUsage("N/A"); + glossary.setLanguage("en-US"); + + AtlasGlossary created = glossaryService.createGlossary(glossary); + glossary.setGuid(created.getGuid()); + } + + private void assignTermTo(AtlasGlossaryTerm term, AtlasEntity entity) throws AtlasBaseException { + AtlasRelatedObjectId relatedObjectId = new AtlasRelatedObjectId(); + relatedObjectId.setGuid(entity.getGuid()); + relatedObjectId.setTypeName(entity.getTypeName()); + + glossaryService.assignTermToEntities(term.getGuid(), Arrays.asList(relatedObjectId)); + } + + private void createTerms() throws AtlasBaseException { + term1 = new AtlasGlossaryTerm(); + // Glossary anchor + AtlasGlossaryHeader glossaryId = new AtlasGlossaryHeader(); + glossaryId.setGlossaryGuid(glossary.getGuid()); + + term1.setName("term1"); + term1.setShortDescription("Short description"); + term1.setLongDescription("Long description"); + term1.setAbbreviation("CHK"); + term1.setExamples(Arrays.asList("Personal", "Joint")); + term1.setUsage("N/A"); + term1.setAnchor(glossaryId); + AtlasGlossaryTerm created1 = glossaryService.createTerm(term1); + term1.setGuid(created1.getGuid()); + } + private void createEntities() { dbEntity = TestUtilsV2.createDBEntity(); tableEntity = TestUtilsV2.createTableEntity(dbEntity);