Skip to content

A
atlas
Commit d0c1b30c by Le Ma Committed by Madhan Neethiraj
Options

ATLAS-3533: search with term and tag doesn't return right results

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent e607ba1a
Showing with 150 additions and 20 deletions
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
......@@ -548,7 +548,7 @@ public final class TestUtilsV2 {
public static final String PII = "PII";
public static final String PHI = "PHI";
public static final String SUPER_TYPE_NAME = "Base";
public static final String SUPER_TYPE_NAME = "Referenceable";
public static final String STORAGE_DESC_TYPE = "hive_storagedesc";
public static final String PARTITION_STRUCT_TYPE = "partition_struct_type";
public static final String PARTITION_CLASS_TYPE = "partition_class_type";
......
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
......@@ -69,13 +69,14 @@ public class ClassificationSearchProcessor extends SearchProcessor {
private final AtlasGraphQuery tagGraphQueryWithAttributes;
private final Map<String, Object> gremlinQueryBindings;
private final String gremlinTagFilterQuery;
private final Predicate traitPredicate;
// Some index engines may take space as a delimiter, when basic search is
// executed, unsatisfying results may be returned.
// eg, an entity A has classification "cls" and B has "cls 1"
// when user execute a exact search for "cls", only A should be returned
// but both A and B are returned. To avoid this, we should filter the res.
private boolean whiteSpaceFilter = false;
private boolean whiteSpaceFilter = false;
public ClassificationSearchProcessor(SearchContext context) {
super(context);
......@@ -146,8 +147,13 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
LOG.debug("Using query string '{}'.", indexQuery);
traitPredicate = buildTraitPredict(classificationType);
inMemoryPredicate = inMemoryPredicate == null ? traitPredicate : PredicateUtils.andPredicate(inMemoryPredicate, traitPredicate);
} else {
indexQuery = null;
indexQuery = null;
traitPredicate = null;
}
// index query directly on classification
......@@ -165,12 +171,15 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
Predicate typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
if (typeNamePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
}
Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
if (attributePredicate != null) {
inMemoryPredicate = typeNamePredicate == null ? attributePredicate : PredicateUtils.andPredicate(typeNamePredicate, attributePredicate);
} else {
inMemoryPredicate = typeNamePredicate;
inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
......@@ -360,6 +369,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.warn(e.getMessage(), e);
}
}
} else if (traitPredicate != null) {
CollectionUtils.filter(entityVertices, traitPredicate);
}
super.filter(entityVertices);
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
......@@ -88,20 +88,10 @@ public class EntitySearchProcessor extends SearchProcessor {
}
final Predicate typeNamePredicate;
final Predicate traitPredicate;
final Predicate traitPredicate = buildTraitPredict(classificationType);
final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class));
}
if (!isEntityRootType()) {
typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator().generatePredicate(TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
......
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
......@@ -243,7 +243,7 @@ public class SearchContext {
}
boolean needClassificationProcessor() {
return (classificationType != null || isWildCardSearch());
return (classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
}
boolean isBuiltInClassificationType() {
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
......@@ -32,6 +32,7 @@ import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.atlas.util.SearchPredicateUtil.*;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
......@@ -45,10 +46,15 @@ import java.math.BigInteger;
import java.util.*;
import java.util.regex.Pattern;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.util.SearchPredicateUtil.*;
public abstract class SearchProcessor {
......@@ -179,6 +185,21 @@ public abstract class SearchProcessor {
}
}
protected Predicate buildTraitPredict(AtlasClassificationType classificationType) {
Predicate traitPredicate;
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || context.isWildCardSearch()) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class));
}
return traitPredicate;
}
protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) {
if (structType == null || filterCriteria == null) {
......
repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
......@@ -294,7 +294,7 @@ public class ZipFileResourceTestUtils {
createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
}
private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
public static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
if(typesFromJson == null) {
return;
}
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
......@@ -30,19 +30,26 @@ import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPE
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createTypesAsNeeded;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.glossary.GlossaryService;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.glossary.AtlasGlossary;
import org.apache.atlas.model.glossary.AtlasGlossaryTerm;
import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.ClassificationAssociateRequest;
import org.apache.atlas.model.instance.EntityMutationResponse;
......@@ -55,6 +62,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.web.rest.DiscoveryREST;
import org.apache.atlas.web.rest.EntityREST;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
......@@ -64,6 +72,8 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
......@@ -81,12 +91,16 @@ public class TestEntitiesREST {
@Inject
private AtlasTypeRegistry typeRegistry;
@Inject
private GlossaryService glossaryService;
@Inject
private AtlasTypeDefStore typeStore;
@Inject
private DiscoveryREST discoveryREST;
@Inject
private EntityREST entityREST;
private AtlasGlossary glossary;
private AtlasGlossaryTerm term1;
private AtlasEntity dbEntity;
private AtlasEntity tableEntity;
private AtlasEntity tableEntity2;
......@@ -108,8 +122,14 @@ public class TestEntitiesREST {
}
}
loadGlossaryType();
createEntities();
createGlossary();
createTerms();
initTagMap();
registerEntities();
......@@ -372,6 +392,33 @@ public class TestEntitiesREST {
}
@Test(dependsOnMethods = "testSearchByMultiTags")
public void testSearchByTerms() throws Exception {
// database - phi, felt_classification
// table1 - phi, classification, term: term1 | table2 - classification, term:term2
// column - phi
assignTermTo(term1, tableEntity);
assignTermTo(term1, tableEntity2);
searchParameters = new SearchParameters();
searchParameters.setTermName(term1.getName() + "@testSearchGlossary");
searchParameters.setClassification(CLASSIFICATION);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
searchParameters.setClassification(PII);
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNull(res.getEntities());
searchParameters.setClassification(PHI);
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
}
@Test(dependsOnMethods = "testSearchByMultiTags")
public void testSearchByOtherSystemAttributes() throws Exception {
// database - phi, felt_classification
......@@ -506,6 +553,14 @@ public class TestEntitiesREST {
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
searchParameters = new SearchParameters();
searchParameters.setQuery("classification");
searchParameters.setClassification(PHI);
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
}
@Test(dependsOnMethods = "testSearchBySystemAttributesWithQuery")
......@@ -536,7 +591,7 @@ public class TestEntitiesREST {
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 5);
Assert.assertEquals(res.getEntities().size(), 7);
}
@Test(dependsOnMethods = "testTagSearchBySystemAttributes")
......@@ -582,6 +637,59 @@ public class TestEntitiesREST {
*
*/
private void loadGlossaryType() throws IOException, AtlasBaseException {
registerAtlasTypesDef("/addons/models/0000-Area0/0010-base_model.json");
registerAtlasTypesDef("/addons/models/0000-Area0/0011-glossary_model.json");
}
private void registerAtlasTypesDef (String path) throws IOException, AtlasBaseException {
String projectBaseDirectory = System.getProperty("projectBaseDir");
String baseModel = projectBaseDirectory + path;
File f = new File(baseModel);
String s = FileUtils.readFileToString(f);
createTypesAsNeeded(AtlasType.fromJson(s, AtlasTypesDef.class), typeStore, typeRegistry);
}
private void createGlossary() throws AtlasBaseException {
glossary = new AtlasGlossary();
glossary.setQualifiedName("testSearchGlossary");
glossary.setName("Search glossary");
glossary.setShortDescription("Short description");
glossary.setLongDescription("Long description");
glossary.setUsage("N/A");
glossary.setLanguage("en-US");
AtlasGlossary created = glossaryService.createGlossary(glossary);
glossary.setGuid(created.getGuid());
}
private void assignTermTo(AtlasGlossaryTerm term, AtlasEntity entity) throws AtlasBaseException {
AtlasRelatedObjectId relatedObjectId = new AtlasRelatedObjectId();
relatedObjectId.setGuid(entity.getGuid());
relatedObjectId.setTypeName(entity.getTypeName());
glossaryService.assignTermToEntities(term.getGuid(), Arrays.asList(relatedObjectId));
}
private void createTerms() throws AtlasBaseException {
term1 = new AtlasGlossaryTerm();
// Glossary anchor
AtlasGlossaryHeader glossaryId = new AtlasGlossaryHeader();
glossaryId.setGlossaryGuid(glossary.getGuid());
term1.setName("term1");
term1.setShortDescription("Short description");
term1.setLongDescription("Long description");
term1.setAbbreviation("CHK");
term1.setExamples(Arrays.asList("Personal", "Joint"));
term1.setUsage("N/A");
term1.setAnchor(glossaryId);
AtlasGlossaryTerm created1 = glossaryService.createTerm(term1);
term1.setGuid(created1.getGuid());
}
private void createEntities() {
dbEntity = TestUtilsV2.createDBEntity();
tableEntity = TestUtilsV2.createTableEntity(dbEntity);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment