Commit df85aa7a by Le Ma Committed by Sarath Subramanian

ATLAS-3457: Support Multi-Classification search without attributes

parent d1f34186
...@@ -62,6 +62,7 @@ public class SearchParameters implements Serializable { ...@@ -62,6 +62,7 @@ public class SearchParameters implements Serializable {
public static final String ALL_CLASSIFICATIONS = "_CLASSIFIED"; public static final String ALL_CLASSIFICATIONS = "_CLASSIFIED";
public static final String NO_CLASSIFICATIONS = "_NOT_CLASSIFIED"; public static final String NO_CLASSIFICATIONS = "_NOT_CLASSIFIED";
public static final String ALL_ENTITY_TYPES = "_ALL_ENTITY_TYPES"; public static final String ALL_ENTITY_TYPES = "_ALL_ENTITY_TYPES";
public static final String ALL_CLASSIFICATION_TYPES = "_ALL_CLASSIFICATION_TYPES";
/** /**
* @return The type of query * @return The type of query
......
...@@ -93,6 +93,8 @@ public class AtlasClassificationType extends AtlasStructType { ...@@ -93,6 +93,8 @@ public class AtlasClassificationType extends AtlasStructType {
public AtlasClassificationDef getClassificationDef() { return classificationDef; } public AtlasClassificationDef getClassificationDef() { return classificationDef; }
public static AtlasClassificationType getClassificationRoot() {return CLASSIFICATION_ROOT; }
@Override @Override
void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
super.resolveReferences(typeRegistry); super.resolveReferences(typeRegistry);
...@@ -284,6 +286,10 @@ public class AtlasClassificationType extends AtlasStructType { ...@@ -284,6 +286,10 @@ public class AtlasClassificationType extends AtlasStructType {
return StringUtils.isNotEmpty(classificationName) && allSuperTypes.contains(classificationName); return StringUtils.isNotEmpty(classificationName) && allSuperTypes.contains(classificationName);
} }
public boolean hasAttribute(String attrName) {
return allAttributes.containsKey(attrName);
}
/** /**
* List of all the entity type names that are valid for this classification type. * List of all the entity type names that are valid for this classification type.
* *
...@@ -506,7 +512,7 @@ public class AtlasClassificationType extends AtlasStructType { ...@@ -506,7 +512,7 @@ public class AtlasClassificationType extends AtlasStructType {
add(new AtlasAttributeDef(STATE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); add(new AtlasAttributeDef(STATE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
}}; }};
AtlasClassificationDef classificationDef = new AtlasClassificationDef(CLASSIFICATION_ROOT_NAME, "", "", attributeDefs); AtlasClassificationDef classificationDef = new AtlasClassificationDef(CLASSIFICATION_ROOT_NAME, "Root classification for system attributes", "1.0", attributeDefs);
return new AtlasClassificationType(classificationDef); return new AtlasClassificationType(classificationDef);
} }
......
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
*/ */
package org.apache.atlas.type; package org.apache.atlas.type;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_INT;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
import static org.apache.atlas.type.Constants.*; import static org.apache.atlas.type.Constants.*;
...@@ -638,7 +639,7 @@ public class AtlasEntityType extends AtlasStructType { ...@@ -638,7 +639,7 @@ public class AtlasEntityType extends AtlasStructType {
add(new AtlasAttributeDef(CLASSIFICATION_TEXT_KEY, ATLAS_TYPE_STRING, false, true)); add(new AtlasAttributeDef(CLASSIFICATION_TEXT_KEY, ATLAS_TYPE_STRING, false, true));
add(new AtlasAttributeDef(CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true)); add(new AtlasAttributeDef(CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
add(new AtlasAttributeDef(PROPAGATED_CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true)); add(new AtlasAttributeDef(PROPAGATED_CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
add(new AtlasAttributeDef(IS_INCOMPLETE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); add(new AtlasAttributeDef(IS_INCOMPLETE_PROPERTY_KEY, ATLAS_TYPE_INT, false, true));
add(new AtlasAttributeDef(LABELS_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); add(new AtlasAttributeDef(LABELS_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
add(new AtlasAttributeDef(CUSTOM_ATTRIBUTES_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); add(new AtlasAttributeDef(CUSTOM_ATTRIBUTES_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
}}; }};
......
...@@ -32,7 +32,6 @@ public final class Constants { ...@@ -32,7 +32,6 @@ public final class Constants {
public static final String TYPE_NAME_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typeName"); public static final String TYPE_NAME_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typeName");
public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state"); public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy"); public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
public static final String SUPER_TYPES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "superTypeNames");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy"); public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
public static final String TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "timestamp"); public static final String TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "timestamp");
public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp"); public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp");
...@@ -44,13 +43,10 @@ public final class Constants { ...@@ -44,13 +43,10 @@ public final class Constants {
public static final String HISTORICAL_GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids"); public static final String HISTORICAL_GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids");
public static final String LABELS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "labels"); public static final String LABELS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "labels");
public static final String CUSTOM_ATTRIBUTES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "customAttributes"); public static final String CUSTOM_ATTRIBUTES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "customAttributes");
public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText"); public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames"); public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames");
public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames"); public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames");
public static final String IS_INCOMPLETE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isIncomplete"); public static final String IS_INCOMPLETE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isIncomplete");
private Constants() {} private Constants() {}
} }
\ No newline at end of file
...@@ -164,11 +164,11 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -164,11 +164,11 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")"); indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll(""); indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
Predicate typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class); Predicate typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes); Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
if (attributePredicate != null) { if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(typeNamePredicate, attributePredicate); inMemoryPredicate = typeNamePredicate == null ? attributePredicate : PredicateUtils.andPredicate(typeNamePredicate, attributePredicate);
} else { } else {
inMemoryPredicate = typeNamePredicate; inMemoryPredicate = typeNamePredicate;
} }
......
...@@ -54,7 +54,6 @@ import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOpe ...@@ -54,7 +54,6 @@ import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOpe
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC; import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC;
import static org.apache.atlas.type.AtlasEntityType.ENTITY_ROOT;
public class EntitySearchProcessor extends SearchProcessor { public class EntitySearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class); private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class);
......
...@@ -45,8 +45,9 @@ import org.slf4j.LoggerFactory; ...@@ -45,8 +45,9 @@ import org.slf4j.LoggerFactory;
import java.util.*; import java.util.*;
import static org.apache.atlas.discovery.SearchProcessor.ALL_ENTITY_TYPE_QUERY; import static org.apache.atlas.discovery.SearchProcessor.ALL_TYPE_QUERY;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS; import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES; import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS; import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS; import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
...@@ -77,8 +78,10 @@ public class SearchContext { ...@@ -77,8 +78,10 @@ public class SearchContext {
public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS)); public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS)); public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS)); public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot();
public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot(); public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot();
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException { public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
this.classificationName = searchParameters.getClassification(); this.classificationName = searchParameters.getClassification();
this.searchParameters = searchParameters; this.searchParameters = searchParameters;
...@@ -116,23 +119,28 @@ public class SearchContext { ...@@ -116,23 +119,28 @@ public class SearchContext {
validateAttributes(classificationType, searchParameters.getTagFilters()); validateAttributes(classificationType, searchParameters.getTagFilters());
if (classificationType != null && !isBuiltInClassificationType()) { if (classificationType != null && !isBuiltInClassificationType()) {
classificationTypeAndSubTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName()); if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
classificationTypeAndSubTypesQryStr = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypesQryStr() : classificationType.getTypeQryStr(); classificationTypeAndSubTypes = Collections.singleton(ALL_TYPE_QUERY);
classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
} else {
classificationTypeAndSubTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName());
classificationTypeAndSubTypesQryStr = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypesQryStr() : classificationType.getTypeQryStr();
}
} else { } else {
classificationTypeAndSubTypes = Collections.emptySet(); classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = ""; classificationTypeAndSubTypesQryStr = "";
} }
if (entityType != null) { if (entityType != null) {
if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) { if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
typeAndSubTypes = Collections.singleton(ALL_ENTITY_TYPE_QUERY); typeAndSubTypes = Collections.singleton(ALL_TYPE_QUERY);
typeAndSubTypesQryStr = ALL_ENTITY_TYPE_QUERY; typeAndSubTypesQryStr = ALL_TYPE_QUERY;
} else { } else {
typeAndSubTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName()); typeAndSubTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
typeAndSubTypesQryStr = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypesQryStr() : entityType.getTypeQryStr(); typeAndSubTypesQryStr = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypesQryStr() : entityType.getTypeQryStr();
} }
} else { } else {
typeAndSubTypes = Collections.emptySet(); typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = ""; typeAndSubTypesQryStr = "";
} }
...@@ -304,6 +312,8 @@ public class SearchContext { ...@@ -304,6 +312,8 @@ public class SearchContext {
ret = MATCH_ALL_CLASSIFIED; ret = MATCH_ALL_CLASSIFIED;
} else if (StringUtils.equals(classificationName, MATCH_ALL_NOT_CLASSIFIED.getTypeName())) { } else if (StringUtils.equals(classificationName, MATCH_ALL_NOT_CLASSIFIED.getTypeName())) {
ret = MATCH_ALL_NOT_CLASSIFIED; ret = MATCH_ALL_NOT_CLASSIFIED;
} else if (StringUtils.equals(classificationName, ALL_CLASSIFICATION_TYPES)){
ret = MATCH_ALL_CLASSIFICATION_TYPES;
} else { } else {
ret = typeRegistry.getClassificationTypeByName(classificationName); ret = typeRegistry.getClassificationTypeByName(classificationName);
} }
......
...@@ -45,6 +45,10 @@ import java.math.BigInteger; ...@@ -45,6 +45,10 @@ import java.math.BigInteger;
import java.util.*; import java.util.*;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.util.SearchPredicateUtil.*; import static org.apache.atlas.util.SearchPredicateUtil.*;
public abstract class SearchProcessor { public abstract class SearchProcessor {
...@@ -62,7 +66,7 @@ public abstract class SearchProcessor { ...@@ -62,7 +66,7 @@ public abstract class SearchProcessor {
public static final String SPACE_STRING = " "; public static final String SPACE_STRING = " ";
public static final String BRACE_OPEN_STR = "("; public static final String BRACE_OPEN_STR = "(";
public static final String BRACE_CLOSE_STR = ")"; public static final String BRACE_CLOSE_STR = ")";
public static final String ALL_ENTITY_TYPE_QUERY = "[* TO *]"; public static final String ALL_TYPE_QUERY = "[* TO *]";
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>(); private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final Map<SearchParameters.Operator, VertexAttributePredicateGenerator> OPERATOR_PREDICATE_MAP = new HashMap<>(); private static final Map<SearchParameters.Operator, VertexAttributePredicateGenerator> OPERATOR_PREDICATE_MAP = new HashMap<>();
...@@ -136,6 +140,21 @@ public abstract class SearchProcessor { ...@@ -136,6 +140,21 @@ public abstract class SearchProcessor {
return context.getEntityType() == SearchContext.MATCH_ALL_ENTITY_TYPES; return context.getEntityType() == SearchContext.MATCH_ALL_ENTITY_TYPES;
} }
protected boolean isClassificationRootType() {
return context.getClassificationType() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
}
protected boolean isSystemAttribute(String attrName) {
return AtlasEntityType.getEntityRoot().hasAttribute(attrName) || AtlasClassificationType.getClassificationRoot().hasAttribute(attrName);
}
protected boolean isPipeSeparatedSystemAttribute(String attrName) {
return StringUtils.equals(attrName, CLASSIFICATION_NAMES_KEY) ||
StringUtils.equals(attrName, PROPAGATED_CLASSIFICATION_NAMES_KEY) ||
StringUtils.equals(attrName, LABELS_PROPERTY_KEY) ||
StringUtils.equals(attrName, CUSTOM_ATTRIBUTES_PROPERTY_KEY);
}
protected int collectResultVertices(final List<AtlasVertex> ret, final int startIdx, final int limit, int resultIdx, final List<AtlasVertex> entityVertices) { protected int collectResultVertices(final List<AtlasVertex> ret, final int startIdx, final int limit, int resultIdx, final List<AtlasVertex> entityVertices) {
for (AtlasVertex entityVertex : entityVertices) { for (AtlasVertex entityVertex : entityVertices) {
resultIdx++; resultIdx++;
...@@ -185,7 +204,7 @@ public abstract class SearchProcessor { ...@@ -185,7 +204,7 @@ public abstract class SearchProcessor {
graphFiltered.add(attributeName); graphFiltered.add(attributeName);
} }
if (structType instanceof AtlasEntityType) { if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) {
// Capture the entity attributes // Capture the entity attributes
context.getEntityAttributes().add(attributeName); context.getEntityAttributes().add(attributeName);
} }
...@@ -438,7 +457,7 @@ public abstract class SearchProcessor { ...@@ -438,7 +457,7 @@ public abstract class SearchProcessor {
return PredicateUtils.anyPredicate(predicates); return PredicateUtils.anyPredicate(predicates);
} }
} }
} else if (indexAttributes.contains(criteria.getAttributeName())){ } else if (indexAttributes.contains(criteria.getAttributeName()) && !isPipeSeparatedSystemAttribute(criteria.getAttributeName())){
return toInMemoryPredicate(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue()); return toInMemoryPredicate(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
} }
......
...@@ -46,6 +46,7 @@ import java.util.List; ...@@ -46,6 +46,7 @@ import java.util.List;
import java.util.Set; import java.util.Set;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES; import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer.getTypesToCreate; import static org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer.getTypesToCreate;
import static org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer.getTypesToUpdate; import static org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer.getTypesToUpdate;
...@@ -217,7 +218,11 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore { ...@@ -217,7 +218,11 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
AtlasClassificationDef ret = typeRegistry.getClassificationDefByName(name); AtlasClassificationDef ret = typeRegistry.getClassificationDefByName(name);
if (ret == null) { if (ret == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name); ret = StringUtils.equalsIgnoreCase(name, ALL_CLASSIFICATION_TYPES) ? AtlasClassificationType.getClassificationRoot().getClassificationDef() : null;
if (ret == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name);
}
} }
return ret; return ret;
......
...@@ -32,6 +32,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; ...@@ -32,6 +32,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.runner.LocalSolrRunner; import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasType;
import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.atlas.utils.TestResourceFileUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
...@@ -701,4 +702,12 @@ public class AtlasTypeDefGraphStoreTest { ...@@ -701,4 +702,12 @@ public class AtlasTypeDefGraphStoreTest {
assertNotNull(entityDefByName); assertNotNull(entityDefByName);
assertEquals(entityDefByName, AtlasEntityType.getEntityRoot().getEntityDef()); assertEquals(entityDefByName, AtlasEntityType.getEntityRoot().getEntityDef());
} }
}
@Test
public void testGetOnAllClassificationTypes() throws AtlasBaseException {
AtlasClassificationDef classificationTypeDef = typeDefStore.getClassificationDefByName("_ALL_CLASSIFICATION_TYPES");
assertNotNull(classificationTypeDef);
assertEquals(classificationTypeDef, AtlasClassificationType.getClassificationRoot().getClassificationDef());
}
}
\ No newline at end of file
...@@ -25,6 +25,7 @@ import static org.apache.atlas.TestUtilsV2.PHI; ...@@ -25,6 +25,7 @@ import static org.apache.atlas.TestUtilsV2.PHI;
import static org.apache.atlas.TestUtilsV2.PII; import static org.apache.atlas.TestUtilsV2.PII;
import static org.apache.atlas.TestUtilsV2.TABLE_TYPE; import static org.apache.atlas.TestUtilsV2.TABLE_TYPE;
import static org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition.AND; import static org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition.AND;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
...@@ -122,6 +123,17 @@ public class TestEntitiesREST { ...@@ -122,6 +123,17 @@ public class TestEntitiesREST {
} }
@Test @Test
public void testGetEntities() throws Exception {
final AtlasEntitiesWithExtInfo response = entityREST.getByGuids(createdGuids.get(DATABASE_TYPE), false, false);
final List<AtlasEntity> entities = response.getEntities();
Assert.assertNotNull(entities);
Assert.assertEquals(entities.size(), 1);
verifyAttributes(entities);
}
@Test
public void testBasicSearch() throws Exception { public void testBasicSearch() throws Exception {
// search entities with classification named classification // search entities with classification named classification
searchParameters = new SearchParameters(); searchParameters = new SearchParameters();
...@@ -133,7 +145,33 @@ public class TestEntitiesREST { ...@@ -133,7 +145,33 @@ public class TestEntitiesREST {
Assert.assertEquals(res.getEntities().size(), 2); Assert.assertEquals(res.getEntities().size(), 2);
} }
@Test(dependsOnMethods = "testBasicSearch") @Test
public void testSearchByMultiSystemAttributes() throws Exception {
searchParameters = new SearchParameters();
searchParameters.setTypeName("_ALL_ENTITY_TYPES");
SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
SearchParameters.FilterCriteria subFc1 = new SearchParameters.FilterCriteria();
SearchParameters.FilterCriteria subFc2 = new SearchParameters.FilterCriteria();
subFc1.setAttributeName(MODIFICATION_TIMESTAMP_PROPERTY_KEY);
subFc1.setOperator(SearchParameters.Operator.LT);
subFc1.setAttributeValue(String.valueOf(System.currentTimeMillis()));
subFc2.setAttributeName(TIMESTAMP_PROPERTY_KEY);
subFc2.setOperator(SearchParameters.Operator.LT);
subFc2.setAttributeValue(String.valueOf(System.currentTimeMillis()));
fc.setCriterion(Arrays.asList(subFc1, subFc2));
fc.setCondition(AND);
searchParameters.setEntityFilters(fc);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertTrue(res.getEntities().size() > 5);
}
@Test
public void testWildCardBasicSearch() throws Exception { public void testWildCardBasicSearch() throws Exception {
//table - classification //table - classification
...@@ -214,10 +252,11 @@ public class TestEntitiesREST { ...@@ -214,10 +252,11 @@ public class TestEntitiesREST {
Assert.assertNull(res.getEntities()); Assert.assertNull(res.getEntities());
} }
@Test(dependsOnMethods = "testWildCardBasicSearch") @Test(dependsOnMethods = "testBasicSearchWithAttr")
public void testBasicSearchWithSubTypes() throws Exception{ public void testBasicSearchWithSubTypes() throws Exception{
// basic search with subtypes // table - classification
// column - phi
searchParameters = new SearchParameters(); searchParameters = new SearchParameters();
searchParameters.setClassification(TestUtilsV2.CLASSIFICATION); searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
searchParameters.setIncludeSubClassifications(true); searchParameters.setIncludeSubClassifications(true);
...@@ -229,6 +268,7 @@ public class TestEntitiesREST { ...@@ -229,6 +268,7 @@ public class TestEntitiesREST {
// table - classification // table - classification
// database - fetl_classification // database - fetl_classification
// column - phi
addTagTo(FETL_CLASSIFICATION, DATABASE_TYPE); addTagTo(FETL_CLASSIFICATION, DATABASE_TYPE);
final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(DATABASE_TYPE).get(0), TestUtilsV2.FETL_CLASSIFICATION); final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(DATABASE_TYPE).get(0), TestUtilsV2.FETL_CLASSIFICATION);
...@@ -248,12 +288,12 @@ public class TestEntitiesREST { ...@@ -248,12 +288,12 @@ public class TestEntitiesREST {
Assert.assertEquals(res.getEntities().size(), 2); Assert.assertEquals(res.getEntities().size(), 2);
} }
@Test(dependsOnMethods = "testWildCardBasicSearch") @Test(dependsOnMethods = "testBasicSearchWithSubTypes")
public void testGraphQueryFilter() throws Exception { public void testGraphQueryFilter() throws Exception {
// database - pii, felt_classification // table - classification
// table - pii, classification, // database - fetl_classification
// col - phi // column - phi
searchParameters = new SearchParameters(); searchParameters = new SearchParameters();
searchParameters.setQuery("sample_string"); searchParameters.setQuery("sample_string");
searchParameters.setClassification(PHI); searchParameters.setClassification(PHI);
...@@ -282,6 +322,9 @@ public class TestEntitiesREST { ...@@ -282,6 +322,9 @@ public class TestEntitiesREST {
Assert.assertNotNull(result_tag); Assert.assertNotNull(result_tag);
Assert.assertEquals(result_tag.getTypeName(), PHI); Assert.assertEquals(result_tag.getTypeName(), PHI);
// table - phi, classification
// database - fetl_classification
// column - phi
fc.setAttributeValue("false"); fc.setAttributeValue("false");
res = discoveryREST.searchWithParameters(searchParameters); res = discoveryREST.searchWithParameters(searchParameters);
...@@ -290,10 +333,85 @@ public class TestEntitiesREST { ...@@ -290,10 +333,85 @@ public class TestEntitiesREST {
Assert.assertEquals(res.getEntities().get(0).getTypeName(), TABLE_TYPE); Assert.assertEquals(res.getEntities().get(0).getTypeName(), TABLE_TYPE);
} }
@Test(dependsOnMethods = "testBasicSearch") @Test(dependsOnMethods = "testGraphQueryFilter")
public void testSearchByMultiTags() throws Exception {
addTagTo(PHI, DATABASE_TYPE);
// database - phi, felt_classification
// table1 - phi, classification, table2 - classification,
// column - phi
searchParameters = new SearchParameters();
searchParameters.setIncludeSubClassifications(false);
searchParameters.setTypeName("_ALL_ENTITY_TYPES");
SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
SearchParameters.FilterCriteria subFc1 = new SearchParameters.FilterCriteria();
SearchParameters.FilterCriteria subFc2 = new SearchParameters.FilterCriteria();
subFc1.setAttributeName(CLASSIFICATION_NAMES_KEY);
subFc1.setOperator(SearchParameters.Operator.CONTAINS);
subFc1.setAttributeValue(PHI);
subFc2.setAttributeName(CLASSIFICATION_NAMES_KEY);
subFc2.setOperator(SearchParameters.Operator.CONTAINS);
subFc2.setAttributeValue(CLASSIFICATION);
fc.setCriterion(Arrays.asList(subFc1, subFc2));
fc.setCondition(AND);
searchParameters.setEntityFilters(fc);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
subFc2.setAttributeValue(FETL_CLASSIFICATION);
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
}
@Test(dependsOnMethods = "testSearchByMultiTags")
public void testSearchByOtherSystemAttributes() throws Exception {
// database - phi, felt_classification
// table1 - phi, classification, table2 - classification,
// col1 - phi, col2 - phi
searchParameters = new SearchParameters();
searchParameters.setTypeName("_ALL_ENTITY_TYPES");
SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
fc.setAttributeName(CLASSIFICATION_NAMES_KEY);
fc.setOperator(SearchParameters.Operator.EQ);
fc.setAttributeValue(CLASSIFICATION);
searchParameters.setEntityFilters(fc);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
fc.setOperator(SearchParameters.Operator.CONTAINS);
fc.setAttributeValue("cla");
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 3);
fc.setOperator(SearchParameters.Operator.CONTAINS);
fc.setAttributeValue(PHI);
res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 4);
}
@Test(dependsOnMethods = "testSearchByOtherSystemAttributes")
public void testBasicSearchWithFilter() throws Exception { public void testBasicSearchWithFilter() throws Exception {
//table - classification // database - phi, felt_classification
// table1 - phi, classification, table2 - classification,
// col - phi
searchParameters = new SearchParameters(); searchParameters = new SearchParameters();
searchParameters.setIncludeSubClassifications(false); searchParameters.setIncludeSubClassifications(false);
searchParameters.setClassification(TestUtilsV2.CLASSIFICATION); searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
...@@ -317,34 +435,12 @@ public class TestEntitiesREST { ...@@ -317,34 +435,12 @@ public class TestEntitiesREST {
Assert.assertNull(res.getEntities()); Assert.assertNull(res.getEntities());
} }
@Test @Test(dependsOnMethods = "testBasicSearchWithFilter")
public void testSearchByMultiSystemAttributes() throws Exception {
searchParameters = new SearchParameters();
searchParameters.setTypeName("_ALL_ENTITY_TYPES");
SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
SearchParameters.FilterCriteria subFc1 = new SearchParameters.FilterCriteria();
SearchParameters.FilterCriteria subFc2 = new SearchParameters.FilterCriteria();
subFc1.setAttributeName(MODIFICATION_TIMESTAMP_PROPERTY_KEY);
subFc1.setOperator(SearchParameters.Operator.LT);
subFc1.setAttributeValue(String.valueOf(System.currentTimeMillis()));
subFc2.setAttributeName(TIMESTAMP_PROPERTY_KEY);
subFc2.setOperator(SearchParameters.Operator.LT);
subFc2.setAttributeValue(String.valueOf(System.currentTimeMillis()));
fc.setCriterion(Arrays.asList(subFc1, subFc2));
fc.setCondition(AND);
searchParameters.setEntityFilters(fc);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertTrue(res.getEntities().size() > 5);
}
@Test
public void testSearchBySingleSystemAttribute() throws Exception { public void testSearchBySingleSystemAttribute() throws Exception {
// database - phi, felt_classification
// table1 - phi, classification, table2 - classification,
// col - phi
searchParameters = new SearchParameters(); searchParameters = new SearchParameters();
searchParameters.setTypeName("_ALL_ENTITY_TYPES"); searchParameters.setTypeName("_ALL_ENTITY_TYPES");
...@@ -381,8 +477,12 @@ public class TestEntitiesREST { ...@@ -381,8 +477,12 @@ public class TestEntitiesREST {
Assert.assertNull(res.getEntities()); Assert.assertNull(res.getEntities());
} }
@Test @Test(dependsOnMethods = "testSearchBySingleSystemAttribute")
public void testSearchBySystemAttributesWithQuery() throws Exception { public void testSearchBySystemAttributesWithQuery() throws Exception {
// database - phi, felt_classification
// table1 - phi, classification, table2 - classification,
// col - phi
searchParameters = new SearchParameters(); searchParameters = new SearchParameters();
searchParameters.setTypeName("_ALL_ENTITY_TYPES"); searchParameters.setTypeName("_ALL_ENTITY_TYPES");
SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria(); SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
...@@ -404,10 +504,42 @@ public class TestEntitiesREST { ...@@ -404,10 +504,42 @@ public class TestEntitiesREST {
searchParameters.setQuery("sample_string"); searchParameters.setQuery("sample_string");
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters); AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNull(res.getEntities()); Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
} }
@Test(dependsOnMethods = "testBasicSearchWithSubTypes") @Test(dependsOnMethods = "testSearchBySystemAttributesWithQuery")
public void testTagSearchBySystemAttributes() throws Exception {
// database - phi, felt_classification
// table1 - phi, classification, table2 - classification,
// col - phi
searchParameters = new SearchParameters();
searchParameters.setClassification("_ALL_CLASSIFICATION_TYPES");
SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
SearchParameters.FilterCriteria subFc1 = new SearchParameters.FilterCriteria();
SearchParameters.FilterCriteria subFc2 = new SearchParameters.FilterCriteria();
subFc1.setAttributeName(TIMESTAMP_PROPERTY_KEY);
subFc1.setOperator(SearchParameters.Operator.LT);
subFc1.setAttributeValue(String.valueOf(System.currentTimeMillis()));
subFc2.setAttributeName(STATE_PROPERTY_KEY);
subFc2.setOperator(SearchParameters.Operator.EQ);
subFc2.setAttributeValue("ACTIVE");
fc.setCriterion(Arrays.asList(subFc1, subFc2));
fc.setCondition(AND);
searchParameters.setTagFilters(fc);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 5);
}
@Test(dependsOnMethods = "testTagSearchBySystemAttributes")
public void testUpdateWithSerializedEntities() throws Exception { public void testUpdateWithSerializedEntities() throws Exception {
//Check with serialization and deserialization of entity attributes for the case //Check with serialization and deserialization of entity attributes for the case
...@@ -436,17 +568,6 @@ public class TestEntitiesREST { ...@@ -436,17 +568,6 @@ public class TestEntitiesREST {
Assert.assertEquals(newGuids.size(), 3); Assert.assertEquals(newGuids.size(), 3);
} }
@Test
public void testGetEntities() throws Exception {
final AtlasEntitiesWithExtInfo response = entityREST.getByGuids(createdGuids.get(DATABASE_TYPE), false, false);
final List<AtlasEntity> entities = response.getEntities();
Assert.assertNotNull(entities);
Assert.assertEquals(entities.size(), 1);
verifyAttributes(entities);
}
/* Disabled until EntityREST.deleteByIds() is implemented /* Disabled until EntityREST.deleteByIds() is implemented
* *
@Test(dependsOnMethods = "testGetEntities") @Test(dependsOnMethods = "testGetEntities")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment