Skip to content

A
atlas
Commit f6e27b59 by Suma Shivaprasad
Options

ATLAS-1345 Enhance search APIs to resolve hierarchical references (apoorvnaik via sumasai)

parent efc8d09c
Showing with 150 additions and 19 deletions
intg/src/main/java/org/apache/atlas/model/SearchFilter.java
......@@ -41,6 +41,7 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
@XmlAccessorType(XmlAccessType.PROPERTY)
public class SearchFilter {
public static final String PARAM_TYPE = "type";
public static final String PARAM_NAME = "name";
public static final String PARAM_SUPERTYPE = "supertype";
public static final String PARAM_NOT_SUPERTYPE = "notsupertype";
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1345 Enhance search APIs to resolve hierarchical references (apoorvnaik via sumasai)
ATLAS-1287 Subtasks: ATLAS-1288/ATLAS-1289 Integrated V2 API for Lineage,Entity Details,Tag assign to entity,Tags listing,tag create (kevalbhatt)
ATLAS-1303 Update hashCode and equals method to use standard JDK libraries (apoorvnaik via svimal2106)
ATLAS-1364 HiveHook : Fix Auth issue with doAs (sumasai)
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
......@@ -38,10 +38,14 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.util.FilterUtil;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeRegistry.AtlasTransientTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.Transformer;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -836,7 +840,7 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
@Override
@GraphTransaction
public AtlasTypesDef searchTypesDef(SearchFilter searchFilter) throws AtlasBaseException {
AtlasTypesDef typesDef = new AtlasTypesDef();
final AtlasTypesDef typesDef = new AtlasTypesDef();
Predicate searchPredicates = FilterUtil.getPredicateFromSearchFilter(searchFilter);
try {
List<AtlasEnumDef> enumDefs = getEnumDefStore(typeRegistry).getAll();
......@@ -848,24 +852,67 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
try {
List<AtlasStructDef> structDefs = getStructDefStore(typeRegistry).getAll();
CollectionUtils.filter(structDefs, searchPredicates);
typesDef.setStructDefs(structDefs);
Collection typeCollection = CollectionUtils.collect(structDefs, new Transformer() {
@Override
public Object transform(Object o) {
try {
return new AtlasStructType((AtlasStructDef) o, typeRegistry);
} catch (AtlasBaseException e) {
LOG.warn("Type validation failed for {}", ((AtlasStructDef) o).getName(), e);
return null;
}
}
});
CollectionUtils.filter(typeCollection, searchPredicates);
for (Object o : typeCollection) {
if (o != null)
typesDef.getStructDefs().add(((AtlasStructType)o).getStructDef());
}
} catch (AtlasBaseException ex) {
LOG.error("Failed to retrieve the StructDefs", ex);
}
try {
List<AtlasClassificationDef> classificationDefs = getClassificationDefStore(typeRegistry).getAll();
CollectionUtils.filter(classificationDefs, searchPredicates);
typesDef.setClassificationDefs(classificationDefs);
Collection typeCollection = CollectionUtils.collect(classificationDefs, new Transformer() {
@Override
public Object transform(Object o) {
try {
return new AtlasClassificationType((AtlasClassificationDef) o, typeRegistry);
} catch (AtlasBaseException e) {
LOG.warn("Type validation failed for {}", ((AtlasClassificationDef) o).getName(), e);
return null;
}
}
});
CollectionUtils.filter(typeCollection, searchPredicates);
for (Object o : typeCollection) {
if (o != null)
typesDef.getClassificationDefs().add(((AtlasClassificationType)o).getClassificationDef());
}
} catch (AtlasBaseException ex) {
LOG.error("Failed to retrieve the ClassificationDefs", ex);
}
try {
List<AtlasEntityDef> entityDefs = getEntityDefStore(typeRegistry).getAll();
CollectionUtils.filter(entityDefs, searchPredicates);
typesDef.setEntityDefs(entityDefs);
Collection typeCollection = CollectionUtils.collect(entityDefs, new Transformer() {
@Override
public Object transform(Object o) {
try {
return new AtlasEntityType((AtlasEntityDef) o, typeRegistry);
} catch (AtlasBaseException e) {
LOG.warn("Type validation failed for {}", ((AtlasEntityDef) o).getName(), e);
return null;
}
}
});
CollectionUtils.filter(typeCollection, searchPredicates);
for (Object o : typeCollection) {
if (o != null)
typesDef.getEntityDefs().add(((AtlasEntityType)o).getEntityDef());
}
} catch (AtlasBaseException ex) {
LOG.error("Failed to retrieve the EntityDefs", ex);
}
......
repository/src/main/java/org/apache/atlas/repository/util/FilterUtil.java
......@@ -22,8 +22,9 @@ import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.collections.functors.NotPredicate;
......@@ -31,11 +32,13 @@ import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class FilterUtil {
public static Predicate getPredicateFromSearchFilter(SearchFilter searchFilter) {
List<Predicate> predicates = new ArrayList<>();
final String type = searchFilter.getParam(SearchFilter.PARAM_TYPE);
final String name = searchFilter.getParam(SearchFilter.PARAM_NAME);
final String supertype = searchFilter.getParam(SearchFilter.PARAM_SUPERTYPE);
final String notSupertype = searchFilter.getParam(SearchFilter.PARAM_NOT_SUPERTYPE);
......@@ -44,6 +47,11 @@ public class FilterUtil {
predicates.add(getTypePredicate(type));
}
// Add filter for the name
if (StringUtils.isNotBlank(name)) {
predicates.add(getNamePredicate(name));
}
// Add filter for the supertype
if (StringUtils.isNotBlank(supertype)) {
predicates.add(getSuperTypePredicate(supertype));
......@@ -57,22 +65,49 @@ public class FilterUtil {
return PredicateUtils.allPredicate(predicates);
}
private static Predicate getNamePredicate(final String name) {
return new Predicate() {
private boolean isAtlasType(Object o) {
return o instanceof AtlasType;
}
private boolean isAtlasTypeDef(Object o) {
return o instanceof AtlasBaseTypeDef;
}
@Override
public boolean evaluate(Object o) {
return o != null &&
(isAtlasType(o) && Objects.equals(((AtlasType) o).getTypeName(), name)) ||
(isAtlasTypeDef(o) && Objects.equals(((AtlasBaseTypeDef) o).getName(), name));
}
};
}
private static Predicate getSuperTypePredicate(final String supertype) {
return new Predicate() {
private boolean isClassificationDef(Object o) {
private boolean isClassificationTypeDef(Object o) {
return o instanceof AtlasClassificationDef;
}
private boolean isEntityDef(Object o) {
private boolean isClassificationType(Object o) {
return o instanceof AtlasClassificationType;
}
private boolean isEntityTypeDef(Object o) {
return o instanceof AtlasEntityDef;
}
private boolean isEntityType(Object o) {
return o instanceof AtlasEntityType;
}
@Override
public boolean evaluate(Object o) {
return (isClassificationDef(o) &&
((AtlasClassificationDef) o).getSuperTypes().contains(supertype))||
(isEntityDef(o) &&
((AtlasEntityDef)o).getSuperTypes().contains(supertype));
return (isClassificationType(o) && ((AtlasClassificationType) o).getAllSuperTypes().contains(supertype))||
(isClassificationTypeDef(o) && ((AtlasClassificationDef)o).getSuperTypes().contains(supertype)) ||
(isEntityType(o) && ((AtlasEntityType)o).getAllSuperTypes().contains(supertype)) ||
(isEntityTypeDef(o) && ((AtlasEntityDef)o).getSuperTypes().contains(supertype));
}
};
}
......@@ -81,7 +116,25 @@ public class FilterUtil {
return new Predicate() {
@Override
public boolean evaluate(Object o) {
if (o instanceof AtlasBaseTypeDef) {
if (o instanceof AtlasType) {
AtlasType atlasType = (AtlasType)o;
switch (type.toUpperCase()) {
case "CLASS":
case "ENTITY":
return atlasType.getTypeCategory() == TypeCategory.ENTITY;
case "TRAIT":
case "CLASSIFICATION":
return atlasType.getTypeCategory() == TypeCategory.CLASSIFICATION;
case "STRUCT":
return atlasType.getTypeCategory() == TypeCategory.STRUCT;
case "ENUM":
return atlasType.getTypeCategory() == TypeCategory.ENUM;
default:
// This shouldn't have happened
return false;
}
} else if (o instanceof AtlasBaseTypeDef){
AtlasBaseTypeDef typeDef = (AtlasBaseTypeDef)o;
switch (type.toUpperCase()) {
......
repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
......@@ -165,12 +165,26 @@ public class AtlasTypeDefGraphStoreTest {
@DataProvider
public Object[][] validUpdateDeptTypes(){
AtlasTypesDef typesDef = TestUtilsV2.defineValidUpdatedDeptEmployeeTypes();
return new Object[][] {
{TestUtilsV2.defineValidUpdatedDeptEmployeeTypes()}
{typesDef}
};
}
@DataProvider
public Object[][] allCreatedTypes(){
// Capture all the types that are getting created or updated here.
AtlasTypesDef updatedTypeDefs = TestUtilsV2.defineValidUpdatedDeptEmployeeTypes();
AtlasTypesDef allTypeDefs = new AtlasTypesDef();
allTypeDefs.getEnumDefs().addAll(updatedTypeDefs.getEnumDefs());
allTypeDefs.getStructDefs().addAll(updatedTypeDefs.getStructDefs());
allTypeDefs.getClassificationDefs().addAll(updatedTypeDefs.getClassificationDefs());
allTypeDefs.getEntityDefs().addAll(updatedTypeDefs.getEntityDefs());
allTypeDefs.getEntityDefs().addAll(TestUtilsV2.getEntityWithValidSuperType());
return new Object[][] {{allTypeDefs}};
}
@DataProvider
public Object[][] invalidCreateTypes(){
// TODO: Create invalid type in TestUtilsV2
return new Object[][] {
......@@ -309,7 +323,7 @@ public class AtlasTypeDefGraphStoreTest {
}
// This should run after all the update calls
@Test(dependsOnMethods = {"testUpdate"}, dataProvider = "validUpdateDeptTypes")
@Test(dependsOnMethods = {"testUpdate"}, dataProvider = "allCreatedTypes")
public void testDelete(AtlasTypesDef atlasTypesDef){
try {
typeDefStore.deleteTypesDef(atlasTypesDef);
......@@ -387,4 +401,19 @@ public class AtlasTypeDefGraphStoreTest {
}
@Test(dependsOnMethods = "testGet")
public void testSearchFunctionality() {
SearchFilter searchFilter = new SearchFilter();
searchFilter.setParam(SearchFilter.PARAM_SUPERTYPE, "Person");
try {
AtlasTypesDef typesDef = typeDefStore.searchTypesDef(searchFilter);
assertNotNull(typesDef);
assertNotNull(typesDef.getEntityDefs());
assertEquals(typesDef.getEntityDefs().size(), 3);
} catch (AtlasBaseException e) {
fail("Search should've succeeded", e);
}
}
}
\ No newline at end of file
webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java
......@@ -57,7 +57,6 @@ import java.util.Set;
@Path("v2/types")
@Singleton
public class TypesREST {
private static final Logger LOG = LoggerFactory.getLogger(TypesREST.class);
private AtlasTypeDefStore typeDefStore;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment