Skip to content

A
atlas
Commit 2bbbd1a5 by Sarath Subramanian Committed by Madhan Neethiraj
Options

ATLAS-1630: basic search implementation using fulltext with support for filter…

ATLAS-1630: basic search implementation using fulltext with support for filter by type & classification
parent 98f02ff7
Showing with 189 additions and 27 deletions
intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
......@@ -42,8 +42,10 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
public class AtlasSearchResult implements Serializable {
private String queryText;
private AtlasQueryType queryType;
private String queryText;
private String type;
private String classification;
private List<AtlasEntityHeader> entities;
private AttributeSearchResult attributes;
private List<AtlasFullTextResult> fullTextResult;
......@@ -58,13 +60,21 @@ public class AtlasSearchResult implements Serializable {
setFullTextResult(null);
}
public AtlasQueryType getQueryType() { return queryType; }
public void setQueryType(AtlasQueryType queryType) { this.queryType = queryType; }
public String getQueryText() { return queryText; }
public void setQueryText(String queryText) { this.queryText = queryText; }
public AtlasQueryType getQueryType() { return queryType; }
public String getType() { return type; }
public void setQueryType(AtlasQueryType queryType) { this.queryType = queryType; }
public void setType(String type) { this.type = type; }
public String getClassification() { return classification; }
public void setClassification(String classification) { this.classification = classification; }
public List<AtlasEntityHeader> getEntities() { return entities; }
......@@ -83,21 +93,25 @@ public class AtlasSearchResult implements Serializable {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AtlasSearchResult that = (AtlasSearchResult) o;
return Objects.equals(queryText, that.queryText) &&
Objects.equals(queryType, that.queryType) &&
return Objects.equals(queryType, that.queryType) &&
Objects.equals(queryText, that.queryText) &&
Objects.equals(type, that.type) &&
Objects.equals(classification, that.classification) &&
Objects.equals(entities, that.entities) &&
Objects.equals(attributes, that.attributes) &&
Objects.equals(fullTextResult, that.fullTextResult);
}
@Override
public int hashCode() { return Objects.hash(queryText, queryType, entities, attributes, fullTextResult); }
public int hashCode() { return Objects.hash(queryText, queryType, entities, attributes, fullTextResult, type, classification); }
@Override
public String toString() {
return "AtlasSearchResult{" +
"queryText='" + queryText + '\'' +
", queryType=" + queryType +
"queryType=" + queryType +
", queryText='" + queryText + '\'' +
", type=" + type +
", classification=" + classification +
", entities=" + entities +
", attributes=" + attributes +
", fullTextResult=" + fullTextResult +
......@@ -131,7 +145,7 @@ public class AtlasSearchResult implements Serializable {
}
}
public enum AtlasQueryType { DSL, FULL_TEXT, GREMLIN }
public enum AtlasQueryType { DSL, FULL_TEXT, GREMLIN, BASIC }
@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
......
intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
......@@ -45,9 +45,10 @@ public class AtlasClassificationType extends AtlasStructType {
private final AtlasClassificationDef classificationDef;
private List<AtlasClassificationType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> allSubTypes = Collections.emptySet();
private List<AtlasClassificationType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> allSubTypes = Collections.emptySet();
private Set<String> typeAndAllSubTypes = Collections.emptySet();
public AtlasClassificationType(AtlasClassificationDef classificationDef) {
super(classificationDef);
......@@ -87,11 +88,14 @@ public class AtlasClassificationType extends AtlasStructType {
}
}
this.superTypes = Collections.unmodifiableList(s);
this.allSuperTypes = Collections.unmodifiableSet(allS);
this.allAttributes = Collections.unmodifiableMap(allA);
this.uniqAttributes = getUniqueAttributes(this.allAttributes);
this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.superTypes = Collections.unmodifiableList(s);
this.allSuperTypes = Collections.unmodifiableSet(allS);
this.allAttributes = Collections.unmodifiableMap(allA);
this.uniqAttributes = getUniqueAttributes(this.allAttributes);
this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.typeAndAllSubTypes.add(this.getTypeName());
}
@Override
......@@ -106,6 +110,7 @@ public class AtlasClassificationType extends AtlasStructType {
private void addSubType(AtlasClassificationType subType) {
allSubTypes.add(subType.getTypeName());
typeAndAllSubTypes.add(subType.getTypeName());
}
public Set<String> getSuperTypes() {
......@@ -115,9 +120,11 @@ public class AtlasClassificationType extends AtlasStructType {
public Set<String> getAllSuperTypes() { return allSuperTypes; }
public Set<String> getAllSubTypes() {
return allSubTypes;
return Collections.unmodifiableSet(allSubTypes);
}
public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); }
public boolean isSuperTypeOf(AtlasClassificationType classificationType) {
return classificationType != null && allSubTypes.contains(classificationType.getTypeName());
}
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -47,9 +47,10 @@ public class AtlasEntityType extends AtlasStructType {
private final AtlasEntityDef entityDef;
private List<AtlasEntityType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> allSubTypes = Collections.emptySet();
private List<AtlasEntityType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> allSubTypes = Collections.emptySet();
private Set<String> typeAndAllSubTypes = Collections.emptySet();
public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef);
......@@ -87,11 +88,14 @@ public class AtlasEntityType extends AtlasStructType {
}
}
this.superTypes = Collections.unmodifiableList(s);
this.allSuperTypes = Collections.unmodifiableSet(allS);
this.allAttributes = Collections.unmodifiableMap(allA);
this.uniqAttributes = getUniqueAttributes(this.allAttributes);
this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.superTypes = Collections.unmodifiableList(s);
this.allSuperTypes = Collections.unmodifiableSet(allS);
this.allAttributes = Collections.unmodifiableMap(allA);
this.uniqAttributes = getUniqueAttributes(this.allAttributes);
this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.typeAndAllSubTypes.add(this.getTypeName());
}
@Override
......@@ -114,6 +118,8 @@ public class AtlasEntityType extends AtlasStructType {
public Set<String> getAllSubTypes() { return Collections.unmodifiableSet(allSubTypes); }
public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); }
public boolean isSuperTypeOf(AtlasEntityType entityType) {
return entityType != null && allSubTypes.contains(entityType.getTypeName());
}
......@@ -316,6 +322,7 @@ public class AtlasEntityType extends AtlasStructType {
private void addSubType(AtlasEntityType subType) {
allSubTypes.add(subType.getTypeName());
typeAndAllSubTypes.add(subType.getTypeName());
}
private void getTypeHierarchyInfo(AtlasTypeRegistry typeRegistry,
......
repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
......@@ -40,4 +40,15 @@ public interface AtlasDiscoveryService {
* @return AtlasSearchResult
*/
AtlasSearchResult searchUsingFullTextQuery(String query, int limit, int offset) throws AtlasBaseException;
/**
*
* @param query search query.
* @param type entity type.
* @param classification classification name.
* @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property.
* @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0.
* @return AtlasSearchResult
*/
AtlasSearchResult searchUsingBasicQuery(String query, String type, String classification, int limit, int offset) throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
......@@ -42,9 +42,14 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Option;
......@@ -59,6 +64,8 @@ import java.util.List;
import java.util.Map;
import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
import static org.apache.atlas.AtlasErrorCode.UNKNOWN_TYPENAME;
import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND;
public class EntityDiscoveryService implements AtlasDiscoveryService {
private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class);
......@@ -66,12 +73,16 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private final AtlasGraph graph;
private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
private final EntityGraphRetriever entityRetriever;
private final AtlasGremlinQueryProvider gremlinQueryProvider;
private final AtlasTypeRegistry typeRegistry;
@Inject
EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry) {
this.graph = AtlasGraphProvider.getGraphInstance();
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
this.typeRegistry = typeRegistry;
}
@Override
......@@ -145,6 +156,75 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return ret;
}
@Override
public AtlasSearchResult searchUsingBasicQuery(String query, String typeName, String classification, int limit, int offset) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(query, AtlasQueryType.BASIC);
if (LOG.isDebugEnabled()) {
LOG.debug("Executing basic search query: {} with type: {} and classification: {}", query, typeName, classification);
}
QueryParams params = validateSearchParams(limit, offset);
String basicQuery = "g.V()";
if (StringUtils.isNotEmpty(typeName)) {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType == null) {
throw new AtlasBaseException(UNKNOWN_TYPENAME, typeName);
}
String typeFilterExpr = gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER);
basicQuery += String.format(typeFilterExpr,
StringUtils.join(entityType.getTypeAndAllSubTypes(), "','"));
ret.setType(typeName);
}
if (StringUtils.isNotEmpty(classification)) {
AtlasClassificationType classificationType = typeRegistry.getClassificationTypeByName(classification);
if (classificationType == null) {
throw new AtlasBaseException(CLASSIFICATION_NOT_FOUND, classification);
}
String classificationFilterExpr = gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER);
basicQuery += String.format(classificationFilterExpr,
StringUtils.join(classificationType.getTypeAndAllSubTypes(), "','"));
ret.setClassification(classification);
}
basicQuery += String.format(gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_QUERY_FILTER), query);
basicQuery += String.format(gremlinQueryProvider.getQuery(AtlasGremlinQuery.TO_RANGE_LIST), params.offset(), params.limit());
try {
Object result = graph.executeGremlinScript(basicQuery, false);
if (result instanceof List && CollectionUtils.isNotEmpty((List) result)) {
List queryResult = (List) result;
Object firstElement = queryResult.get(0);
if (firstElement instanceof AtlasVertex) {
for (Object element : queryResult) {
if (element instanceof AtlasVertex) {
ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex) element));
} else {
LOG.warn("searchUsingBasicQuery({}): expected an AtlasVertex; found unexpected entry in result {}", basicQuery, element);
}
}
}
}
} catch (ScriptException e) {
throw new AtlasBaseException(DISCOVERY_QUERY_FAILED, basicQuery);
}
return ret;
}
private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams params) throws AtlasBaseException {
List<AtlasFullTextResult> ret = new ArrayList<>();
Iterator<Result> iter = query.vertices();
......
repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
......@@ -64,6 +64,15 @@ public class AtlasGremlin2QueryProvider extends AtlasGremlinQueryProvider {
"loop('src', {it.loops <= %s}, {((it.object.'__superTypeNames') ? " +
"(it.object.'__superTypeNames'.contains('DataSet')) : false)})." +
"path().toList()";
case BASIC_SEARCH_QUERY_FILTER:
return ".has('entityText', com.thinkaurelius.titan.core.attribute.Text.CONTAINS, '%s')";
case BASIC_SEARCH_TYPE_FILTER:
return ".has('__typeName', T.in, ['%s'])";
case BASIC_SEARCH_CLASSIFICATION_FILTER:
return ".has('__traitNames', T.in, ['%s'])";
case TO_RANGE_LIST:
return " [%s..<%s].toList()";
}
// Should never reach this point
return null;
......
repository/src/main/java/org/apache/atlas/util/AtlasGremlinQueryProvider.java
......@@ -53,7 +53,12 @@ public abstract class AtlasGremlinQueryProvider {
// Lineage Queries
FULL_LINEAGE,
PARTIAL_LINEAGE;
PARTIAL_LINEAGE,
// Discovery Queries
BASIC_SEARCH_QUERY_FILTER,
BASIC_SEARCH_TYPE_FILTER,
BASIC_SEARCH_CLASSIFICATION_FILTER,
TO_RANGE_LIST
}
}
webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
......@@ -90,4 +90,32 @@ public class DiscoveryREST {
return ret;
}
/**
* Retrieve data for the specified fulltext query
* @param query Fulltext query
* @param type limit the result to only entities of specified type or its sub-types
* @param classification limit the result to only entities tagged with the given classification or or its sub-types
* @param limit limit the result set to only include the specified number of entries
* @param offset start offset of the result set (useful for pagination)
* @return Search results
* @throws AtlasBaseException
* @HTTP 200 On successful FullText lookup with some results, might return an empty list if execution succeeded
* without any results
* @HTTP 400 Invalid fulltext or query parameters
*/
@GET
@Path("/basic")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasSearchResult searchUsingBasic(@QueryParam("query") String query,
@QueryParam("type") String type,
@QueryParam("classification") String classification,
@QueryParam("limit") int limit,
@QueryParam("offset") int offset) throws AtlasBaseException {
AtlasSearchResult ret = atlasDiscoveryService.searchUsingBasicQuery(query, type, classification, limit, offset);
return ret;
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment