Skip to content

A
atlas
Commit c31f1440 by Bolke de Bruin Committed by Madhan Neethiraj
Options

ATLAS-3399: added support for order-by in basic-search

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 1c781deb
Showing with 190 additions and 12 deletions
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
......@@ -18,6 +18,8 @@
package org.apache.atlas.repository.graphdb;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import java.util.Iterator;
/**
......@@ -36,6 +38,14 @@ public interface AtlasIndexQuery<V, E> {
Iterator<Result<V, E>> vertices();
/**
* Gets the sorted query results
* @param offset starting offset
* @param limit max number of results
* @return
*/
Iterator<Result<V, E>> vertices(int offset, int limit, String sortBy, Order sortOrder);
/**
* Gets the query results
* @param offset starting offset
* @param limit max number of results
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java
......@@ -25,6 +25,7 @@ import org.apache.atlas.repository.graphdb.AtlasVertex;
import com.google.common.base.Function;
import com.google.common.collect.Iterators;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.janusgraph.core.JanusGraphIndexQuery;
import org.janusgraph.core.JanusGraphVertex;
......@@ -78,6 +79,29 @@ public class AtlasJanusIndexQuery implements AtlasIndexQuery<AtlasJanusVertex, A
}
@Override
public Iterator<Result<AtlasJanusVertex, AtlasJanusEdge>> vertices(int offset, int limit, String sortBy, Order sortOrder) {
Preconditions.checkArgument(offset >=0, "Index offset should be greater than or equals to 0");
Preconditions.checkArgument(limit >=0, "Index limit should be greater than or equals to 0");
Iterator<JanusGraphIndexQuery.Result<JanusGraphVertex>> results = query
.orderBy(sortBy, sortOrder)
.offset(offset)
.limit(limit)
.vertices().iterator();
Function<JanusGraphIndexQuery.Result<JanusGraphVertex>, Result<AtlasJanusVertex, AtlasJanusEdge>> function =
new Function<JanusGraphIndexQuery.Result<JanusGraphVertex>, Result<AtlasJanusVertex, AtlasJanusEdge>>() {
@Override
public Result<AtlasJanusVertex, AtlasJanusEdge> apply(JanusGraphIndexQuery.Result<JanusGraphVertex> source) {
return new ResultImpl(source);
}
};
return Iterators.transform(results, function);
}
@Override
public Long vertexTotals() {
return query.vertexTotals();
}
......
graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
......@@ -609,12 +609,7 @@ public class Solr6Index implements IndexProvider {
final String queryFilter = buildQueryFilter(query.getCondition(), information.get(collection));
solrQuery.addFilterQuery(queryFilter);
if (!query.getOrder().isEmpty()) {
final List<IndexQuery.OrderEntry> orders = query.getOrder();
for (final IndexQuery.OrderEntry order1 : orders) {
final String item = order1.getKey();
final SolrQuery.ORDER order = order1.getOrder() == Order.ASC ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc;
solrQuery.addSort(new SolrQuery.SortClause(item, order));
}
addOrderToQuery(solrQuery, query.getOrder());
}
solrQuery.setStart(0);
if (query.hasLimit()) {
......@@ -626,6 +621,14 @@ public class Solr6Index implements IndexProvider {
doc -> doc.getFieldValue(keyIdField).toString());
}
private void addOrderToQuery(SolrQuery solrQuery, List<IndexQuery.OrderEntry> orders) {
for (final IndexQuery.OrderEntry order1 : orders) {
final String item = order1.getKey();
final SolrQuery.ORDER order = order1.getOrder() == Order.ASC ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc;
solrQuery.addSort(new SolrQuery.SortClause(item, order));
}
}
private <E> Stream<E> executeQuery(Integer limit, int offset, String collection, SolrQuery solrQuery,
Function<SolrDocument, E> function) throws PermanentBackendException {
try {
......@@ -654,6 +657,9 @@ public class Solr6Index implements IndexProvider {
} else {
solrQuery.setRows(batchSize);
}
if (!query.getOrders().isEmpty()) {
addOrderToQuery(solrQuery, query.getOrders());
}
for(final Parameter parameter: query.getParameters()) {
if (parameter.value() instanceof String[]) {
......
intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
......@@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonValue;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.atlas.SortOrder;
import java.io.Serializable;
import java.util.HashMap;
......@@ -44,6 +45,7 @@ public class SearchParameters implements Serializable {
private String typeName;
private String classification;
private String termName;
private String sortBy;
private boolean excludeDeletedEntities;
private boolean includeClassificationAttributes;
private boolean includeSubTypes = true;
......@@ -54,6 +56,7 @@ public class SearchParameters implements Serializable {
private FilterCriteria entityFilters;
private FilterCriteria tagFilters;
private Set<String> attributes;
private SortOrder sortOrder;
public static final String WILDCARD_CLASSIFICATIONS = "*";
public static final String ALL_CLASSIFICATIONS = "_CLASSIFIED";
......@@ -258,6 +261,30 @@ public class SearchParameters implements Serializable {
this.attributes = attributes;
}
/**
* @return Attribute on which to sort the results
*/
public String getSortBy() { return sortBy; }
/**
* Sort the results based on sortBy attribute
* @param sortBy Attribute on which to sort the results
*/
public void setSortBy(String sortBy) { this.sortBy = sortBy; }
/**
* @return Sorting order of the results
*/
public SortOrder getSortOrder() {
return sortOrder;
}
/**
* Sorting order to sort the results
* @param sortOrder ASCENDING vs DESCENDING
*/
public void setSortOrder(SortOrder sortOrder) { this.sortOrder = sortOrder; }
@Override
public boolean equals(Object o) {
if (this == o) return true;
......@@ -273,13 +300,15 @@ public class SearchParameters implements Serializable {
Objects.equals(termName, that.termName) &&
Objects.equals(entityFilters, that.entityFilters) &&
Objects.equals(tagFilters, that.tagFilters) &&
Objects.equals(attributes, that.attributes);
Objects.equals(attributes, that.attributes) &&
Objects.equals(sortBy, that.sortBy) &&
Objects.equals(sortOrder, that.sortOrder);
}
@Override
public int hashCode() {
return Objects.hash(query, typeName, classification, termName, excludeDeletedEntities, includeClassificationAttributes,
limit, offset, entityFilters, tagFilters, attributes);
limit, offset, entityFilters, tagFilters, attributes, sortBy, sortOrder);
}
public StringBuilder toString(StringBuilder sb) {
......@@ -299,6 +328,8 @@ public class SearchParameters implements Serializable {
sb.append(", entityFilters=").append(entityFilters);
sb.append(", tagFilters=").append(tagFilters);
sb.append(", attributes=").append(attributes);
sb.append(", sortBy=").append(sortBy).append('\'');
sb.append(", sortOrder=").append(sortOrder).append('\'');
sb.append('}');
return sb;
......@@ -461,5 +492,6 @@ public class SearchParameters implements Serializable {
public String toString() {
return getSymbol();
}
}
}
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
......@@ -17,6 +17,7 @@
*/
package org.apache.atlas.discovery;
import org.apache.atlas.SortOrder;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.instance.AtlasEntity;
......@@ -58,6 +59,8 @@ import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPE
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC;
public class ClassificationSearchProcessor extends SearchProcessor {
......@@ -72,7 +75,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
private final String gremlinTagFilterQuery;
private final Map<String, Object> gremlinQueryBindings;
public ClassificationSearchProcessor(SearchContext context) {
super(context);
......@@ -83,6 +85,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getClassificationTypes();
final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
final String sortBy = context.getSearchParameters().getSortBy();
final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
......@@ -187,6 +191,11 @@ public class ClassificationSearchProcessor extends SearchProcessor {
entityPredicateTraitNames = PredicateUtils.andPredicate(entityPredicateTraitNames, activePredicate);
}
if (sortBy != null && !sortBy.isEmpty()) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
entityGraphQueryTraitNames.orderBy(sortBy, qrySortOrder);
}
gremlinTagFilterQuery = null;
gremlinQueryBindings = null;
}
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
......@@ -17,6 +17,7 @@
*/
package org.apache.atlas.discovery;
import org.apache.atlas.SortOrder;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
......@@ -24,11 +25,14 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -39,6 +43,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
......@@ -46,6 +51,8 @@ import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPE
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC;
public class EntitySearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class);
......@@ -66,6 +73,8 @@ public class EntitySearchProcessor extends SearchProcessor {
final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getEntityTypes();
final String typeAndSubTypesQryStr = context.getEntityTypesQryStr();
final String sortBy = context.getSearchParameters().getSortBy();
final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
final AtlasClassificationType classificationType = context.getClassificationType();
final Set<String> classificationTypeAndSubTypes = context.getClassificationTypes();
......@@ -190,12 +199,18 @@ public class EntitySearchProcessor extends SearchProcessor {
graphQueryPredicate = activePredicate;
}
}
if (sortBy != null && !sortBy.isEmpty()) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
graphQuery.orderBy(sortBy, qrySortOrder);
}
} else {
graphQuery = null;
graphQueryPredicate = null;
}
// Prepare the graph query and in-memory filter for the filtering phase
filterGraphQueryPredicate = typeNamePredicate;
......@@ -213,6 +228,7 @@ public class EntitySearchProcessor extends SearchProcessor {
if (context.getSearchParameters().getExcludeDeletedEntities()) {
filterGraphQueryPredicate = PredicateUtils.andPredicate(filterGraphQueryPredicate, activePredicate);
}
}
@Override
......@@ -241,6 +257,19 @@ public class EntitySearchProcessor extends SearchProcessor {
final List<AtlasVertex> entityVertices = new ArrayList<>();
SortOrder sortOrder = context.getSearchParameters().getSortOrder();
String sortBy = context.getSearchParameters().getSortBy();
final AtlasEntityType entityType = context.getEntityType();
AtlasStructType.AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute == null) {
sortBy = null;
} else {
sortBy = sortByAttribute.getVertexPropertyName();
}
if (sortOrder == null) { sortOrder = ASCENDING; }
for (; ret.size() < limit; qryOffset += limit) {
entityVertices.clear();
......@@ -253,7 +282,14 @@ public class EntitySearchProcessor extends SearchProcessor {
final boolean isLastResultPage;
if (indexQuery != null) {
Iterator<AtlasIndexQuery.Result> idxQueryResult = indexQuery.vertices(qryOffset, limit);
Iterator<AtlasIndexQuery.Result> idxQueryResult;
if (StringUtils.isEmpty(sortBy)) {
idxQueryResult = indexQuery.vertices(qryOffset, limit);
} else {
Order qrySortOrder = sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc;
idxQueryResult = indexQuery.vertices(qryOffset, limit, sortBy, qrySortOrder);
}
getVerticesFromIndexQueryResult(idxQueryResult, entityVertices);
......
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
......@@ -106,6 +106,9 @@ public class SearchContext {
// Invalid attributes will raise an exception with 400 error code
validateAttributes(entityType, searchParameters.getEntityFilters());
// Invalid attribute will raise an exception with 400 error code
validateAttributes(entityType, searchParameters.getSortBy());
// Invalid attributes will raise an exception with 400 error code
validateAttributes(classificationType, searchParameters.getTagFilters());
......@@ -253,13 +256,18 @@ public class SearchContext {
}
} else {
String attributeName = filterCriteria.getAttributeName();
validateAttributes(structType, attributeName);
}
}
}
private void validateAttributes(final AtlasStructType structType, final String... attributeNames) throws AtlasBaseException {
for (String attributeName : attributeNames) {
if (StringUtils.isNotEmpty(attributeName) && structType.getAttributeType(attributeName) == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, structType.getTypeName());
}
}
}
}
private boolean hasAttributeFilter(FilterCriteria filterCriteria) {
return filterCriteria != null &&
......
webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
......@@ -188,11 +188,14 @@ public class DiscoveryREST {
public AtlasSearchResult searchUsingBasic(@QueryParam("query") String query,
@QueryParam("typeName") String typeName,
@QueryParam("classification") String classification,
@QueryParam("sortBy") String sortByAttribute,
@QueryParam("sortOrder") SortOrder sortOrder,
@QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities,
@QueryParam("limit") int limit,
@QueryParam("offset") int offset) throws AtlasBaseException {
Servlets.validateQueryParamLength("typeName", typeName);
Servlets.validateQueryParamLength("classification", classification);
Servlets.validateQueryParamLength("sortBy", sortByAttribute);
if (StringUtils.isNotEmpty(query) && query.length() > maxFullTextQueryLength) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_FULLTEXT_QUERY_STR_LENGTH);
}
......@@ -212,6 +215,8 @@ public class DiscoveryREST {
searchParameters.setExcludeDeletedEntities(excludeDeletedEntities);
searchParameters.setLimit(limit);
searchParameters.setOffset(offset);
searchParameters.setSortBy(sortByAttribute);
searchParameters.setSortOrder(sortOrder);
return discoveryService.searchWithParameters(searchParameters);
} finally {
......@@ -690,6 +695,7 @@ public class DiscoveryREST {
if (parameters != null) {
Servlets.validateQueryParamLength("typeName", parameters.getTypeName());
Servlets.validateQueryParamLength("classification", parameters.getClassification());
Servlets.validateQueryParamLength("sortBy", parameters.getSortBy());
if (StringUtils.isNotEmpty(parameters.getQuery()) && parameters.getQuery().length() > maxFullTextQueryLength) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_FULLTEXT_QUERY_STR_LENGTH);
}
......
webapp/src/test/java/org/apache/atlas/web/integration/BasicSearchIT.java
......@@ -129,6 +129,12 @@ public class BasicSearchIT extends BaseResourceIT {
assertNotNull(searchResult.getEntities());
assertEquals(searchResult.getEntities().size(), testExpectation.expectedCount);
}
if (testExpectation.searchParameters.getSortBy() != null && !testExpectation.searchParameters.getSortBy().isEmpty()) {
assertNotNull(searchResult.getEntities());
assertEquals(searchResult.getEntities().get(0).getAttribute("name"),
"testtable_3");
}
}
} catch (IOException | AtlasServiceException e) {
fail(e.getMessage());
......
webapp/src/test/resources/json/search-parameters/entity-filters.json
......@@ -21,6 +21,29 @@
"expectedCount": 3
},
{
"testDescription": "Asset.name contains testtable order by name descending",
"searchParameters": {
"typeName": "hive_table",
"excludeDeletedEntities": true,
"classification": "",
"query": "",
"limit": 25,
"offset": 0,
"sortBy": "name",
"sortOrder": "DESCENDING",
"entityFilters": {
"attributeName": "name",
"operator": "contains",
"attributeValue": "testtable"
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3
},
{
"testDescription": "hive_column.name contains 30",
"searchParameters": {
"typeName": "hive_column",
......
webapp/src/test/resources/json/search-parameters/tag-filters.json
......@@ -12,5 +12,22 @@
"classification":"fooTag"
},
"expectedCount": 1
},
{
"testDescription": "Search for exact Tag name order by",
"searchParameters": {
"entityFilters":null,
"tagFilters":null,
"attributes":null,
"query":null,
"excludeDeletedEntities":true,
"limit":25,
"typeName":null,
"sortBy": "name",
"sortOrder": "DESCENDING",
"classification":"fooTag"
},
"expectedCount": 1
}
]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment