Skip to content

A
atlas
Commit bb9384ca by Sarath Subramanian
Options

ATLAS-1988: Implement REST API to search for related entities - Add excludeDeletedEntities flag

(cherry picked from commit 0183beca085b95de4e8a24da85bae843011ce732)
parent f2a49bea
master
No related merge requests found
Showing with 55 additions and 38 deletions
client/src/main/java/org/apache/atlas/AtlasClient.java
...@@ -101,6 +101,7 @@ public class AtlasClient extends AtlasBaseClient { ...@@ -101,6 +101,7 @@ public class AtlasClient extends AtlasBaseClient {
public static final String NAME = "name"; public static final String NAME = "name";
public static final String DESCRIPTION = "description"; public static final String DESCRIPTION = "description";
public static final String OWNER = "owner"; public static final String OWNER = "owner";
public static final String CREATE_TIME = "createTime";
public static final String INFRASTRUCTURE_SUPER_TYPE = "Infrastructure"; public static final String INFRASTRUCTURE_SUPER_TYPE = "Infrastructure";
public static final String DATA_SET_SUPER_TYPE = "DataSet"; public static final String DATA_SET_SUPER_TYPE = "DataSet";
......
repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
...@@ -73,9 +73,10 @@ public interface AtlasDiscoveryService { ...@@ -73,9 +73,10 @@ public interface AtlasDiscoveryService {
* @param relation relation name. * @param relation relation name.
* @param sortByAttribute sort the result using this attribute name, default value is 'name' * @param sortByAttribute sort the result using this attribute name, default value is 'name'
* @param sortOrder sorting order * @param sortOrder sorting order
* @param excludeDeletedEntities exclude deleted entities in search result.
* @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property. * @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property.
* @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0. * @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0.
* @return AtlasSearchResult * @return AtlasSearchResult
*/ */
AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttribute, SortOrder sortOrder, int limit, int offset) throws AtlasBaseException; AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttribute, SortOrder sortOrder, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException;
} }
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
...@@ -76,14 +76,19 @@ import java.util.*; ...@@ -76,14 +76,19 @@ import java.util.*;
import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND; import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND;
import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED; import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
import static org.apache.atlas.AtlasErrorCode.UNKNOWN_TYPENAME; import static org.apache.atlas.AtlasErrorCode.UNKNOWN_TYPENAME;
import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.SortOrder.DESCENDING; import static org.apache.atlas.SortOrder.DESCENDING;
import static org.apache.atlas.model.TypeCategory.ARRAY; import static org.apache.atlas.model.TypeCategory.ARRAY;
import static org.apache.atlas.model.TypeCategory.MAP; import static org.apache.atlas.model.TypeCategory.MAP;
import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE; import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_DESCENDING_SORT; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_DESCENDING_SORT;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_ASCENDING_SORT; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_ASCENDING_SORT;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TO_RANGE_LIST;
@Component @Component
public class EntityDiscoveryService implements AtlasDiscoveryService { public class EntityDiscoveryService implements AtlasDiscoveryService {
...@@ -364,9 +369,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -364,9 +369,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
} }
if (excludeDeletedEntities) { if (excludeDeletedEntities) {
bindings.put("state", Status.ACTIVE.toString()); bindings.put("state", ACTIVE.toString());
basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER); basicQuery += gremlinQueryProvider.getQuery(BASIC_SEARCH_STATE_FILTER);
} }
if (isGuidPrefixSearch) { if (isGuidPrefixSearch) {
...@@ -378,7 +383,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -378,7 +383,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
bindings.put("startIdx", params.offset()); bindings.put("startIdx", params.offset());
bindings.put("endIdx", params.offset() + params.limit()); bindings.put("endIdx", params.offset() + params.limit());
basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.TO_RANGE_LIST); basicQuery += gremlinQueryProvider.getQuery(TO_RANGE_LIST);
ScriptEngine scriptEngine = graph.getGremlinScriptEngine(); ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
...@@ -504,8 +509,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -504,8 +509,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override @Override
@GraphTransaction @GraphTransaction
public AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttributeName, public AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttributeName, SortOrder sortOrder,
SortOrder sortOrder, int limit, int offset) throws AtlasBaseException { boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.RELATIONSHIP); AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.RELATIONSHIP);
if (StringUtils.isEmpty(guid) || StringUtils.isEmpty(relation)) { if (StringUtils.isEmpty(guid) || StringUtils.isEmpty(relation)) {
...@@ -543,21 +548,37 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -543,21 +548,37 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
sortByAttributeName = sortByAttribute.getQualifiedName(); sortByAttributeName = sortByAttribute.getQualifiedName();
if (sortOrder == null) { if (sortOrder == null) {
sortOrder = SortOrder.ASCENDING; sortOrder = ASCENDING;
} }
} }
String relatedEntitiesQuery = getRelatedEntitiesQuery(sortOrder); QueryParams params = validateSearchParams(limit, offset);
ScriptEngine scriptEngine = graph.getGremlinScriptEngine(); ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
Bindings bindings = scriptEngine.createBindings(); Bindings bindings = scriptEngine.createBindings();
QueryParams params = validateSearchParams(limit, offset); Set<String> states = getEntityStates();
String relatedEntitiesQuery = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH);
if (excludeDeletedEntities) {
states.remove(DELETED.toString());
}
if (sortOrder == ASCENDING) {
relatedEntitiesQuery += gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_ASCENDING_SORT);
bindings.put("sortAttributeName", sortByAttributeName);
} else if (sortOrder == DESCENDING) {
relatedEntitiesQuery += gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_DESCENDING_SORT);
bindings.put("sortAttributeName", sortByAttributeName);
}
relatedEntitiesQuery += gremlinQueryProvider.getQuery(TO_RANGE_LIST);
bindings.put("g", graph); bindings.put("g", graph);
bindings.put("guid", guid); bindings.put("guid", guid);
bindings.put("relation", relation); bindings.put("relation", relation);
bindings.put("sortAttributeName", sortByAttributeName); bindings.put("states", Collections.unmodifiableSet(states));
bindings.put("offset", params.offset()); bindings.put("startIdx", params.offset());
bindings.put("limit", params.offset() + params.limit()); bindings.put("endIdx", params.offset() + params.limit());
try { try {
Object result = graph.executeGremlinScript(scriptEngine, bindings, relatedEntitiesQuery, false); Object result = graph.executeGremlinScript(scriptEngine, bindings, relatedEntitiesQuery, false);
...@@ -725,7 +746,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -725,7 +746,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
} }
private boolean skipDeletedEntities(boolean excludeDeletedEntities, AtlasVertex<?, ?> vertex) { private boolean skipDeletedEntities(boolean excludeDeletedEntities, AtlasVertex<?, ?> vertex) {
return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED; return excludeDeletedEntities && GraphHelper.getStatus(vertex) == DELETED;
} }
private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesLengthInIdxQuery) { private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesLengthInIdxQuery) {
...@@ -767,17 +788,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { ...@@ -767,17 +788,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return ret; return ret;
} }
private String getRelatedEntitiesQuery(SortOrder sortOrder) { private Set<String> getEntityStates() {
final String ret; return new HashSet<>(Arrays.asList(ACTIVE.toString(), DELETED.toString()));
if (sortOrder == null) {
ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH);
} else if (sortOrder == DESCENDING) {
ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_DESCENDING_SORT);
} else {
ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_ASCENDING_SORT);
}
return ret;
} }
} }
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
...@@ -232,11 +232,13 @@ public final class EntityGraphRetriever { ...@@ -232,11 +232,13 @@ public final class EntityGraphRetriever {
Object name = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.NAME)); Object name = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.NAME));
Object description = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.DESCRIPTION)); Object description = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.DESCRIPTION));
Object owner = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.OWNER)); Object owner = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.OWNER));
Object createTime = entityVertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
Object displayText = name != null ? name : ret.getAttribute(AtlasClient.QUALIFIED_NAME); Object displayText = name != null ? name : ret.getAttribute(AtlasClient.QUALIFIED_NAME);
ret.setAttribute(AtlasClient.NAME, name); ret.setAttribute(AtlasClient.NAME, name);
ret.setAttribute(AtlasClient.DESCRIPTION, description); ret.setAttribute(AtlasClient.DESCRIPTION, description);
ret.setAttribute(AtlasClient.OWNER, owner); ret.setAttribute(AtlasClient.OWNER, owner);
ret.setAttribute(AtlasClient.CREATE_TIME, createTime);
if (displayText != null) { if (displayText != null) {
ret.setDisplayText(displayText.toString()); ret.setDisplayText(displayText.toString());
......
repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
...@@ -96,11 +96,11 @@ public class AtlasGremlin2QueryProvider extends AtlasGremlinQueryProvider { ...@@ -96,11 +96,11 @@ public class AtlasGremlin2QueryProvider extends AtlasGremlinQueryProvider {
case COMPARE_CONTAINS: case COMPARE_CONTAINS:
return ".filter({it.getProperty('%s').contains(%s)})"; return ".filter({it.getProperty('%s').contains(%s)})";
case RELATIONSHIP_SEARCH: case RELATIONSHIP_SEARCH:
return "g.V('__guid', guid).both(relation)[offset..<limit].toList()"; return "g.V('__guid', guid).both(relation).has('__state', T.in, states)";
case RELATIONSHIP_SEARCH_DESCENDING_SORT:
return "g.V('__guid', guid).both(relation)[offset..<limit].order{it.b.getProperty(sortAttributeName) <=> it.a.getProperty(sortAttributeName)}.toList()";
case RELATIONSHIP_SEARCH_ASCENDING_SORT: case RELATIONSHIP_SEARCH_ASCENDING_SORT:
return "g.V('__guid', guid).both(relation)[offset..<limit].order{it.a.getProperty(sortAttributeName) <=> it.b.getProperty(sortAttributeName)}.toList()"; return ".order{it.a.getProperty(sortAttributeName) <=> it.b.getProperty(sortAttributeName)}";
case RELATIONSHIP_SEARCH_DESCENDING_SORT:
return ".order{it.b.getProperty(sortAttributeName) <=> it.a.getProperty(sortAttributeName)}";
} }
// Should never reach this point // Should never reach this point
return null; return null;
......
webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
...@@ -285,21 +285,22 @@ public class DiscoveryREST { ...@@ -285,21 +285,22 @@ public class DiscoveryREST {
@Path("relationship") @Path("relationship")
@Consumes(Servlets.JSON_MEDIA_TYPE) @Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") String guid, public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") String guid,
@QueryParam("relation") String relation, @QueryParam("relation") String relation,
@QueryParam("sortBy") String sortByAttribute, @QueryParam("sortBy") String sortByAttribute,
@QueryParam("sortOrder") SortOrder sortOrder, @QueryParam("sortOrder") SortOrder sortOrder,
@QueryParam("limit") int limit, @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities,
@QueryParam("offset") int offset) throws AtlasBaseException { @QueryParam("limit") int limit,
@QueryParam("offset") int offset) throws AtlasBaseException {
AtlasPerfTracer perf = null; AtlasPerfTracer perf = null;
try { try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.relatedEntitiesSearchUsingGremlin(" + guid + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.relatedEntitiesSearchUsingGremlin(" + guid +
", " + relation + ", " + sortByAttribute + ", " + sortOrder + ", " + limit + ", " + offset + ")"); ", " + relation + ", " + sortByAttribute + ", " + sortOrder + ", " + excludeDeletedEntities + ", " + ", " + limit + ", " + offset + ")");
} }
return atlasDiscoveryService.searchRelatedEntities(guid, relation, sortByAttribute, sortOrder, limit, offset); return atlasDiscoveryService.searchRelatedEntities(guid, relation, sortByAttribute, sortOrder, excludeDeletedEntities, limit, offset);
} finally { } finally {
AtlasPerfTracer.log(perf); AtlasPerfTracer.log(perf);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment