Commit 46f539e1 by Sarath Subramanian Committed by Madhan Neethiraj

ATLAS-1308: Discovery/Search REST API - v2

parent 6c3b981a
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.commons.configuration.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import static org.apache.atlas.AtlasClient.LIMIT;
import static org.apache.atlas.AtlasClient.OFFSET;
import static org.apache.atlas.AtlasClient.QUERY;
public class AtlasDiscoveryClientV2 extends AtlasBaseClient {
private static final String DISCOVERY_URI = BASE_URI + "v2/search";
private static final String DSL_URI = DISCOVERY_URI + "/dsl";
private static final String FULL_TEXT_URI = DISCOVERY_URI + "/fulltext";
private static final APIInfo DSL_SEARCH = new APIInfo(DSL_URI, HttpMethod.GET, Response.Status.OK);
private static final APIInfo FULL_TEXT_SEARCH = new APIInfo(FULL_TEXT_URI, HttpMethod.GET, Response.Status.OK);
public AtlasDiscoveryClientV2(String[] baseUrl, String[] basicAuthUserNamePassword) {
super(baseUrl, basicAuthUserNamePassword);
}
public AtlasDiscoveryClientV2(String... baseUrls) throws AtlasException {
super(baseUrls);
}
public AtlasDiscoveryClientV2(UserGroupInformation ugi, String doAsUser, String... baseUrls) {
super(ugi, doAsUser, baseUrls);
}
protected AtlasDiscoveryClientV2() {
super();
}
@VisibleForTesting
AtlasDiscoveryClientV2(WebResource service, Configuration configuration) {
super(service, configuration);
}
public AtlasSearchResult dslSearch(final String query) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add(QUERY, query);
return callAPI(DSL_SEARCH, AtlasSearchResult.class, queryParams);
}
public AtlasSearchResult dslSearchWithParams(final String query, final int limit, final int offset) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add(QUERY, query);
queryParams.add(LIMIT, String.valueOf(limit));
queryParams.add(OFFSET, String.valueOf(offset));
return callAPI(DSL_SEARCH, AtlasSearchResult.class, queryParams);
}
public AtlasSearchResult fullTextSearch(final String query) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add(QUERY, query);
return callAPI(FULL_TEXT_SEARCH, AtlasSearchResult.class, queryParams);
}
public AtlasSearchResult fullTextSearchWithParams(final String query, final int limit, final int offset) throws AtlasServiceException {
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add(QUERY, query);
queryParams.add(LIMIT, String.valueOf(limit));
queryParams.add(OFFSET, String.valueOf(offset));
return callAPI(FULL_TEXT_SEARCH, AtlasSearchResult.class, queryParams);
}
}
\ No newline at end of file
......@@ -20,13 +20,14 @@ package org.apache.atlas;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.Response;
import java.text.MessageFormat;
import java.util.Arrays;
import javax.ws.rs.core.Response;
public enum AtlasErrorCode {
NO_SEARCH_RESULTS(204, "ATLAS2041E", "Given search filter {0} did not yield any results"),
// All Bad request enums go here
UNKNOWN_TYPE(400, "ATLAS4001E", "Unknown type {0} for {1}.{2}"),
CIRCULAR_REFERENCE(400, "ATLAS4002E", "{0}: invalid supertypes - circular reference back to self {1}"),
INCOMPATIBLE_SUPERTYPE(400, "ATLAS4003E", "{0}: incompatible supertype {1}"),
......@@ -60,6 +61,8 @@ public enum AtlasErrorCode {
INSTANCE_GUID_NOT_FOUND(404, "ATLAS4045E", "Given instance guid {0} is invalid"),
INSTANCE_LINEAGE_INVALID_PARAMS(404, "ATLAS4046E", "Invalid lineage query parameters passed {0}: {1}"),
INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"),
DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"),
// All data conflict errors go here
TYPE_ALREADY_EXISTS(409, "ATLAS4091E", "Given type {0} already exists"),
......
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.model.discovery;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
public class AtlasSearchResult implements Serializable {
private String queryText;
private AtlasQueryType queryType;
private List<AtlasEntityHeader> entities;
private AttributeSearchResult attributes;
private List<AtlasFullTextResult> fullTextResult;
public AtlasSearchResult() {}
public AtlasSearchResult(String queryText, AtlasQueryType queryType) {
setQueryText(queryText);
setQueryType(queryType);
setEntities(null);
setAttributes(null);
setFullTextResult(null);
}
public String getQueryText() { return queryText; }
public void setQueryText(String queryText) { this.queryText = queryText; }
public AtlasQueryType getQueryType() { return queryType; }
public void setQueryType(AtlasQueryType queryType) { this.queryType = queryType; }
public List<AtlasEntityHeader> getEntities() { return entities; }
public void setEntities(List<AtlasEntityHeader> entities) { this.entities = entities; }
public AttributeSearchResult getAttributes() { return attributes; }
public void setAttributes(AttributeSearchResult attributes) { this.attributes = attributes; }
public List<AtlasFullTextResult> getFullTextResult() { return fullTextResult; }
public void setFullTextResult(List<AtlasFullTextResult> fullTextResult) { this.fullTextResult = fullTextResult; }
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AtlasSearchResult that = (AtlasSearchResult) o;
return Objects.equals(queryText, that.queryText) &&
Objects.equals(queryType, that.queryType) &&
Objects.equals(entities, that.entities) &&
Objects.equals(attributes, that.attributes) &&
Objects.equals(fullTextResult, that.fullTextResult);
}
@Override
public int hashCode() { return Objects.hash(queryText, queryType, entities, attributes, fullTextResult); }
@Override
public String toString() {
return "AtlasSearchResult{" +
"queryText='" + queryText + '\'' +
", queryType=" + queryType +
", entities=" + entities +
", attributes=" + attributes +
", fullTextResult=" + fullTextResult +
'}';
}
public void addEntity(AtlasEntityHeader newEntity) {
if (entities == null) {
entities = new ArrayList<>();
}
if (entities.isEmpty()) {
entities.add(newEntity);
} else {
removeEntity(newEntity);
entities.add(newEntity);
}
}
public void removeEntity(AtlasEntityHeader entity) {
List<AtlasEntityHeader> entities = this.entities;
if (CollectionUtils.isNotEmpty(entities)) {
Iterator<AtlasEntityHeader> iter = entities.iterator();
while (iter.hasNext()) {
AtlasEntityHeader currEntity = iter.next();
if (StringUtils.equals(currEntity.getGuid(), entity.getGuid())) {
iter.remove();
}
}
}
}
public enum AtlasQueryType { DSL, FULL_TEXT, GREMLIN }
@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
public static class AttributeSearchResult {
private List<String> name;
private List<List<Object>> values;
public AttributeSearchResult() { }
public AttributeSearchResult(List<String> name, List<List<Object>> values) {
this.name = name;
this.values = values;
}
public List<String> getName() { return name; }
public void setName(List<String> name) { this.name = name; }
public List<List<Object>> getValues() { return values; }
public void setValues(List<List<Object>> values) { this.values = values; }
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AttributeSearchResult that = (AttributeSearchResult) o;
return Objects.equals(name, that.name) &&
Objects.equals(values, that.values);
}
@Override
public int hashCode() { return Objects.hash(name, values); }
@Override
public String toString() {
return "AttributeSearchResult{" +
"name=" + name + ", " +
"values=" + values +
'}';
}
}
@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
public static class AtlasFullTextResult {
AtlasEntityHeader entity;
Double score;
public AtlasFullTextResult() {}
public AtlasFullTextResult(AtlasEntityHeader entity, Double score) {
this.entity = entity;
this.score = score;
}
public AtlasEntityHeader getEntity() { return entity; }
public void setEntity(AtlasEntityHeader entity) { this.entity = entity; }
public Double getScore() { return score; }
public void setScore(Double score) { this.score = score; }
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AtlasFullTextResult that = (AtlasFullTextResult) o;
return Objects.equals(entity, that.entity) &&
Objects.equals(score, that.score);
}
@Override
public int hashCode() { return Objects.hash(entity, score); }
@Override
public String toString() {
return "AtlasFullTextResult{" +
"entity=" + entity +
", score=" + score +
'}';
}
}
}
......@@ -77,6 +77,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable {
if (other != null) {
setGuid(other.getGuid());
setStatus(other.getStatus());
setDisplayText(other.getDisplayText());
}
}
......
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1308 Discovery/Search REST API - v2 (sarath.kum4r@gmail.com via mneethiraj)
ATLAS-1408 added validation to prevent creating types with . (dot) in name (ashutoshm via mneethiraj)
ATLAS-1277 Add feather use 'order by ' in the DSL search (zhangqiang2 via sumasai)
ATLAS-1379 Avoid object query overhead when report query selects class type alias (guptaneeru via dkantor)
......
......@@ -24,15 +24,17 @@ import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.AtlasLineageService;
import org.apache.atlas.discovery.DataSetLineageService;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.EntityDiscoveryService;
import org.apache.atlas.discovery.EntityLineageService;
import org.apache.atlas.discovery.LineageService;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.model.lineage.AtlasLineageService;
import org.apache.atlas.discovery.AtlasDiscoveryService;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.audit.EntityAuditListener;
import org.apache.atlas.repository.audit.EntityAuditRepository;
......@@ -94,6 +96,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
bind(AtlasDiscoveryService.class).to(EntityDiscoveryService.class).asEagerSingleton();
bind(LineageService.class).to(DataSetLineageService.class).asEagerSingleton();
bind(AtlasLineageService.class).to(EntityLineageService.class).asEagerSingleton();
......
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
public interface AtlasDiscoveryService {
/**
*
* @param query search query in DSL format.
* @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property.
* @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0.
* @return AtlasSearchResult
*/
AtlasSearchResult searchUsingDslQuery(String query, int limit, int offset) throws AtlasBaseException;
/**
*
* @param query search query.
* @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property.
* @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0.
* @return AtlasSearchResult
*/
AtlasSearchResult searchUsingFullTextQuery(String query, int limit, int offset);
}
......@@ -16,10 +16,11 @@
* limitations under the License.
*/
package org.apache.atlas.model.lineage;
package org.apache.atlas.discovery;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
public interface AtlasLineageService {
......
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult;
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.query.Expressions.AliasExpression;
import org.apache.atlas.query.Expressions.Expression;
import org.apache.atlas.query.Expressions.SelectExpression;
import org.apache.atlas.query.GremlinQuery;
import org.apache.atlas.query.GremlinTranslator;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.query.QueryParser;
import org.apache.atlas.query.QueryProcessor;
import org.apache.atlas.query.SelectExpressionHelper;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Option;
import scala.util.Either;
import scala.util.parsing.combinator.Parsers.NoSuccess;
import javax.inject.Inject;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
public class EntityDiscoveryService implements AtlasDiscoveryService {
private final AtlasGraph graph;
private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class);
@Inject
EntityDiscoveryService(MetadataRepository metadataRepository) {
this.graph = AtlasGraphProvider.getGraphInstance();
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
}
@Override
public AtlasSearchResult searchUsingDslQuery(String dslQuery, int limit, int offset) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(dslQuery, AtlasQueryType.DSL);
GremlinQuery gremlinQuery = toGremlinQuery(dslQuery, limit, offset);
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Executing DSL query: {}", dslQuery);
}
Object result = graph.executeGremlinScript(gremlinQuery.queryStr(), false);
if (result instanceof List) {
List queryResult = (List) result;
if (isAtlasVerticesList(queryResult)) {
for (Object entity : queryResult) {
ret.addEntity(toAtlasEntityHeader(entity));
}
} else if (isTraitList(queryResult)) {
ret.setEntities(toTraitResult(queryResult));
} else if (gremlinQuery.hasSelectList()) {
ret.setAttributes(toAttributesResult(queryResult, gremlinQuery));
}
}
} catch (ScriptException e) {
throw new AtlasBaseException(DISCOVERY_QUERY_FAILED, gremlinQuery.queryStr());
}
return ret;
}
@Override
public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, int limit, int offset) {
AtlasSearchResult ret = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT);
QueryParams params = validateSearchParams(limit, offset);
AtlasIndexQuery idxQuery = toAtlasIndexQuery(fullTextQuery);
if (LOG.isDebugEnabled()) {
LOG.debug("Executing Full text query: {}", fullTextQuery);
}
ret.setFullTextResult(getIndexQueryResults(idxQuery, params));
return ret;
}
private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams params) {
List<AtlasFullTextResult> ret = new ArrayList<>();
Iterator<Result> iter = query.vertices();
while (iter.hasNext() && ret.size() < params.limit()) {
Result idxQueryResult = iter.next();
AtlasVertex vertex = idxQueryResult.getVertex();
String guid = vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
if (guid != null) {
AtlasEntityHeader entity = toAtlasEntityHeader(idxQueryResult.getVertex());
Double score = idxQueryResult.getScore();
ret.add(new AtlasFullTextResult(entity, score));
}
}
return ret;
}
private GremlinQuery toGremlinQuery(String query, int limit, int offset) throws AtlasBaseException {
QueryParams params = validateSearchParams(limit, offset);
Either<NoSuccess, Expression> either = QueryParser.apply(query, params);
if (either.isLeft()) {
throw new AtlasBaseException(DISCOVERY_QUERY_FAILED, query);
}
Expression expression = either.right().get();
Expression validExpression = QueryProcessor.validate(expression);
GremlinQuery gremlinQuery = new GremlinTranslator(validExpression, graphPersistenceStrategy).translate();
if (LOG.isDebugEnabled()) {
LOG.debug("Translated Gremlin Query: {}", gremlinQuery.queryStr());
}
return gremlinQuery;
}
private QueryParams validateSearchParams(int limitParam, int offsetParam) {
int defaultLimit = AtlasConfiguration.SEARCH_DEFAULT_LIMIT.getInt();
int maxLimit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt();
int limit = defaultLimit;
if (limitParam > 0 && limitParam <= maxLimit) {
limit = limitParam;
}
int offset = 0;
if (offsetParam > 0) {
offset = offsetParam;
}
return new QueryParams(limit, offset);
}
private AtlasEntityHeader toAtlasEntityHeader(Object vertexObj) {
AtlasEntityHeader ret = new AtlasEntityHeader();
if (vertexObj instanceof AtlasVertex) {
AtlasVertex vertex = (AtlasVertex) vertexObj;
ret.setTypeName(vertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class));
ret.setGuid(vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class));
ret.setDisplayText(vertex.getProperty(Constants.QUALIFIED_NAME, String.class));
String state = vertex.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
if (state != null) {
Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.STATUS_ACTIVE : Status.STATUS_DELETED);
ret.setStatus(status);
}
}
return ret;
}
private AtlasIndexQuery toAtlasIndexQuery(String fullTextQuery) {
String graphQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, fullTextQuery);
return graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery);
}
private boolean isAtlasVerticesList(List list) {
boolean ret = false;
if (CollectionUtils.isNotEmpty(list)) {
ret = list.get(0) instanceof AtlasVertex;
}
return ret;
}
private boolean isTraitList(List list) {
boolean ret = false;
if (CollectionUtils.isNotEmpty(list)) {
Object firstObj = list.get(0);
if (firstObj instanceof Map) {
Map map = (Map) firstObj;
Set keys = map.keySet();
ret = (keys.contains("theInstance") || keys.contains("theTrait"));
}
}
return ret;
}
private List<AtlasEntityHeader> toTraitResult(List list) {
List<AtlasEntityHeader> ret = new ArrayList();
for (Object mapObj : list) {
Map map = (Map) mapObj;
if (MapUtils.isNotEmpty(map)) {
for (Object key : map.keySet()) {
List values = (List) map.get(key);
if (StringUtils.equals(key.toString(), "theInstance") && isAtlasVerticesList(values)) {
ret.add(toAtlasEntityHeader(values.get(0)));
}
}
}
}
return ret;
}
private AttributeSearchResult toAttributesResult(List list, GremlinQuery query) {
AttributeSearchResult ret = new AttributeSearchResult();
List<String> names = new ArrayList<>();
List<List<Object>> values = new ArrayList<>();
// extract select attributes from gremlin query
Option<SelectExpression> selectExpr = SelectExpressionHelper.extractSelectExpression(query.expr());
if (selectExpr.isDefined()) {
List<AliasExpression> aliases = selectExpr.get().toJavaList();
if (CollectionUtils.isNotEmpty(aliases)) {
for (AliasExpression alias : aliases) {
names.add(alias.alias());
}
ret.setName(names);
}
}
for (Object mapObj : list) {
Map map = (mapObj instanceof Map ? (Map) mapObj : null);
if (MapUtils.isNotEmpty(map)) {
for (Object key : map.keySet()) {
Object vals = map.get(key);
values.add((List<Object>) vals);
}
ret.setValues(values);
}
}
return ret;
}
}
......@@ -27,7 +27,6 @@ import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
import org.apache.atlas.model.lineage.AtlasLineageService;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
......@@ -36,7 +35,6 @@ import org.apache.commons.collections.CollectionUtils;
import javax.inject.Inject;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
......
......@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableCollection
import org.apache.atlas.AtlasException
import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory}
import org.apache.atlas.typesystem.types._
import scala.collection.JavaConverters._
object Expressions {
......@@ -732,6 +733,8 @@ object Expressions {
var prefix = if(forGroupBy) { "" } else { s"""${child} select """ }
s"""${prefix}${selectListWithAlias.mkString("", ", ", "")}"""
}
def toJavaList = selectListWithAlias.asJava
}
case class LoopExpression(val input: Expression, val loopingExpression: Expression,
......
......@@ -20,16 +20,15 @@ package org.apache.atlas.web.resources;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.aspect.Monitored;
import org.apache.atlas.discovery.AtlasLineageService;
import org.apache.atlas.discovery.DiscoveryException;
import org.apache.atlas.discovery.LineageService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
import org.apache.atlas.model.lineage.AtlasLineageService;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.SchemaNotFoundException;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.atlas.web.util.LineageUtils;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONException;
......
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.rest;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.discovery.AtlasDiscoveryService;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.web.util.Servlets;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
@Path("v2/search")
@Singleton
public class DiscoveryREST {
private final AtlasDiscoveryService atlasDiscoveryService;
@Inject
public DiscoveryREST(AtlasDiscoveryService discoveryService) {
this.atlasDiscoveryService = discoveryService;
}
@GET
@Path("/dsl")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasSearchResult searchUsingDSL(@QueryParam("query") String query,
@QueryParam("limit") int limit,
@QueryParam("offset") int offset) throws AtlasBaseException {
AtlasSearchResult ret = atlasDiscoveryService.searchUsingDslQuery(query, limit, offset);
return ret;
}
@GET
@Path("/fulltext")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasSearchResult searchUsingFullText(@QueryParam("query") String query,
@QueryParam("limit") int limit,
@QueryParam("offset") int offset) throws AtlasBaseException {
AtlasSearchResult ret = atlasDiscoveryService.searchUsingFullTextQuery(query, limit, offset);
return ret;
}
}
\ No newline at end of file
......@@ -19,10 +19,10 @@
package org.apache.atlas.web.rest;
import org.apache.atlas.discovery.AtlasLineageService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
import org.apache.atlas.model.lineage.AtlasLineageService;
import org.apache.atlas.web.util.Servlets;
import javax.inject.Inject;
......
......@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet;
import kafka.consumer.ConsumerTimeoutException;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasDiscoveryClientV2;
import org.apache.atlas.AtlasEntitiesClientV2;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.AtlasTypedefClientV2;
......@@ -85,6 +86,7 @@ public abstract class BaseResourceIT {
protected AtlasClient atlasClientV1;
protected AtlasTypedefClientV2 typedefClientV2;
protected AtlasEntitiesClientV2 entitiesClientV2;
protected AtlasDiscoveryClientV2 discoveryClientV2;
public static final Logger LOG = LoggerFactory.getLogger(BaseResourceIT.class);
protected static final int MAX_WAIT_TIME = 60000;
......@@ -104,10 +106,12 @@ public abstract class BaseResourceIT {
atlasClientV1 = new AtlasClient(atlasUrls, new String[]{"admin", "admin"});
typedefClientV2 = new AtlasTypedefClientV2(atlasUrls, new String[]{"admin", "admin"});
entitiesClientV2 = new AtlasEntitiesClientV2(atlasUrls, new String[]{"admin", "admin"});
discoveryClientV2 = new AtlasDiscoveryClientV2(atlasUrls, new String[]{"admin", "admin"});
} else {
atlasClientV1 = new AtlasClient(atlasUrls);
typedefClientV2 = new AtlasTypedefClientV2(atlasUrls);
entitiesClientV2 = new AtlasEntitiesClientV2(atlasUrls);
discoveryClientV2 = new AtlasDiscoveryClientV2(atlasUrls);
}
}
......
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.resources;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import javax.ws.rs.core.MultivaluedMap;
import java.util.List;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
/**
* Search V2 Integration Tests.
*/
public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
private String dbName;
@BeforeClass
public void setUp() throws Exception {
super.setUp();
dbName = "db" + randomString();
createTypes();
createInstance(createHiveDBInstanceV1(dbName));
}
@Test
public void testSearchByDSL() throws Exception {
String dslQuery = "from "+ DATABASE_TYPE + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
AtlasSearchResult searchResult = discoveryClientV2.dslSearch(dslQuery);
assertNotNull(searchResult);
assertEquals(searchResult.getQueryText(), dslQuery);
assertEquals(searchResult.getQueryType(), AtlasQueryType.DSL);
List<AtlasEntityHeader> entities = searchResult.getEntities();
assertNotNull(entities);
assertEquals(entities.size(), 1);
AtlasEntityHeader dbEntity = entities.get(0);
assertEquals(dbEntity.getTypeName(), DATABASE_TYPE);
assertEquals(dbEntity.getDisplayText(), dbName);
assertEquals(dbEntity.getStatus(), Status.STATUS_ACTIVE);
assertNotNull(dbEntity.getGuid());
assertNull(searchResult.getAttributes());
assertNull(searchResult.getFullTextResult());
}
@Test
public void testSearchDSLLimits() throws Exception {
String dslQuery = "from "+ DATABASE_TYPE + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
AtlasSearchResult searchResult = discoveryClientV2.dslSearch(dslQuery);
assertNotNull(searchResult);
//higher limit, all results returned
searchResult = discoveryClientV2.dslSearchWithParams(dslQuery, 10, 0);
assertEquals(searchResult.getEntities().size(), 1);
//default limit and offset -1, all results returned
searchResult = discoveryClientV2.dslSearchWithParams(dslQuery, -1, -1);
assertEquals(searchResult.getEntities().size(), 1);
//uses the limit parameter passed
searchResult = discoveryClientV2.dslSearchWithParams(dslQuery, 1, 0);
assertEquals(searchResult.getEntities().size(), 1);
//uses the offset parameter passed
searchResult = discoveryClientV2.dslSearchWithParams(dslQuery, 10, 1);
assertNull(searchResult.getEntities());
//limit > 0
searchResult = discoveryClientV2.dslSearchWithParams(dslQuery, 0, 10);
assertNull(searchResult.getEntities());
//limit > maxlimit
searchResult = discoveryClientV2.dslSearchWithParams(dslQuery, Integer.MAX_VALUE, 10);
assertNull(searchResult.getEntities());
//offset >= 0
searchResult = discoveryClientV2.dslSearchWithParams(dslQuery, 10, -2);
assertEquals(searchResult.getEntities().size(), 1);
}
@Test(expectedExceptions = AtlasServiceException.class)
public void testSearchByDSLForUnknownType() throws Exception {
String dslQuery = "from blah";
discoveryClientV2.dslSearch(dslQuery);
}
@Test
public void testSearchUsingDSL() throws Exception {
String query = "from "+ DATABASE_TYPE + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
AtlasSearchResult searchResult = discoveryClientV2.dslSearch(query);
assertNotNull(searchResult);
assertEquals(searchResult.getQueryText(), query);
assertEquals(searchResult.getQueryType(), AtlasQueryType.DSL);
List<AtlasEntityHeader> entities = searchResult.getEntities();
assertNotNull(entities);
assertEquals(entities.size(), 1);
AtlasEntityHeader dbEntity = entities.get(0);
assertEquals(dbEntity.getTypeName(), DATABASE_TYPE);
assertEquals(dbEntity.getDisplayText(), dbName);
assertEquals(dbEntity.getStatus(), Status.STATUS_ACTIVE);
assertNotNull(dbEntity.getGuid());
assertNull(searchResult.getAttributes());
assertNull(searchResult.getFullTextResult());
}
@Test
public void testSearchFullTextOnDSLFailure() throws Exception {
String query = "*";
AtlasSearchResult searchResult = discoveryClientV2.fullTextSearch(query);
assertNotNull(searchResult);
assertEquals(searchResult.getQueryText(), query);
assertEquals(searchResult.getQueryType(), AtlasQueryType.FULL_TEXT);
}
@Test(dependsOnMethods = "testSearchDSLLimits")
public void testSearchUsingFullText() throws Exception {
AtlasSearchResult searchResult = discoveryClientV2.fullTextSearchWithParams(dbName, 10, 0);
assertNotNull(searchResult);
assertEquals(searchResult.getQueryText(), dbName);
assertEquals(searchResult.getQueryType(), AtlasQueryType.FULL_TEXT);
List<AtlasFullTextResult> fullTextResults = searchResult.getFullTextResult();
assertEquals(fullTextResults.size(), 1);
AtlasFullTextResult result = fullTextResults.get(0);
assertNotNull(result.getEntity());
assertEquals(result.getEntity().getTypeName(), DATABASE_TYPE);
assertNotNull(result.getScore());
//API works without limit and offset
String query = dbName;
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("query", query);
searchResult = discoveryClientV2.fullTextSearch(query);
assertNotNull(searchResult);
assertEquals(searchResult.getFullTextResult().size(), 1);
//verify passed in limits and offsets are used
//higher limit and 0 offset returns all results
searchResult = discoveryClientV2.fullTextSearchWithParams(query, 10, 0);
assertEquals(searchResult.getFullTextResult().size(), 1);
//offset is used
searchResult = discoveryClientV2.fullTextSearchWithParams(query, 10, 1);
assertEquals(searchResult.getFullTextResult().size(), 1);
//limit is used
searchResult = discoveryClientV2.fullTextSearchWithParams(query, 1, 0);
assertEquals(searchResult.getFullTextResult().size(), 1);
//higher offset returns 0 results
searchResult = discoveryClientV2.fullTextSearchWithParams(query, 1, 2);
assertEquals(searchResult.getFullTextResult().size(), 1);
}
private void createTypes() throws Exception {
HierarchicalTypeDefinition<ClassType> dslTestTypeDefinition = TypesUtil
.createClassTypeDef("dsl_test_type", ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<TraitType> classificationTraitDefinition = TypesUtil
.createTraitTypeDef("Classification", ImmutableSet.<String>of(),
TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(classificationTraitDefinition), ImmutableList.of(dslTestTypeDefinition));
createType(typesDef);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment