Skip to content

A
atlas
Commit bcec42e3 by apoorvnaik Committed by Madhan Neethiraj
Options

ATLAS-1947: AtlasSearchResult to include referredEntity headers

parent 0d8f9f8d
Showing with 739 additions and 57 deletions
intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
......@@ -31,6 +31,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
......@@ -40,14 +41,15 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class AtlasSearchResult implements Serializable {
private AtlasQueryType queryType;
private SearchParameters searchParameters;
private String queryText;
private String type;
private String classification;
private List<AtlasEntityHeader> entities;
private AttributeSearchResult attributes;
private List<AtlasFullTextResult> fullTextResult;
private AtlasQueryType queryType;
private SearchParameters searchParameters;
private String queryText;
private String type;
private String classification;
private List<AtlasEntityHeader> entities;
private AttributeSearchResult attributes;
private List<AtlasFullTextResult> fullTextResult;
private Map<String, AtlasEntityHeader> referredEntities;
public AtlasSearchResult() {}
......@@ -62,6 +64,7 @@ public class AtlasSearchResult implements Serializable {
setEntities(null);
setAttributes(null);
setFullTextResult(null);
setReferredEntities(null);
}
public AtlasSearchResult(SearchParameters searchParameters) {
......@@ -73,6 +76,7 @@ public class AtlasSearchResult implements Serializable {
setEntities(null);
setAttributes(null);
setFullTextResult(null);
setReferredEntities(null);
}
}
......@@ -80,6 +84,14 @@ public class AtlasSearchResult implements Serializable {
public void setQueryType(AtlasQueryType queryType) { this.queryType = queryType; }
public SearchParameters getSearchParameters() {
return searchParameters;
}
public void setSearchParameters(SearchParameters searchParameters) {
this.searchParameters = searchParameters;
}
public String getQueryText() { return queryText; }
public void setQueryText(String queryText) { this.queryText = queryText; }
......@@ -104,6 +116,17 @@ public class AtlasSearchResult implements Serializable {
public void setFullTextResult(List<AtlasFullTextResult> fullTextResult) { this.fullTextResult = fullTextResult; }
public Map<String, AtlasEntityHeader> getReferredEntities() {
return referredEntities;
}
public void setReferredEntities(Map<String, AtlasEntityHeader> referredEntities) {
this.referredEntities = referredEntities;
}
@Override
public int hashCode() { return Objects.hash(queryType, searchParameters, queryText, type, classification, entities, attributes, fullTextResult, referredEntities); }
@Override
public boolean equals(Object o) {
if (this == o) return true;
......@@ -116,24 +139,8 @@ public class AtlasSearchResult implements Serializable {
Objects.equals(classification, that.classification) &&
Objects.equals(entities, that.entities) &&
Objects.equals(attributes, that.attributes) &&
Objects.equals(fullTextResult, that.fullTextResult);
}
@Override
public int hashCode() { return Objects.hash(queryType, searchParameters, queryText, type, classification, entities, attributes, fullTextResult); }
@Override
public String toString() {
return "AtlasSearchResult{" +
"queryType=" + queryType +
", searchParameters='" + searchParameters + '\'' +
", queryText='" + queryText + '\'' +
", type=" + type +
", classification=" + classification +
", entities=" + entities +
", attributes=" + attributes +
", fullTextResult=" + fullTextResult +
'}';
Objects.equals(fullTextResult, that.fullTextResult) &&
Objects.equals(referredEntities, that.referredEntities);
}
public void addEntity(AtlasEntityHeader newEntity) {
......@@ -163,12 +170,19 @@ public class AtlasSearchResult implements Serializable {
}
}
public void setSearchParameters(SearchParameters searchParameters) {
this.searchParameters = searchParameters;
}
public SearchParameters getSearchParameters() {
return searchParameters;
@Override
public String toString() {
return "AtlasSearchResult{" +
"queryType=" + queryType +
", searchParameters='" + searchParameters + '\'' +
", queryText='" + queryText + '\'' +
", type=" + type +
", classification=" + classification +
", entities=" + entities +
", attributes=" + attributes +
", fullTextResult=" + fullTextResult +
", referredEntities=" + referredEntities +
'}';
}
public enum AtlasQueryType { DSL, FULL_TEXT, GREMLIN, BASIC, ATTRIBUTE }
......
intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
......@@ -207,9 +207,12 @@ public class SearchParameters {
return Objects.hash(query, typeName, classification, excludeDeletedEntities, limit, offset, entityFilters, tagFilters, attributes);
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("SearchParameters{");
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
}
sb.append('{');
sb.append("query='").append(query).append('\'');
sb.append(", typeName='").append(typeName).append('\'');
sb.append(", classification='").append(classification).append('\'');
......@@ -220,7 +223,13 @@ public class SearchParameters {
sb.append(", tagFilters=").append(tagFilters);
sb.append(", attributes=").append(attributes);
sb.append('}');
return sb.toString();
return sb;
}
@Override
public String toString() {
return toString(new StringBuilder()).toString();
}
......@@ -297,16 +306,25 @@ public class SearchParameters {
return Objects.hash(attributeName, operator, attributeValue, condition, criterion);
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("FilterCriteria{");
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
}
sb.append('{');
sb.append("attributeName='").append(attributeName).append('\'');
sb.append(", operator=").append(operator);
sb.append(", attributeValue='").append(attributeValue).append('\'');
sb.append(", condition=").append(condition);
sb.append(", criterion=").append(criterion);
sb.append('}');
return sb.toString();
return sb;
}
@Override
public String toString() {
return toString(new StringBuilder()).toString();
}
}
......
repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
......@@ -64,5 +64,5 @@ public interface AtlasDiscoveryService {
* @return Matching entities
* @throws AtlasBaseException
*/
AtlasSearchResult searchUsingBasicQuery(SearchParameters searchParameters) throws AtlasBaseException;
AtlasSearchResult searchWithParameters(SearchParameters searchParameters) throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
public class ClassificationSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(ClassificationSearchProcessor.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("ClassificationSearchProcessor");
private final AtlasIndexQuery indexQuery;
private final AtlasGraphQuery allGraphQuery;
private final AtlasGraphQuery filterGraphQuery;
public ClassificationSearchProcessor(SearchContext context) {
super(context);
AtlasClassificationType classificationType = context.getClassificationType();
FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes();
Set<String> solrAttributes = new HashSet<>();
Set<String> gremlinAttributes = new HashSet<>();
Set<String> allAttributes = new HashSet<>();
processSearchAttributes(classificationType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
// for classification search, if any attribute can't be handled by Solr - switch to all Gremlin
boolean useSolrSearch = typeAndSubTypes.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false);
if (useSolrSearch) {
StringBuilder solrQuery = new StringBuilder();
constructTypeTestQuery(solrQuery, typeAndSubTypes);
constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes);
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll("");
indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString);
} else {
indexQuery = null;
}
AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
allGraphQuery = toGremlinFilterQuery(classificationType, filterCriteria, allAttributes, query);
query = context.getGraph().query().in(Constants.TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes);
filterGraphQuery = query; // TODO: filer based on tag attributes
}
@Override
public List<AtlasVertex> execute() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ClassificationSearchProcessor.execute({})", context);
}
List<AtlasVertex> ret = new ArrayList<>();
AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "ClassificationSearchProcessor.execute(" + context + ")");
}
try {
int qryOffset = (nextProcessor == null) ? context.getSearchParameters().getOffset() : 0;
int limit = context.getSearchParameters().getLimit();
int resultIdx = qryOffset;
Set<String> processedGuids = new HashSet<>();
while (ret.size() < limit) {
if (context.terminateSearch()) {
LOG.warn("query terminated: {}", context.getSearchParameters());
break;
}
List<AtlasVertex> classificationVertices;
if (indexQuery != null) {
Iterator<AtlasIndexQuery.Result> queryResult = indexQuery.vertices(qryOffset, limit);
if (!queryResult.hasNext()) { // no more results from solr - end of search
break;
}
classificationVertices = getVerticesFromIndexQueryResult(queryResult);
} else {
Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset, limit).iterator();
if (!queryResult.hasNext()) { // no more results - end of search
break;
}
classificationVertices = getVertices(queryResult);
}
qryOffset += limit;
List<AtlasVertex> entityVertices = new ArrayList<>();
for (AtlasVertex classificationVertex : classificationVertices) {
Iterable<AtlasEdge> edges = classificationVertex.getEdges(AtlasEdgeDirection.IN);
for (AtlasEdge edge : edges) {
AtlasVertex entityVertex = edge.getOutVertex();
String guid = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
if (!processedGuids.contains(guid)) {
if (!context.getSearchParameters().getExcludeDeletedEntities() || AtlasGraphUtilsV1.getState(entityVertex) == AtlasEntity.Status.ACTIVE) {
entityVertices.add(entityVertex);
}
processedGuids.add(guid);
}
}
}
entityVertices = super.filter(entityVertices);
for (AtlasVertex entityVertex : entityVertices) {
resultIdx++;
if (resultIdx < context.getSearchParameters().getOffset()) {
continue;
}
ret.add(entityVertex);
if (ret.size() == limit) {
break;
}
}
}
} finally {
AtlasPerfTracer.log(perf);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== ClassificationSearchProcessor.execute({}): ret.size()={}", context, ret.size());
}
return ret;
}
@Override
public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size());
}
AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices));
query.addConditionsFrom(filterGraphQuery);
List<AtlasVertex> ret = getVertices(query.vertices().iterator());
ret = super.filter(ret);
if (LOG.isDebugEnabled()) {
LOG.debug("<== ClassificationSearchProcessor.filter({}): ret.size()={}", entityVertices.size(), ret.size());
}
return ret;
}
}
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
public class EntitySearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("EntitySearchProcessor");
private final AtlasIndexQuery indexQuery;
private final AtlasGraphQuery partialGraphQuery;
private final AtlasGraphQuery allGraphQuery;
public EntitySearchProcessor(SearchContext context) {
super(context);
AtlasEntityType entityType = context.getEntityType();
AtlasClassificationType classificationType = context.getClassificationType();
FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
Set<String> solrAttributes = new HashSet<>();
Set<String> gremlinAttributes = new HashSet<>();
Set<String> allAttributes = new HashSet<>();
processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
boolean useSolrSearch = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY && canApplySolrFilter(entityType, filterCriteria, false);
if (useSolrSearch) {
StringBuilder solrQuery = new StringBuilder();
constructTypeTestQuery(solrQuery, typeAndSubTypes);
constructFilterQuery(solrQuery, entityType, filterCriteria, solrAttributes);
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll("");
indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString);
if (CollectionUtils.isNotEmpty(gremlinAttributes) || classificationType != null) {
AtlasGraphQuery query = context.getGraph().query();
addClassificationNameConditionIfNecessary(query);
partialGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
} else {
partialGraphQuery = null;
}
} else {
indexQuery = null;
partialGraphQuery = null;
}
AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
addClassificationNameConditionIfNecessary(query);
allGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
if (context.getSearchParameters().getExcludeDeletedEntities()) {
allGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
}
}
@Override
public List<AtlasVertex> execute() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> EntitySearchProcessor.execute({})", context);
}
List<AtlasVertex> ret = new ArrayList<>();
AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntitySearchProcessor.execute(" + context + ")");
}
try {
int qryOffset = (nextProcessor == null) ? context.getSearchParameters().getOffset() : 0;
int limit = context.getSearchParameters().getLimit();
int resultIdx = qryOffset;
while (ret.size() < limit) {
if (context.terminateSearch()) {
LOG.warn("query terminated: {}", context.getSearchParameters());
break;
}
List<AtlasVertex> vertices;
if (indexQuery != null) {
Iterator<AtlasIndexQuery.Result> queryResult = indexQuery.vertices(qryOffset, limit);
if (!queryResult.hasNext()) { // no more results from solr - end of search
break;
}
vertices = getVerticesFromIndexQueryResult(queryResult);
if (partialGraphQuery != null) {
AtlasGraphQuery guidQuery = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(vertices));
guidQuery.addConditionsFrom(partialGraphQuery);
vertices = getVertices(guidQuery.vertices().iterator());
}
} else {
Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset, limit).iterator();
if (!queryResult.hasNext()) { // no more results from query - end of search
break;
}
vertices = getVertices(queryResult);
}
qryOffset += limit;
vertices = super.filter(vertices);
for (AtlasVertex vertex : vertices) {
resultIdx++;
if (resultIdx < context.getSearchParameters().getOffset()) {
continue;
}
ret.add(vertex);
if (ret.size() == limit) {
break;
}
}
}
} finally {
AtlasPerfTracer.log(perf);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== EntitySearchProcessor.execute({}): ret.size()={}", context, ret.size());
}
return ret;
}
@Override
public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> EntitySearchProcessor.filter({})", entityVertices.size());
}
AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices));
query.addConditionsFrom(allGraphQuery);
List<AtlasVertex> ret = getVertices(query.vertices().iterator());
ret = super.filter(ret);
if (LOG.isDebugEnabled()) {
LOG.debug("<== EntitySearchProcessor.filter({}): ret.size()={}", entityVertices.size(), ret.size());
}
return ret;
}
private void addClassificationNameConditionIfNecessary(AtlasGraphQuery query) {
if (context.getClassificationType() != null && !context.needClassificationProcessor()) {
query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, context.getClassificationType().getTypeAndAllSubTypes());
}
}
}
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
public class FullTextSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(FullTextSearchProcessor.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("FullTextSearchProcessor");
private final AtlasIndexQuery indexQuery;
public FullTextSearchProcessor(SearchContext context) {
super(context);
SearchParameters searchParameters = context.getSearchParameters();
String queryString = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, searchParameters.getQuery());
indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString);
}
@Override
public List<AtlasVertex> execute() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> FullTextSearchProcessor.execute({})", context);
}
List<AtlasVertex> ret = new ArrayList<>();
AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FullTextSearchProcessor.execute(" + context + ")");
}
try {
int qryOffset = nextProcessor == null ? context.getSearchParameters().getOffset() : 0;
int limit = context.getSearchParameters().getLimit();
int resultIdx = qryOffset;
while (ret.size() < limit) {
if (context.terminateSearch()) {
LOG.warn("query terminated: {}", context.getSearchParameters());
break;
}
Iterator<AtlasIndexQuery.Result> idxQueryResult = indexQuery.vertices(qryOffset, limit);
if (!idxQueryResult.hasNext()) { // no more results from solr - end of search
break;
}
qryOffset += limit;
List<AtlasVertex> vertices = getVerticesFromIndexQueryResult(idxQueryResult);
vertices = super.filter(vertices);
for (AtlasVertex vertex : vertices) {
resultIdx++;
if (resultIdx < context.getSearchParameters().getOffset()) {
continue;
}
ret.add(vertex);
if (ret.size() == limit) {
break;
}
}
}
} finally {
AtlasPerfTracer.log(perf);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== FullTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
}
return ret;
}
}
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.Set;
public class SearchContext {
private final SearchParameters searchParameters;
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
private final Set<String> indexedKeys;
private final AtlasEntityType entityType;
private final AtlasClassificationType classificationType;
private SearchProcessor searchProcessor;
private boolean terminateSearch = false;
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) {
this.searchParameters = searchParameters;
this.typeRegistry = typeRegistry;
this.graph = graph;
this.indexedKeys = indexedKeys;
this.entityType = typeRegistry.getEntityTypeByName(searchParameters.getTypeName());
this.classificationType = typeRegistry.getClassificationTypeByName(searchParameters.getClassification());
if (needFullTextrocessor()) {
addProcessor(new FullTextSearchProcessor(this));
}
if (needClassificationProcessor()) {
addProcessor(new ClassificationSearchProcessor(this));
}
if (needEntityProcessor()) {
addProcessor(new EntitySearchProcessor(this));
}
}
public SearchParameters getSearchParameters() { return searchParameters; }
public AtlasTypeRegistry getTypeRegistry() { return typeRegistry; }
public AtlasGraph getGraph() { return graph; }
public Set<String> getIndexedKeys() { return indexedKeys; }
public AtlasEntityType getEntityType() { return entityType; }
public AtlasClassificationType getClassificationType() { return classificationType; }
public SearchProcessor getSearchProcessor() { return searchProcessor; }
public boolean terminateSearch() { return this.terminateSearch; }
public void terminateSearch(boolean terminateSearch) { this.terminateSearch = terminateSearch; }
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
}
sb.append("searchParameters=");
if (searchParameters != null) {
searchParameters.toString(sb);
}
return sb;
}
@Override
public String toString() {
return toString(new StringBuilder()).toString();
}
public boolean needFullTextrocessor() {
return StringUtils.isNotEmpty(searchParameters.getQuery());
}
public boolean needClassificationProcessor() {
return classificationType != null && (hasAttributeFilter(searchParameters.getTagFilters()) || entityType == null);
}
public boolean needEntityProcessor() {
return entityType != null;
}
private boolean hasAttributeFilter(FilterCriteria filterCriteria) {
return filterCriteria != null &&
(CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) || StringUtils.isNotEmpty(filterCriteria.getAttributeName()));
}
private void addProcessor(SearchProcessor processor) {
if (this.searchProcessor == null) {
this.searchProcessor = processor;
} else {
this.searchProcessor.addProcessor(processor);
}
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
......@@ -128,6 +128,10 @@ public final class EntityGraphRetriever {
return ret;
}
public AtlasEntityHeader toAtlasEntityHeader(String guid) throws AtlasBaseException {
return toAtlasEntityHeader(getEntityVertex(guid));
}
public AtlasEntityHeader toAtlasEntityHeader(AtlasVertex entityVertex) throws AtlasBaseException {
return toAtlasEntityHeader(entityVertex, Collections.<String>emptySet());
}
......@@ -233,14 +237,15 @@ public final class EntityGraphRetriever {
if (CollectionUtils.isNotEmpty(attributes)) {
for (String attrName : attributes) {
String nonQualifiedAttrName = toNonQualifiedName(attrName);
if (ret.hasAttribute(attrName)) {
continue;
}
Object attrValue = getVertexAttribute(entityVertex, entityType.getAttribute(attrName));
Object attrValue = getVertexAttribute(entityVertex, entityType.getAttribute(nonQualifiedAttrName));
if (attrValue != null) {
ret.setAttribute(attrName, attrValue);
ret.setAttribute(nonQualifiedAttrName, attrValue);
}
}
}
......@@ -249,6 +254,17 @@ public final class EntityGraphRetriever {
return ret;
}
private String toNonQualifiedName(String attrName) {
String ret;
if (attrName.contains(".")) {
String[] attributeParts = attrName.split("\\.");
ret = attributeParts[attributeParts.length - 1];
} else {
ret = attrName;
}
return ret;
}
private AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity entity) {
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping system attributes for type {}", entity.getTypeName());
......
repository/src/main/java/org/apache/atlas/util/SearchTracker.java
......@@ -18,7 +18,7 @@
package org.apache.atlas.util;
import org.apache.atlas.annotation.AtlasService;
import org.apache.atlas.discovery.SearchPipeline.PipelineContext;
import org.apache.atlas.discovery.SearchContext;
import java.util.HashMap;
import java.util.Map;
......@@ -26,13 +26,13 @@ import java.util.Set;
@AtlasService
public class SearchTracker {
private Map<String, PipelineContext> activeSearches = new HashMap<>();
private Map<String, SearchContext> activeSearches = new HashMap<>();
/**
*
* @param context
*/
public String add(PipelineContext context) {
public String add(SearchContext context) {
String searchId = Thread.currentThread().getName();
activeSearches.put(searchId, context);
......@@ -45,13 +45,13 @@ public class SearchTracker {
* @param searchId
* @return
*/
public PipelineContext terminate(String searchId) {
PipelineContext ret = null;
public SearchContext terminate(String searchId) {
SearchContext ret = null;
if (activeSearches.containsKey(searchId)) {
PipelineContext pipelineToTerminate = activeSearches.remove(searchId);
SearchContext pipelineToTerminate = activeSearches.remove(searchId);
pipelineToTerminate.setForceTerminate(true);
pipelineToTerminate.terminateSearch(true);
ret = pipelineToTerminate;
}
......@@ -59,7 +59,7 @@ public class SearchTracker {
return ret;
}
public PipelineContext remove(String id) {
public SearchContext remove(String id) {
return activeSearches.remove(id);
}
......
repository/src/test/java/org/apache/atlas/TestModules.java
......@@ -143,10 +143,7 @@ public class TestModules {
typeDefChangeListenerMultibinder.addBinding().to(DefaultMetadataService.class);
typeDefChangeListenerMultibinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton();
bind(SearchPipeline.class).asEagerSingleton();
bind(SearchTracker.class).asEagerSingleton();
bind(SolrStep.class).asEagerSingleton();
bind(GremlinStep.class).asEagerSingleton();
bind(AtlasEntityStore.class).to(AtlasEntityStoreV1.class);
bind(AtlasRelationshipStore.class).to(AtlasRelationshipStoreV1.class);
......
webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
......@@ -25,7 +25,7 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.authorize.AtlasActionTypes;
import org.apache.atlas.authorize.AtlasResourceTypes;
import org.apache.atlas.authorize.simple.AtlasAuthorizationUtils;
import org.apache.atlas.discovery.SearchPipeline;
import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasExportResult;
......@@ -434,7 +434,7 @@ public class AdminResource {
@Path("activeSearches/{id}")
@Produces(Servlets.JSON_MEDIA_TYPE)
public boolean terminateActiveSearch(@PathParam("id") String searchId) {
SearchPipeline.PipelineContext terminate = activeSearches.terminate(searchId);
SearchContext terminate = activeSearches.terminate(searchId);
return null != terminate;
}
......
webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
......@@ -249,7 +249,7 @@ public class DiscoveryREST {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "TagFilters specified without tag name");
}
return atlasDiscoveryService.searchUsingBasicQuery(parameters);
return atlasDiscoveryService.searchWithParameters(parameters);
} finally {
AtlasPerfTracer.log(perf);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment