Skip to content

A
atlas
Commit e0fb7dc1 by Madhan Neethiraj
Options

ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 4)

parent 3d3be408
Showing with 68 additions and 38 deletions
repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
...@@ -58,7 +58,7 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -58,7 +58,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (useSolrSearch) { if (useSolrSearch) {
StringBuilder solrQuery = new StringBuilder(); StringBuilder solrQuery = new StringBuilder();
constructTypeTestQuery(solrQuery, classificationType, typeAndSubTypes); constructTypeTestQuery(solrQuery, typeAndSubTypes);
constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes); constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes);
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")"); String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
...@@ -95,16 +95,22 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -95,16 +95,22 @@ public class ClassificationSearchProcessor extends SearchProcessor {
} }
try { try {
final int startIdx = context.getSearchParameters().getOffset(); final int startIdx = context.getSearchParameters().getOffset();
final int limit = context.getSearchParameters().getLimit(); final int limit = context.getSearchParameters().getLimit();
int qryOffset = nextProcessor == null ? startIdx : 0; final boolean activeOnly = context.getSearchParameters().getExcludeDeletedEntities();
int resultIdx = qryOffset;
// query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
// have been dropped: like non-active-entities or duplicate-entities (same entity pointed to by multiple
// classifications in the result)
//
// first 'startIdx' number of entries will be ignored
int qryOffset = 0;
int resultIdx = qryOffset;
final Set<String> processedGuids = new HashSet<>(); final Set<String> processedGuids = new HashSet<>();
final List<AtlasVertex> entityVertices = new ArrayList<>(); final List<AtlasVertex> entityVertices = new ArrayList<>();
final List<AtlasVertex> classificationVertices = new ArrayList<>(); final List<AtlasVertex> classificationVertices = new ArrayList<>();
for (; ret.size() < limit; qryOffset += limit) { for (; ret.size() < limit; qryOffset += limit) {
entityVertices.clear(); entityVertices.clear();
classificationVertices.clear(); classificationVertices.clear();
...@@ -138,15 +144,20 @@ public class ClassificationSearchProcessor extends SearchProcessor { ...@@ -138,15 +144,20 @@ public class ClassificationSearchProcessor extends SearchProcessor {
for (AtlasEdge edge : edges) { for (AtlasEdge edge : edges) {
AtlasVertex entityVertex = edge.getOutVertex(); AtlasVertex entityVertex = edge.getOutVertex();
String guid = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
if (!processedGuids.contains(guid)) { if (activeOnly && AtlasGraphUtilsV1.getState(entityVertex) != AtlasEntity.Status.ACTIVE) {
if (!context.getSearchParameters().getExcludeDeletedEntities() || AtlasGraphUtilsV1.getState(entityVertex) == AtlasEntity.Status.ACTIVE) { continue;
entityVertices.add(entityVertex); }
}
String guid = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
processedGuids.add(guid); if (processedGuids.contains(guid)) {
continue;
} }
entityVertices.add(entityVertex);
processedGuids.add(guid);
} }
} }
......
repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.atlas.discovery; package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria; import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
...@@ -60,7 +61,7 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -60,7 +61,7 @@ public class EntitySearchProcessor extends SearchProcessor {
StringBuilder solrQuery = new StringBuilder(); StringBuilder solrQuery = new StringBuilder();
if (typeSearchBySolr) { if (typeSearchBySolr) {
constructTypeTestQuery(solrQuery, entityType, typeAndSubTypes); constructTypeTestQuery(solrQuery, typeAndSubTypes);
} }
if (attrSearchBySolr) { if (attrSearchBySolr) {
...@@ -70,6 +71,10 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -70,6 +71,10 @@ public class EntitySearchProcessor extends SearchProcessor {
} }
if (solrQuery.length() > 0) { if (solrQuery.length() > 0) {
if (context.getSearchParameters().getExcludeDeletedEntities()) {
constructStateTestQuery(solrQuery);
}
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")"); String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")"); solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
...@@ -128,10 +133,14 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -128,10 +133,14 @@ public class EntitySearchProcessor extends SearchProcessor {
} }
try { try {
final int startIdx = context.getSearchParameters().getOffset(); final int startIdx = context.getSearchParameters().getOffset();
final int limit = context.getSearchParameters().getLimit(); final int limit = context.getSearchParameters().getLimit();
int qryOffset = (nextProcessor == null && (graphQuery == null || indexQuery == null)) ? startIdx : 0;
int resultIdx = qryOffset; // when subsequent filtering stages are involved, query should start at 0 even though startIdx can be higher
//
// first 'startIdx' number of entries will be ignored
int qryOffset = (nextProcessor != null || (graphQuery != null && indexQuery != null)) ? 0 : startIdx;
int resultIdx = qryOffset;
final List<AtlasVertex> entityVertices = new ArrayList<>(); final List<AtlasVertex> entityVertices = new ArrayList<>();
...@@ -154,13 +163,6 @@ public class EntitySearchProcessor extends SearchProcessor { ...@@ -154,13 +163,6 @@ public class EntitySearchProcessor extends SearchProcessor {
while (idxQueryResult.hasNext()) { while (idxQueryResult.hasNext()) {
AtlasVertex vertex = idxQueryResult.next().getVertex(); AtlasVertex vertex = idxQueryResult.next().getVertex();
// skip non-entity vertices
if (!AtlasGraphUtilsV1.isEntityVertex(vertex)) {
LOG.warn("EntitySearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId()); // might cause duplicate entries in result
continue;
}
entityVertices.add(vertex); entityVertices.add(vertex);
} }
......
repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.apache.atlas.discovery; package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
...@@ -74,6 +75,10 @@ public class FullTextSearchProcessor extends SearchProcessor { ...@@ -74,6 +75,10 @@ public class FullTextSearchProcessor extends SearchProcessor {
} }
} }
if (context.getSearchParameters().getExcludeDeletedEntities()) {
queryString.append(AND_STR).append("(ACTIVE)");
}
queryString.append(")"); queryString.append(")");
indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString.toString()); indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString.toString());
...@@ -94,10 +99,16 @@ public class FullTextSearchProcessor extends SearchProcessor { ...@@ -94,10 +99,16 @@ public class FullTextSearchProcessor extends SearchProcessor {
} }
try { try {
final int startIdx = context.getSearchParameters().getOffset(); final int startIdx = context.getSearchParameters().getOffset();
final int limit = context.getSearchParameters().getLimit(); final int limit = context.getSearchParameters().getLimit();
int qryOffset = nextProcessor == null ? startIdx : 0; final boolean activeOnly = context.getSearchParameters().getExcludeDeletedEntities();
int resultIdx = qryOffset;
// query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
// have been dropped: like vertices of non-entity or non-active-entity
//
// first 'startIdx' number of entries will be ignored
int qryOffset = 0;
int resultIdx = qryOffset;
final List<AtlasVertex> entityVertices = new ArrayList<>(); final List<AtlasVertex> entityVertices = new ArrayList<>();
...@@ -121,8 +132,14 @@ public class FullTextSearchProcessor extends SearchProcessor { ...@@ -121,8 +132,14 @@ public class FullTextSearchProcessor extends SearchProcessor {
// skip non-entity vertices // skip non-entity vertices
if (!AtlasGraphUtilsV1.isEntityVertex(vertex)) { if (!AtlasGraphUtilsV1.isEntityVertex(vertex)) {
LOG.warn("FullTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId()); // might cause duplicate entries in result if (LOG.isDebugEnabled()) {
LOG.debug("FullTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId());
}
continue;
}
if (activeOnly && AtlasGraphUtilsV1.getState(vertex) != AtlasEntity.Status.ACTIVE) {
continue; continue;
} }
......
repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
...@@ -181,7 +181,7 @@ public abstract class SearchProcessor { ...@@ -181,7 +181,7 @@ public abstract class SearchProcessor {
return ret; return ret;
} }
protected void constructTypeTestQuery(StringBuilder solrQuery, AtlasStructType type, Set<String> typeAndAllSubTypes) { protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING); String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) { if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) {
...@@ -193,14 +193,6 @@ public abstract class SearchProcessor { ...@@ -193,14 +193,6 @@ public abstract class SearchProcessor {
.append(typeAndSubtypesString) .append(typeAndSubtypesString)
.append(")"); .append(")");
} }
if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) {
if (solrQuery.length() > 0) {
solrQuery.append(AND_STR);
}
solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
}
} }
protected void constructFilterQuery(StringBuilder solrQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> solrAttributes) { protected void constructFilterQuery(StringBuilder solrQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> solrAttributes) {
...@@ -219,6 +211,14 @@ public abstract class SearchProcessor { ...@@ -219,6 +211,14 @@ public abstract class SearchProcessor {
} }
} }
protected void constructStateTestQuery(StringBuilder solrQuery) {
if (solrQuery.length() > 0) {
solrQuery.append(AND_STR);
}
solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
}
private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, int level) { private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, int level) {
return toSolrQuery(type, criteria, solrAttributes, new StringBuilder(), level); return toSolrQuery(type, criteria, solrAttributes, new StringBuilder(), level);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment