ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering

repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java

@@ -20,7 +20,6 @@ package org.apache.atlas.discovery;
 import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.*;
-import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.commons.collections.CollectionUtils;
@@ -34,60 +33,70 @@ public class EntitySearchProcessor extends SearchProcessor {
     private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("EntitySearchProcessor");
 
     private final AtlasIndexQuery indexQuery;
-    private final AtlasGraphQuery partialGraphQuery;
-    private final AtlasGraphQuery allGraphQuery;
+    private final AtlasGraphQuery graphQuery;
+    private final AtlasGraphQuery filterGraphQuery;
 
     public EntitySearchProcessor(SearchContext context) {
         super(context);
 
-        AtlasEntityType         entityType         = context.getEntityType();
-        AtlasClassificationType classificationType = context.getClassificationType();
-        FilterCriteria          filterCriteria     = context.getSearchParameters().getEntityFilters();
-        Set<String>             typeAndSubTypes    = entityType.getTypeAndAllSubTypes();
-        Set<String>             solrAttributes     = new HashSet<>();
-        Set<String>             gremlinAttributes  = new HashSet<>();
-        Set<String>             allAttributes      = new HashSet<>();
+        final AtlasEntityType entityType         = context.getEntityType();
+        final FilterCriteria  filterCriteria     = context.getSearchParameters().getEntityFilters();
+        final Set<String>     typeAndSubTypes    = entityType.getTypeAndAllSubTypes();
+        final Set<String>     solrAttributes     = new HashSet<>();
+        final Set<String>     gremlinAttributes  = new HashSet<>();
+        final Set<String>     allAttributes      = new HashSet<>();
 
 
         processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
 
-        boolean useSolrSearch = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY && canApplySolrFilter(entityType, filterCriteria, false);
+        final boolean typeSearchBySolr = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+        final boolean attrSearchBySolr = canApplySolrFilter(entityType, filterCriteria, false);
 
-        if (useSolrSearch) {
         StringBuilder solrQuery = new StringBuilder();
 
+        if (typeSearchBySolr) {
             constructTypeTestQuery(solrQuery, typeAndSubTypes);
+        }
+
+        if (attrSearchBySolr) {
             constructFilterQuery(solrQuery, entityType, filterCriteria, solrAttributes);
+        } else {
+            gremlinAttributes.addAll(solrAttributes);
+        }
 
+        if (solrQuery.length() > 0) {
             String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
 
             solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
             solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll("");
 
             indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString);
+        } else {
+            indexQuery = null;
+        }
 
-            if (CollectionUtils.isNotEmpty(gremlinAttributes) || classificationType != null) {
+        if (CollectionUtils.isNotEmpty(gremlinAttributes) || !typeSearchBySolr) {
             AtlasGraphQuery query = context.getGraph().query();
 
-                addClassificationNameConditionIfNecessary(query);
+            if (!typeSearchBySolr) {
+                query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
+            }
 
-                partialGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
-            } else {
-                partialGraphQuery = null;
+            graphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
+
+            if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery == null) {
+                graphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
             }
         } else {
-            indexQuery      = null;
-            partialGraphQuery = null;
+            graphQuery = null;
         }
 
         AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
 
-        addClassificationNameConditionIfNecessary(query);
-
-        allGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
+        filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
 
         if (context.getSearchParameters().getExcludeDeletedEntities()) {
-            allGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
+            filterGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
         }
     }
 
@@ -128,15 +137,15 @@ public class EntitySearchProcessor extends SearchProcessor {
 
                     vertices = getVerticesFromIndexQueryResult(queryResult);
 
-                    if (partialGraphQuery != null) {
+                    if (graphQuery != null) {
                         AtlasGraphQuery guidQuery = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(vertices));
 
-                        guidQuery.addConditionsFrom(partialGraphQuery);
+                        guidQuery.addConditionsFrom(graphQuery);
 
                         vertices = getVertices(guidQuery.vertices().iterator());
                     }
                 } else {
-                    Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset, limit).iterator();
+                    Iterator<AtlasVertex> queryResult = graphQuery.vertices(qryOffset, limit).iterator();
 
                     if (!queryResult.hasNext()) { // no more results from query - end of search
                         break;
@@ -182,7 +191,7 @@ public class EntitySearchProcessor extends SearchProcessor {
 
         AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices));
 
-        query.addConditionsFrom(allGraphQuery);
+        query.addConditionsFrom(filterGraphQuery);
 
         List<AtlasVertex> ret = getVertices(query.vertices().iterator());
 
@@ -194,10 +203,4 @@ public class EntitySearchProcessor extends SearchProcessor {
 
         return ret;
     }
-
-    private void addClassificationNameConditionIfNecessary(AtlasGraphQuery query) {
-        if (context.getClassificationType() != null && !context.needClassificationProcessor()) {
-            query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, context.getClassificationType().getTypeAndAllSubTypes());
-        }
-    }
 }

repository/src/main/java/org/apache/atlas/discovery/SearchContext.java

@@ -109,7 +109,7 @@ public class SearchContext {
     }
 
     public boolean needClassificationProcessor() {
-        return classificationType != null && (hasAttributeFilter(searchParameters.getTagFilters()) || entityType == null);
+        return classificationType != null;
     }
 
     public boolean needEntityProcessor() {

repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java

@@ -88,7 +88,7 @@ public abstract class SearchProcessor {
     public abstract List<AtlasVertex> execute();
 
     public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
-        return nextProcessor == null ? entityVertices : nextProcessor.filter(entityVertices);
+        return nextProcessor == null || CollectionUtils.isEmpty(entityVertices) ? entityVertices : nextProcessor.filter(entityVertices);
     }
 
 
@@ -193,12 +193,20 @@ public abstract class SearchProcessor {
             String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes, 0);
 
             if (StringUtils.isNotEmpty(filterQuery)) {
-                solrQuery.append(AND_STR).append(filterQuery);
+                if (solrQuery.length() > 0) {
+                    solrQuery.append(AND_STR);
+                }
+
+                solrQuery.append(filterQuery);
             }
         }
 
         if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) {
-            solrQuery.append(AND_STR).append("v.\"__state\":").append("ACTIVE");
+            if (solrQuery.length() > 0) {
+                solrQuery.append(AND_STR);
+            }
+
+            solrQuery.append("v.\"__state\":").append("ACTIVE");
         }
     }