ATLAS-1818: basic-search enhancements to improve search performance

distro/src/conf/atlas-application.properties

@@ -62,6 +62,8 @@ atlas.graph.index.search.backend=${titan.index.backend}
 
 ${titan.index.properties}
 
+# Solr-specific configuration property
+atlas.graph.index.search.max-result-set-size=150
 
 #########  Notification Configs  #########
 atlas.notification.embedded=true

distro/src/conf/solr/solrconfig.xml

@@ -277,19 +277,19 @@
                and old cache.  
       -->
     <filterCache class="solr.FastLRUCache"
-                 size="512"
-                 initialSize="512"
-                 autowarmCount="0"/>
+                 size="2000"
+                 initialSize="2000"
+                 autowarmCount="1000"/>
 
     <!-- Query Result Cache
          
          Caches results of searches - ordered lists of document ids
          (DocList) based on a query, a sort, and the range of documents requested.  
       -->
-    <queryResultCache class="solr.LRUCache"
-                     size="512"
-                     initialSize="512"
-                     autowarmCount="0"/>
+    <queryResultCache class="solr.FastLRUCache"
+                      size="26000"
+                      initialSize="26000"
+                      autowarmCount="400"/>
    
     <!-- Document Cache
 
@@ -297,10 +297,10 @@
          document).  Since Lucene internal document ids are transient,
          this cache will not be autowarmed.  
       -->
-    <documentCache class="solr.LRUCache"
-                   size="512"
-                   initialSize="512"
-                   autowarmCount="0"/>
+    <documentCache class="solr.FastLRUCache"
+                   size="26000"
+                   initialSize="26000"
+                   autowarmCount="400"/>
     
     <!-- custom cache currently used by block join --> 
     <cache name="perSegFilter"

graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java

@@ -158,6 +158,20 @@ public interface AtlasGraph<V, E> {
     AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString);
 
     /**
+     * Creates an index query.
+     *
+     * @param indexName index name
+     * @param queryString the query
+     * @param offset specify the offset that should be applied for the query. This is useful for paging through
+     *               list of results
+     *
+     * @see <a
+     * href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html">
+     * Elastic Search Reference</a> for query syntax
+     */
+    AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString, int offset);
+
+    /**
      * Gets the management object associated with this graph and opens a transaction
      * for changes that are made.
      * @return

graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java

@@ -17,39 +17,6 @@
  */
 package org.apache.atlas.repository.graphdb.titan0;
 
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.script.Bindings;
-import javax.script.ScriptContext;
-import javax.script.ScriptEngine;
-import javax.script.ScriptEngineManager;
-import javax.script.ScriptException;
-
-import org.apache.atlas.AtlasErrorCode;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.groovy.GroovyExpression;
-import org.apache.atlas.repository.graphdb.AtlasEdge;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
-import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
-import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
-import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.repository.graphdb.GremlinVersion;
-import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
-import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
-import org.apache.atlas.typesystem.types.IDataType;
-
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
@@ -65,9 +32,40 @@ import com.tinkerpop.blueprints.Element;
 import com.tinkerpop.blueprints.Vertex;
 import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
 import com.tinkerpop.pipes.util.structures.Row;
+import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.GremlinVersion;
+import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
+import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
+import org.apache.atlas.typesystem.types.IDataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.script.Bindings;
+import javax.script.ScriptContext;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 
 /**
  * Titan 0.5.4 implementation of AtlasGraph.
@@ -163,7 +161,12 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
 
     @Override
     public AtlasIndexQuery<Titan0Vertex, Titan0Edge> indexQuery(String fulltextIndex, String graphQuery) {
-        TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery);
+        return indexQuery(fulltextIndex, graphQuery, 0);
+    }
+
+    @Override
+    public AtlasIndexQuery<Titan0Vertex, Titan0Edge> indexQuery(String fulltextIndex, String graphQuery, int offset) {
+        TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery).offset(offset);
         return new Titan0IndexQuery(this, query);
     }
 

graphdb/titan0/src/test/resources/atlas-application.properties

@@ -50,6 +50,8 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
 atlas.graph.index.search.solr.mode=cloud
 atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
 
+# Solr-specific configuration property
+atlas.graph.index.search.max-result-set-size=150
 
 #########  Hive Lineage Configs  #########
 # This models reflects the base super types for Data and Process

graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java

@@ -17,19 +17,18 @@
  */
 package org.apache.atlas.repository.graphdb.titan1;
 
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.script.Bindings;
-import javax.script.ScriptEngine;
-import javax.script.ScriptException;
-
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.thinkaurelius.titan.core.Cardinality;
+import com.thinkaurelius.titan.core.PropertyKey;
+import com.thinkaurelius.titan.core.SchemaViolationException;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.TitanIndexQuery;
+import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
+import com.thinkaurelius.titan.core.schema.TitanManagement;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.groovy.GroovyExpression;
@@ -57,18 +56,17 @@ import org.apache.tinkerpop.gremlin.structure.io.IoCore;
 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.thinkaurelius.titan.core.Cardinality;
-import com.thinkaurelius.titan.core.PropertyKey;
-import com.thinkaurelius.titan.core.SchemaViolationException;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanIndexQuery;
-import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
-import com.thinkaurelius.titan.core.schema.TitanManagement;
-import com.thinkaurelius.titan.core.util.TitanCleanup;
+import javax.script.Bindings;
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * Titan 1.0.0 implementation of AtlasGraph.
@@ -179,7 +177,12 @@ public class Titan1Graph implements AtlasGraph<Titan1Vertex, Titan1Edge> {
 
     @Override
     public AtlasIndexQuery<Titan1Vertex, Titan1Edge> indexQuery(String fulltextIndex, String graphQuery) {
-        TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery);
+        return indexQuery(fulltextIndex, graphQuery, 0);
+    }
+
+    @Override
+    public AtlasIndexQuery<Titan1Vertex, Titan1Edge> indexQuery(String fulltextIndex, String graphQuery, int offset) {
+        TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery).offset(offset);
         return new Titan1IndexQuery(this, query);
     }
 

repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.services;
+
+import org.apache.atlas.TestOnlyModule;
+import org.apache.atlas.discovery.EntityDiscoveryService;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.lang.StringUtils;
+import org.powermock.reflect.Whitebox;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.testng.Assert.assertEquals;
+
+@Guice(modules = TestOnlyModule.class)
+public class EntityDiscoveryServiceTest {
+
+    private final String TEST_TYPE                = "test";
+    private final String TEST_TYPE1               = "test1";
+    private final String TEST_TYPE2               = "test2";
+    private final String TEST_TYPE3               = "test3";
+    private final String TEST_TYPE_WITH_SUB_TYPES = "testTypeWithSubTypes";
+    private AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
+
+    AtlasEntityDef typeTest         = null;
+    AtlasEntityDef typeTest1        = null;
+    AtlasEntityDef typeTest2        = null;
+    AtlasEntityDef typeTest3        = null;
+    AtlasEntityDef typeWithSubTypes = null;
+
+    private final int maxTypesCountInIdxQuery = 10;
+
+
+    @BeforeClass
+    public void init() throws AtlasBaseException {
+        typeTest         = new AtlasEntityDef(TEST_TYPE);
+        typeTest1        = new AtlasEntityDef(TEST_TYPE1);
+        typeTest2        = new AtlasEntityDef(TEST_TYPE2);
+        typeTest3        = new AtlasEntityDef(TEST_TYPE3);
+        typeWithSubTypes = new AtlasEntityDef(TEST_TYPE_WITH_SUB_TYPES);
+
+        typeTest1.addSuperType(TEST_TYPE_WITH_SUB_TYPES);
+        typeTest2.addSuperType(TEST_TYPE_WITH_SUB_TYPES);
+        typeTest3.addSuperType(TEST_TYPE_WITH_SUB_TYPES);
+
+        AtlasTypeRegistry.AtlasTransientTypeRegistry ttr = typeRegistry.lockTypeRegistryForUpdate();
+
+        ttr.addType(typeTest);
+        ttr.addType(typeWithSubTypes);
+        ttr.addType(typeTest1);
+        ttr.addType(typeTest2);
+        ttr.addType(typeTest3);
+
+        typeRegistry.releaseTypeRegistryForUpdate(ttr, true);
+    }
+
+    @Test
+    public void getSubTypesForType_NullStringReturnsEmptyString() throws Exception {
+        invokeGetSubTypesForType(null, maxTypesCountInIdxQuery);
+    }
+
+    @Test
+    public void getSubTypesForType_BlankStringReturnsEmptyString() throws Exception {
+        invokeGetSubTypesForType(" ", maxTypesCountInIdxQuery);
+    }
+
+    @Test
+    public void getSubTypesForType_EmptyStringReturnsEmptyString() throws Exception {
+        invokeGetSubTypesForType("", maxTypesCountInIdxQuery);
+    }
+
+    @Test
+    public void getSubTypeForTypeWithNoSubType_ReturnsTypeString() throws Exception {
+        String s = invokeGetSubTypesForType(TEST_TYPE, 10);
+
+        assertEquals(s, "(" + TEST_TYPE + ")");
+    }
+
+    @Test
+    public void getSubTypeForTypeWithSubTypes_ReturnsOrClause() throws Exception {
+        String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, maxTypesCountInIdxQuery);
+
+        assertTrue(s.startsWith("(" + TEST_TYPE_WITH_SUB_TYPES));
+        assertTrue(s.contains(" " + TEST_TYPE1));
+        assertTrue(s.contains(" " + TEST_TYPE2));
+        assertTrue(s.contains(" " + TEST_TYPE3));
+        assertTrue(s.endsWith(")"));
+    }
+
+    @Test
+    public void getSubTypeForTypeWithSubTypes_ReturnsEmptyString() throws Exception {
+        String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, 2);
+
+        assertTrue(StringUtils.isBlank(s));
+    }
+
+    private String invokeGetSubTypesForType(String inputString, int maxSubTypes) throws Exception {
+        String s = Whitebox.invokeMethod(EntityDiscoveryService.class, "getTypeFilter", typeRegistry, inputString, maxSubTypes);
+
+        assertNotNull(s);
+        return s;
+    }
+}

typesystem/src/test/resources/atlas-application.properties

@@ -72,7 +72,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
 # Solr cloud mode properties
 atlas.graph.index.search.solr.mode=cloud
 atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
-
+atlas.graph.index.search.max-result-set-size=150
 
 #########  Hive Lineage Configs  #########
 ## Schema