Skip to content

A
atlas
Commit 04451449 by ashutoshm Committed by Madhan Neethiraj
Options

ATLAS-1818: basic-search enhancements to improve search performance

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 138bc6f1
master
No related merge requests found
Showing with 220 additions and 72 deletions
distro/src/conf/atlas-application.properties
......@@ -62,6 +62,8 @@ atlas.graph.index.search.backend=${titan.index.backend}
${titan.index.properties}
# Solr-specific configuration property
atlas.graph.index.search.max-result-set-size=150
######### Notification Configs #########
atlas.notification.embedded=true
......
distro/src/conf/solr/solrconfig.xml
......@@ -277,19 +277,19 @@
and old cache.
-->
<filterCache class="solr.FastLRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
size="2000"
initialSize="2000"
autowarmCount="1000"/>
<!-- Query Result Cache
Caches results of searches - ordered lists of document ids
(DocList) based on a query, a sort, and the range of documents requested.
-->
<queryResultCache class="solr.LRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<queryResultCache class="solr.FastLRUCache"
size="26000"
initialSize="26000"
autowarmCount="400"/>
<!-- Document Cache
......@@ -297,10 +297,10 @@
document). Since Lucene internal document ids are transient,
this cache will not be autowarmed.
-->
<documentCache class="solr.LRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<documentCache class="solr.FastLRUCache"
size="26000"
initialSize="26000"
autowarmCount="400"/>
<!-- custom cache currently used by block join -->
<cache name="perSegFilter"
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
......@@ -158,6 +158,20 @@ public interface AtlasGraph<V, E> {
AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString);
/**
* Creates an index query.
*
* @param indexName index name
* @param queryString the query
* @param offset specify the offset that should be applied for the query. This is useful for paging through
* list of results
*
* @see <a
* href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html">
* Elastic Search Reference</a> for query syntax
*/
AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString, int offset);
/**
* Gets the management object associated with this graph and opens a transaction
* for changes that are made.
* @return
......
graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
......@@ -17,39 +17,6 @@
*/
package org.apache.atlas.repository.graphdb.titan0;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.script.Bindings;
import javax.script.ScriptContext;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
import org.apache.atlas.typesystem.types.IDataType;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
......@@ -65,9 +32,40 @@ import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
import com.tinkerpop.pipes.util.structures.Row;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
import org.apache.atlas.typesystem.types.IDataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.script.Bindings;
import javax.script.ScriptContext;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Titan 0.5.4 implementation of AtlasGraph.
......@@ -163,7 +161,12 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
@Override
public AtlasIndexQuery<Titan0Vertex, Titan0Edge> indexQuery(String fulltextIndex, String graphQuery) {
TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery);
return indexQuery(fulltextIndex, graphQuery, 0);
}
@Override
public AtlasIndexQuery<Titan0Vertex, Titan0Edge> indexQuery(String fulltextIndex, String graphQuery, int offset) {
TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery).offset(offset);
return new Titan0IndexQuery(this, query);
}
......
graphdb/titan0/src/test/resources/atlas-application.properties
......@@ -50,6 +50,8 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
# Solr-specific configuration property
atlas.graph.index.search.max-result-set-size=150
######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process
......
graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java
......@@ -17,19 +17,18 @@
*/
package org.apache.atlas.repository.graphdb.titan1;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
......@@ -57,18 +56,17 @@ import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.SchemaViolationException;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Titan 1.0.0 implementation of AtlasGraph.
......@@ -179,7 +177,12 @@ public class Titan1Graph implements AtlasGraph<Titan1Vertex, Titan1Edge> {
@Override
public AtlasIndexQuery<Titan1Vertex, Titan1Edge> indexQuery(String fulltextIndex, String graphQuery) {
TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery);
return indexQuery(fulltextIndex, graphQuery, 0);
}
@Override
public AtlasIndexQuery<Titan1Vertex, Titan1Edge> indexQuery(String fulltextIndex, String graphQuery, int offset) {
TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery).offset(offset);
return new Titan1IndexQuery(this, query);
}
......
repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.services;
import org.apache.atlas.TestOnlyModule;
import org.apache.atlas.discovery.EntityDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils;
import org.powermock.reflect.Whitebox;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.testng.Assert.assertEquals;
@Guice(modules = TestOnlyModule.class)
public class EntityDiscoveryServiceTest {
private final String TEST_TYPE = "test";
private final String TEST_TYPE1 = "test1";
private final String TEST_TYPE2 = "test2";
private final String TEST_TYPE3 = "test3";
private final String TEST_TYPE_WITH_SUB_TYPES = "testTypeWithSubTypes";
private AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
AtlasEntityDef typeTest = null;
AtlasEntityDef typeTest1 = null;
AtlasEntityDef typeTest2 = null;
AtlasEntityDef typeTest3 = null;
AtlasEntityDef typeWithSubTypes = null;
private final int maxTypesCountInIdxQuery = 10;
@BeforeClass
public void init() throws AtlasBaseException {
typeTest = new AtlasEntityDef(TEST_TYPE);
typeTest1 = new AtlasEntityDef(TEST_TYPE1);
typeTest2 = new AtlasEntityDef(TEST_TYPE2);
typeTest3 = new AtlasEntityDef(TEST_TYPE3);
typeWithSubTypes = new AtlasEntityDef(TEST_TYPE_WITH_SUB_TYPES);
typeTest1.addSuperType(TEST_TYPE_WITH_SUB_TYPES);
typeTest2.addSuperType(TEST_TYPE_WITH_SUB_TYPES);
typeTest3.addSuperType(TEST_TYPE_WITH_SUB_TYPES);
AtlasTypeRegistry.AtlasTransientTypeRegistry ttr = typeRegistry.lockTypeRegistryForUpdate();
ttr.addType(typeTest);
ttr.addType(typeWithSubTypes);
ttr.addType(typeTest1);
ttr.addType(typeTest2);
ttr.addType(typeTest3);
typeRegistry.releaseTypeRegistryForUpdate(ttr, true);
}
@Test
public void getSubTypesForType_NullStringReturnsEmptyString() throws Exception {
invokeGetSubTypesForType(null, maxTypesCountInIdxQuery);
}
@Test
public void getSubTypesForType_BlankStringReturnsEmptyString() throws Exception {
invokeGetSubTypesForType(" ", maxTypesCountInIdxQuery);
}
@Test
public void getSubTypesForType_EmptyStringReturnsEmptyString() throws Exception {
invokeGetSubTypesForType("", maxTypesCountInIdxQuery);
}
@Test
public void getSubTypeForTypeWithNoSubType_ReturnsTypeString() throws Exception {
String s = invokeGetSubTypesForType(TEST_TYPE, 10);
assertEquals(s, "(" + TEST_TYPE + ")");
}
@Test
public void getSubTypeForTypeWithSubTypes_ReturnsOrClause() throws Exception {
String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, maxTypesCountInIdxQuery);
assertTrue(s.startsWith("(" + TEST_TYPE_WITH_SUB_TYPES));
assertTrue(s.contains(" " + TEST_TYPE1));
assertTrue(s.contains(" " + TEST_TYPE2));
assertTrue(s.contains(" " + TEST_TYPE3));
assertTrue(s.endsWith(")"));
}
@Test
public void getSubTypeForTypeWithSubTypes_ReturnsEmptyString() throws Exception {
String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, 2);
assertTrue(StringUtils.isBlank(s));
}
private String invokeGetSubTypesForType(String inputString, int maxSubTypes) throws Exception {
String s = Whitebox.invokeMethod(EntityDiscoveryService.class, "getTypeFilter", typeRegistry, inputString, maxSubTypes);
assertNotNull(s);
return s;
}
}
typesystem/src/test/resources/atlas-application.properties
......@@ -72,7 +72,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
# Solr cloud mode properties
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
atlas.graph.index.search.max-result-set-size=150
######### Hive Lineage Configs #########
## Schema
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment