Skip to content

A
atlas
Commit 17803eed by skoritala Committed by Madhan Neethiraj
Options

ATLAS-3246: enhancements in free-text search functionality

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 8b722eb8
Showing with 320 additions and 131 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -39,7 +39,7 @@ public final class Constants {
public static final String GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "guid");
public static final String RELATIONSHIP_GUID_PROPERTY_KEY = encodePropertyKey(RELATIONSHIP_PROPERTY_KEY_PREFIX + GUID_PROPERTY_KEY);
public static final String HISTORICAL_GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids");
public static final String FREETEXT_REQUEST_HANDLER = "/freetext";
/**
* Entity type name property key.
*/
......
distro/src/bin/atlas_config.py
......@@ -32,7 +32,7 @@ LIB = "lib"
CONF = "conf"
LOG = "logs"
WEBAPP = "server" + os.sep + "webapp"
SOLR_CONF_DIR = "conf" + os.sep + "solr"
CONFIG_SETS_CONF = "server" + os.sep + "solr" + os.sep + "configsets" + os.sep + "_default" + os.sep + "conf"
DATA = "data"
ATLAS_CONF = "ATLAS_CONF"
ATLAS_LOG = "ATLAS_LOG_DIR"
......@@ -112,7 +112,7 @@ def elasticsearchBinDir(dir):
return os.environ.get(SOLR_BIN, os.path.join(dir, "elasticsearch", BIN))
def solrConfDir(dir):
return os.environ.get(SOLR_CONF, os.path.join(dir, SOLR_CONF_DIR))
return os.environ.get(SOLR_CONF, os.path.join(dir, "solr", CONFIG_SETS_CONF))
def solrPort():
return os.environ.get(SOLR_PORT, DEFAULT_SOLR_PORT)
......
distro/src/conf/solr/schema.xml
......@@ -524,20 +524,6 @@
class="solr.UUIDField"
indexed="true" />
<fieldType name="freetext" class="solr.TextField" omitNorms="true"
omitTermFreqAndPositions="true">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.EdgeNGramFilterFactory" minGramSize="2" maxGramSize="16" />
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.KeywordTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
<dynamicField name="*_uuid" type="uuid" indexed="true" stored="true"/>
......@@ -545,16 +531,4 @@
<field name="ttl" type="string" indexed="true" stored="true" />
<field name="expire_at" type="date" indexed="true" stored="true" />
<field name="timestamp" type="date" indexed="true" stored="true" />
<field name="allt1s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt2s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt3s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt4s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt5s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt6s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt7s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt8s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt9s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt10s" type="freetext" multiValued="true" indexed="true" stored="false"/>
</schema>
distro/src/conf/solr/solrconfig.xml
......@@ -478,21 +478,6 @@
</lst>
</requestHandler>
<requestHandler name="/freetext" class="solr.SearchHandler">
<lst name="defaults">
<str name="defType">edismax</str>
<int name="rows">100</int>
<str name="lowercaseOperators">true</str>
<str name="qf">
allt10s^10 allt9s^9 allt8s^8 allt7s^7 allt6s^6 allt5s^5 allt4s^4 allt3s^3 allt2s^2 allt1s^1
</str>
<str name="hl.fl">*</str>
<str name="hl.requireFieldMatch">true</str>
<str name="lowercaseOperators">true</str>
<str name="facet.limit">5</str>
</lst>
</requestHandler>
<!--
The export request handler is used to export full sorted result sets.
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
......@@ -17,15 +17,18 @@
*/
package org.apache.atlas.repository.graphdb;
import java.util.Map;
/**
* Represents a graph client work with indices used by Jansgraph.
*/
public interface AtlasGraphIndexClient {
/**
* The implementers should create a mapping from source propertyName to mapping field name.
* @param indexName the name of index that needs to be worked with.
* @param sourcePropertyName the name of the source attribute.
* @param targetPropertyName the name of the target attribute to which the mapping is getting done.
* The implementers should apply the search weights for the passed in attributes.
* @param collectionName the name of the collection for which the search weight needs to be applied
* @param attributeName2SearchWeightMap the map containing search weights from attribute name to search weights.
*/
void createCopyField(String indexName, String sourcePropertyName, String targetPropertyName);
void applySearchWeight(String collectionName, Map<String, Integer> attributeName2SearchWeightMap);
}
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
......@@ -196,7 +196,7 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
@Override
public AtlasGraphIndexClient getGraphIndexClient() throws AtlasException {
try {
return new AtlasJanusGraphSolrIndexClient();
return new AtlasJanusGraphSolrIndexClient(this);
} catch (Exception e) {
LOG.error("Error encountered in creating Graph Index Client.", e);
throw new AtlasException(e);
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java
......@@ -17,45 +17,144 @@
*/
package org.apache.atlas.repository.graphdb.janus;
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.client.solrj.request.V2Request;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
import java.util.*;
import static org.apache.atlas.repository.Constants.FREETEXT_REQUEST_HANDLER;
public class AtlasJanusGraphSolrIndexClient implements AtlasGraphIndexClient {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphSolrIndexClient.class);
private final SolrClient solrClient;
private final AtlasGraph graph;
public AtlasJanusGraphSolrIndexClient(AtlasGraph graph) {
// get solr client using same settings as that of Janus Graph
this.solrClient = Solr6Index.getSolrClient();
this.graph = graph;
public AtlasJanusGraphSolrIndexClient() throws Exception {
//well, this is temp hack to get solr client using same settings as that of Janus Graph
solrClient = Solr6Index.getSolrClient();
if(solrClient == null) {
LOG.warn("The indexing system is not solr based. Non SOLR based indexing systems are not supported yet.");
LOG.warn("Non SOLR index stores are not supported yet.");
}
}
@Override
public void createCopyField(String collectionName, String srcFieldName, String mappedCopyFieldName) {
if(solrClient == null) {
LOG.error("The indexing system is not solr based. Copy fields can not be created in non SOLR based indexing systems. This request will be treated as no op.");
public void applySearchWeight(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) {
//1) try updating request handler
//2) if update fails, try creating request handler
try {
LOG.info("Attempting to update free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
updateSearchWeights(collectionName, attributeName2SearchWeightMap);
LOG.info("Successfully updated free text request handler {} for collection {}..", FREETEXT_REQUEST_HANDLER, collectionName);
return;
} catch (Throwable t) {
LOG.warn("Error encountered in updating request handler {} for collection {}. Attempting to create one", FREETEXT_REQUEST_HANDLER, collectionName, t);
}
SchemaRequest.AddCopyField addCopyFieldRequest =
new SchemaRequest.AddCopyField(srcFieldName, Arrays.asList(mappedCopyFieldName));
SchemaResponse.UpdateResponse addCopyFieldResponse = null;
try {
addCopyFieldResponse = addCopyFieldRequest.process(solrClient, collectionName);
} catch (SolrServerException | IOException e) {
String msg = String.format("Error encountered in creating the copy field from %s to %s for collection %s.", srcFieldName, mappedCopyFieldName, collectionName);
LOG.error(msg);
throw new RuntimeException(msg, e);
LOG.info("Attempting to create free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
createFreeTextRequestHandler(collectionName, attributeName2SearchWeightMap);
LOG.info("Successfully created free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
} catch (Throwable t) {
String msg = String.format("Error encountered in creating the request handler '%s' for collection '%s'.", FREETEXT_REQUEST_HANDLER, collectionName);
LOG.error(msg, t);
throw new RuntimeException(msg, t);
}
}
private void updateSearchWeights(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) {
try {
updateFreeTextRequestHandler(collectionName, attributeName2SearchWeightMap);
} catch (Throwable t) {
String msg = String.format("Error encountered in updating the request handler '%s' for collection '%s'", FREETEXT_REQUEST_HANDLER, collectionName);
LOG.error(msg, t);
throw new RuntimeException(msg, t);
}
LOG.info("Updated free text request handler for collection {}.", collectionName);
}
private String generateSearchWeightString(AtlasGraphManagement management, String indexName, Map<String, Integer> searchWeightsMap) {
StringBuilder searchWeightBuilder = new StringBuilder();
Set<Map.Entry<String, Integer>> searchWeightFields = searchWeightsMap.entrySet();
for (Map.Entry<String, Integer> entry : searchWeightFields) {
AtlasPropertyKey propertyKey = management.getPropertyKey(entry.getKey());
String indexFieldName = management.getIndexFieldName(indexName, propertyKey);
searchWeightBuilder.append(" ")
.append(indexFieldName)
.append("^")
.append(entry.getValue().intValue());
}
return searchWeightBuilder.toString();
}
private void updateFreeTextRequestHandler(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException {
String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap);
String payLoadString = generatePayLoadForFreeText("update-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString);
performRequestHandlerAction(collectionName, solrClient, payLoadString);
}
private void createFreeTextRequestHandler(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException {
String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap);
String payLoadString = generatePayLoadForFreeText("create-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString);
performRequestHandlerAction(collectionName, solrClient, payLoadString);
}
@VisibleForTesting
static String generatePayLoadForFreeText(String action, String handlerName, String qfValue) {
return String.format("{" +
" %s : { " +
" 'name' : '%s', " +
" 'class': 'solr.SearchHandler' , " +
" 'defaults': " + "{" +
" 'defType': 'edismax' , " +
" 'rows': 100 , " +
" 'lowercaseOperators': true , " +
" 'qf': '%s' , " +
" 'hl.fl': '*' , " +
" 'hl.requireFieldMatch': true , " +
" 'lowercaseOperators': true , " +
" }" +
" }" +
"}", action, handlerName, qfValue);
}
private void performRequestHandlerAction(String collectionName, SolrClient solrClient,
String actionPayLoad) throws IOException, SolrServerException {
V2Request v2Request = new V2Request.Builder(String.format("/collections/%s/config", collectionName))
.withMethod(SolrRequest.METHOD.POST)
.withPayload(actionPayLoad)
.build();
v2Request.process(solrClient);
}
}
intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
......@@ -261,7 +261,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
public static class AtlasAttributeDef implements Serializable {
private static final long serialVersionUID = 1L;
public static final int DEFAULT_SEARCHWEIGHT = -1;
public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3;
public static final String SEARCH_WEIGHT_ATTR_NAME = "searchWeight";
public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference";
private final String STRING_TRUE = "true";
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -23,7 +23,6 @@ import com.google.common.base.Preconditions;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.discovery.SearchIndexer;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
......@@ -59,8 +58,6 @@ import java.util.List;
import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET;
......@@ -92,15 +89,27 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
// Added for type lookup when indexing the new typedefs
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraphIndexClient atlasGraphIndexClient;
private final List<IndexChangeListener> indexChangeListeners = new ArrayList<>();
//allows injection of a dummy graph for testing
private IAtlasGraphProvider provider;
private boolean recomputeIndexedKeys = true;
private Set<String> vertexIndexKeys = new HashSet<>();
private final String[] mappedCopyFieldNames;
public static boolean isValidSearchWeight(int searchWeight) {
if (searchWeight != -1 ) {
if (searchWeight < 1 || searchWeight > 10) {
return false;
}
}
return true;
}
public static boolean isStringAttribute(AtlasStructDef.AtlasAttributeDef attributeDef) {
return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeDef.getTypeName());
}
public enum UniqueKind { NONE, GLOBAL_UNIQUE, PER_TYPE_UNIQUE }
@Inject
......@@ -113,23 +122,19 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
throws IndexException, RepositoryException {
this.provider = provider;
this.typeRegistry = typeRegistry;
mappedCopyFieldNames = new String[11];
for(int i=0; i < mappedCopyFieldNames.length; i++) {
mappedCopyFieldNames[i] = String.format("allt%ds", i);
}
try {
atlasGraphIndexClient = provider.get().getGraphIndexClient();
} catch (Exception e) {
LOG.error("Error encountered in creating solr client.", e);
throw new RepositoryException("Error encountered in creating solr client.", e);
}
//make sure solr index follows graph backed index listener
addIndexListener(new SolrIndexHelper(typeRegistry));
if (!HAConfiguration.isHAEnabled(configuration)) {
initialize(provider.get());
}
}
public void addIndexListener(IndexChangeListener listener) {
indexChangeListeners.add(listener);
}
/**
* Initialize global indices for JanusGraph on server activation.
*
......@@ -191,7 +196,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.error("Failed to update indexes for changed typedefs", e);
attemptRollback(changedTypeDefs, management);
}
notifyChangeListeners();
}
public Set<String> getVertexIndexKeys() {
......@@ -283,7 +288,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
createVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false, 10);
createVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
......@@ -356,12 +361,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.info("Completed deleting indexes for type {}", typeDef.getName());
}
private static boolean isStringAttribute(AtlasStructDef.AtlasAttributeDef attributeDef) {
return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeDef.getTypeName());
}
private void createIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) {
final String propertyName = AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName());
final String propertyName = getEncodedPropertyName(typeName, attributeDef);
AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality());
boolean isUnique = attributeDef.getIsUnique();
boolean isIndexable = attributeDef.getIsIndexable();
......@@ -370,22 +371,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
boolean isArrayType = isArrayType(attribTypeName);
boolean isMapType = isMapType(attribTypeName);
final String uniqPropName = isUnique ? AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + attributeDef.getName()) : null;
int searchWeight = attributeDef.getSearchWeight();
if(attributeDef.getSearchWeight() == DEFAULT_SEARCHWEIGHT) {
if (isStringAttribute(attributeDef)) {
//We will use default search weight of 3 for string attributes.
//this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher.
LOG.info("Applying default search weight of {} for attribute{}.", DEFAULT_SEARCHWEIGHT_FOR_STRINGS, propertyName);
searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
}
}
if(!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) {
String msg = String.format("Invalid search weight '%d' was provided for property %s.", searchWeight, propertyName);
LOG.error(msg);
throw new RuntimeException(msg);
}
try {
AtlasType atlasType = typeRegistry.getType(typeName);
......@@ -424,7 +409,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (isRelationshipType(atlasType)) {
createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false);
} else {
createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false, searchWeight);
createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false);
if (uniqPropName != null) {
createVertexIndex(management, uniqPropName, UniqueKind.PER_TYPE_UNIQUE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, true);
......@@ -463,6 +448,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
/**
* gets the encoded property name for the attribute passed in.
* @param typeName the type system of the attribute
* @param attributeDef the attribute definition
* @return the encoded property name for the attribute passed in.
*/
public static String getEncodedPropertyName(String typeName, AtlasAttributeDef attributeDef) {
return AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName());
}
private void createLabelIfNeeded(final AtlasGraphManagement management, final String propertyName, final String attribTypeName) {
// If any of the referenced typename is of type Entity or Struct then the edge label needs to be created
for (String typeName : AtlasTypeUtil.getReferencedTypeNames(attribTypeName)) {
......@@ -564,10 +559,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
public void createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass,
AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes) {
createVertexIndex(management, propertyName, uniqueKind, propertyClass, cardinality, createCompositeIndex, createCompositeIndexWithTypeAndSuperTypes, -1);
}
private void createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass,
AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes, int searchWeight) {
if (propertyName != null) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
......@@ -585,16 +576,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
//for now, we will support free text based enhancements for SOLR backed index systems only.
//WARNING--it is suggested that we don't change the search weight once is assigned with a number in the range 1 to 10.
//otherwise, we might end up mapping the index to multiple copy fields...unwanted to space usage.
if( searchWeight != -1 && SearchContext.isIndexSolrBased()) {
String encodedPropertyName = management.getIndexFieldName(VERTEX_INDEX, propertyKey);
String mappedCopyFieldName = mappedCopyFieldNames[searchWeight];
atlasGraphIndexClient.createCopyField(VERTEX_INDEX, encodedPropertyName, mappedCopyFieldName);
LOG.info("Created copy field from {} to {} for collection {} for property {}.", encodedPropertyName, mappedCopyFieldName, VERTEX_INDEX, propertyName);
}
if (propertyKey != null) {
if (createCompositeIndex || uniqueKind == UniqueKind.GLOBAL_UNIQUE || uniqueKind == UniqueKind.PER_TYPE_UNIQUE) {
createVertexCompositeIndex(management, propertyClass, propertyKey, uniqueKind == UniqueKind.GLOBAL_UNIQUE);
......@@ -620,7 +601,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (LOG.isDebugEnabled()) {
LOG.debug("Creating vertex-centric index for edge label: {} direction: {} for property: {} of type: {} ",
edgeLabel, edgeDirection.name(), propertyName, propertyClass.getName());
edgeLabel, edgeDirection.name(), propertyName, propertyClass.getName());
}
final String indexName = edgeLabel + propertyKey.getName();
......@@ -824,12 +805,15 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.info("Index creation for type {} complete", typeDef.getName());
}
public static boolean isValidSearchWeight(int searchWeight) {
if(searchWeight != -1 ) {
if(searchWeight < 1 || searchWeight > 10) {
return false;
private void notifyChangeListeners() {
for (IndexChangeListener indexChangeListener : indexChangeListeners) {
try {
indexChangeListener.onChange();
} catch (Throwable t) {
LOG.error("Error encountered in notifying the index change listener {}.", indexChangeListener.getClass().getName(), t);
//we need to throw exception if any of the listeners throw execption.
throw new RuntimeException("Error encountered in notifying the index change listener " + indexChangeListener.getClass().getName(), t);
}
}
return true;
}
}
repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
public interface IndexChangeListener {
void onChange();
}
repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import org.apache.atlas.AtlasException;
import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_TEXT_KEY;
/**
This is a component that will go through all entity type definitions and create free text index
request handler with SOLR. This is a no op class in non-solr index based deployments.
This component needs to be initialized after type definitions are completely fixed with the needed patches (Ordder 3 initialization).
*/
public class SolrIndexHelper implements IndexChangeListener {
private static final Logger LOG = LoggerFactory.getLogger(SolrIndexHelper.class);
public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3;
private final AtlasTypeRegistry typeRegistry;
public SolrIndexHelper(AtlasTypeRegistry typeRegistry) {
this.typeRegistry = typeRegistry;
}
@Override
public void onChange() {
LOG.info("SolrIndexHelper.onChange()");
if(!SearchContext.isIndexSolrBased()) {
LOG.warn("Not a Solr based index store. Free text search is not supported");
return;
}
try {
AtlasGraph atlasGraph = AtlasGraphProvider.getGraphInstance();
AtlasGraphIndexClient atlasGraphIndexClient = atlasGraph.getGraphIndexClient();
Map<String, Integer> attributeName2SearchWeightMap = getAttributesWithSearchWeights();
atlasGraphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, attributeName2SearchWeightMap);
} catch (AtlasException e) {
LOG.error("Error encountered in handling type system change notification.", e);
throw new RuntimeException("Error encountered in handling type system change notification.", e);
}
}
private Map<String, Integer> getAttributesWithSearchWeights() {
Map<String, Integer> attributesWithSearchWeights = new HashMap<>();
Collection<AtlasEntityDef> allEntityDefs = typeRegistry.getAllEntityDefs();
attributesWithSearchWeights.put(CLASSIFICATION_TEXT_KEY,10);
if (CollectionUtils.isNotEmpty(allEntityDefs)) {
for (AtlasEntityDef entityDef : allEntityDefs) {
processEntity(attributesWithSearchWeights, entityDef);
}
}
return attributesWithSearchWeights;
}
private void processEntity(Map<String, Integer> attributesWithSearchWeights, AtlasEntityDef entityDef) {
for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) {
processAttributeDefinition(attributesWithSearchWeights, entityDef, attributeDef);
}
}
private void processAttributeDefinition(Map<String, Integer> attributesWithSearchWeights, AtlasEntityDef entityDef, AtlasAttributeDef attributeDef) {
if (GraphBackedSearchIndexer.isStringAttribute(attributeDef)) {
final String attributeName = GraphBackedSearchIndexer.getEncodedPropertyName(entityDef.getName(), attributeDef);
int searchWeight = attributeDef.getSearchWeight();
if (searchWeight == DEFAULT_SEARCHWEIGHT) {
//We will use default search weight of 3 for string attributes.
//this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher.
searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
} else if (!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) { //validate the value provided in the model.
String msg = String.format("Invalid search weight '%d' for attribute %s.%s", searchWeight, entityDef.getName(), attributeName);
LOG.error(msg);
throw new RuntimeException(msg);
}
LOG.info("Applying search weight {} for attribute {}.{}", searchWeight, entityDef.getName(), attributeName);
attributesWithSearchWeights.put(attributeName, searchWeight);
}
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment