diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index c0851fe..46c8b01 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -39,7 +39,7 @@ public final class Constants { public static final String GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "guid"); public static final String RELATIONSHIP_GUID_PROPERTY_KEY = encodePropertyKey(RELATIONSHIP_PROPERTY_KEY_PREFIX + GUID_PROPERTY_KEY); public static final String HISTORICAL_GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids"); - + public static final String FREETEXT_REQUEST_HANDLER = "/freetext"; /** * Entity type name property key. */ diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py index 5079d4b..f09026f 100755 --- a/distro/src/bin/atlas_config.py +++ b/distro/src/bin/atlas_config.py @@ -32,7 +32,7 @@ LIB = "lib" CONF = "conf" LOG = "logs" WEBAPP = "server" + os.sep + "webapp" -SOLR_CONF_DIR = "conf" + os.sep + "solr" +CONFIG_SETS_CONF = "server" + os.sep + "solr" + os.sep + "configsets" + os.sep + "_default" + os.sep + "conf" DATA = "data" ATLAS_CONF = "ATLAS_CONF" ATLAS_LOG = "ATLAS_LOG_DIR" @@ -112,7 +112,7 @@ def elasticsearchBinDir(dir): return os.environ.get(SOLR_BIN, os.path.join(dir, "elasticsearch", BIN)) def solrConfDir(dir): - return os.environ.get(SOLR_CONF, os.path.join(dir, SOLR_CONF_DIR)) + return os.environ.get(SOLR_CONF, os.path.join(dir, "solr", CONFIG_SETS_CONF)) def solrPort(): return os.environ.get(SOLR_PORT, DEFAULT_SOLR_PORT) diff --git a/distro/src/conf/solr/schema.xml b/distro/src/conf/solr/schema.xml index 3f82ed7..7c2ad3c 100644 --- a/distro/src/conf/solr/schema.xml +++ b/distro/src/conf/solr/schema.xml @@ -524,20 +524,6 @@ class="solr.UUIDField" indexed="true" /> - <fieldType name="freetext" class="solr.TextField" omitNorms="true" - omitTermFreqAndPositions="true"> - <analyzer type="index"> - <tokenizer class="solr.StandardTokenizerFactory"/> - <filter class="solr.LowerCaseFilterFactory"/> - <filter class="solr.EdgeNGramFilterFactory" minGramSize="2" maxGramSize="16" /> - <filter class="solr.RemoveDuplicatesTokenFilterFactory"/> - </analyzer> - <analyzer type="query"> - <tokenizer class="solr.KeywordTokenizerFactory"/> - <filter class="solr.LowerCaseFilterFactory"/> - </analyzer> - </fieldType> - <dynamicField name="*_uuid" type="uuid" indexed="true" stored="true"/> @@ -545,16 +531,4 @@ <field name="ttl" type="string" indexed="true" stored="true" /> <field name="expire_at" type="date" indexed="true" stored="true" /> <field name="timestamp" type="date" indexed="true" stored="true" /> - <field name="allt1s" type="freetext" multiValued="true" indexed="true" stored="false"/> - <field name="allt2s" type="freetext" multiValued="true" indexed="true" stored="false"/> - <field name="allt3s" type="freetext" multiValued="true" indexed="true" stored="false"/> - <field name="allt4s" type="freetext" multiValued="true" indexed="true" stored="false"/> - <field name="allt5s" type="freetext" multiValued="true" indexed="true" stored="false"/> - <field name="allt6s" type="freetext" multiValued="true" indexed="true" stored="false"/> - <field name="allt7s" type="freetext" multiValued="true" indexed="true" stored="false"/> - <field name="allt8s" type="freetext" multiValued="true" indexed="true" stored="false"/> - <field name="allt9s" type="freetext" multiValued="true" indexed="true" stored="false"/> - <field name="allt10s" type="freetext" multiValued="true" indexed="true" stored="false"/> - - </schema> diff --git a/distro/src/conf/solr/solrconfig.xml b/distro/src/conf/solr/solrconfig.xml index a83e03b..21d19ef 100644 --- a/distro/src/conf/solr/solrconfig.xml +++ b/distro/src/conf/solr/solrconfig.xml @@ -478,21 +478,6 @@ </lst> </requestHandler> - <requestHandler name="/freetext" class="solr.SearchHandler"> - <lst name="defaults"> - <str name="defType">edismax</str> - <int name="rows">100</int> - <str name="lowercaseOperators">true</str> - <str name="qf"> - allt10s^10 allt9s^9 allt8s^8 allt7s^7 allt6s^6 allt5s^5 allt4s^4 allt3s^3 allt2s^2 allt1s^1 - </str> - <str name="hl.fl">*</str> - <str name="hl.requireFieldMatch">true</str> - <str name="lowercaseOperators">true</str> - <str name="facet.limit">5</str> - </lst> - </requestHandler> - <!-- The export request handler is used to export full sorted result sets. diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java index 482e6f6..2ced6fb 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java @@ -17,15 +17,18 @@ */ package org.apache.atlas.repository.graphdb; +import java.util.Map; + /** * Represents a graph client work with indices used by Jansgraph. */ public interface AtlasGraphIndexClient { + /** - * The implementers should create a mapping from source propertyName to mapping field name. - * @param indexName the name of index that needs to be worked with. - * @param sourcePropertyName the name of the source attribute. - * @param targetPropertyName the name of the target attribute to which the mapping is getting done. + * The implementers should apply the search weights for the passed in attributes. + * @param collectionName the name of the collection for which the search weight needs to be applied + * @param attributeName2SearchWeightMap the map containing search weights from attribute name to search weights. */ - void createCopyField(String indexName, String sourcePropertyName, String targetPropertyName); + void applySearchWeight(String collectionName, Map<String, Integer> attributeName2SearchWeightMap); + } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java index e973f72..edab08c 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java @@ -196,7 +196,7 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE @Override public AtlasGraphIndexClient getGraphIndexClient() throws AtlasException { try { - return new AtlasJanusGraphSolrIndexClient(); + return new AtlasJanusGraphSolrIndexClient(this); } catch (Exception e) { LOG.error("Error encountered in creating Graph Index Client.", e); throw new AtlasException(e); diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java index f8f8061..a55fc36 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java @@ -17,45 +17,144 @@ */ package org.apache.atlas.repository.graphdb.janus; +import com.google.common.annotations.VisibleForTesting; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient; +import org.apache.atlas.repository.graphdb.AtlasGraphManagement; +import org.apache.atlas.repository.graphdb.AtlasPropertyKey; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.schema.SchemaRequest; -import org.apache.solr.client.solrj.response.schema.SchemaResponse; +import org.apache.solr.client.solrj.request.V2Request; import org.janusgraph.diskstorage.solr.Solr6Index; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Arrays; +import java.util.*; + +import static org.apache.atlas.repository.Constants.FREETEXT_REQUEST_HANDLER; public class AtlasJanusGraphSolrIndexClient implements AtlasGraphIndexClient { private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphSolrIndexClient.class); + private final SolrClient solrClient; + private final AtlasGraph graph; + + + public AtlasJanusGraphSolrIndexClient(AtlasGraph graph) { + // get solr client using same settings as that of Janus Graph + this.solrClient = Solr6Index.getSolrClient(); + this.graph = graph; - public AtlasJanusGraphSolrIndexClient() throws Exception { - //well, this is temp hack to get solr client using same settings as that of Janus Graph - solrClient = Solr6Index.getSolrClient(); if(solrClient == null) { - LOG.warn("The indexing system is not solr based. Non SOLR based indexing systems are not supported yet."); + LOG.warn("Non SOLR index stores are not supported yet."); } } @Override - public void createCopyField(String collectionName, String srcFieldName, String mappedCopyFieldName) { - if(solrClient == null) { - LOG.error("The indexing system is not solr based. Copy fields can not be created in non SOLR based indexing systems. This request will be treated as no op."); + public void applySearchWeight(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) { + //1) try updating request handler + //2) if update fails, try creating request handler + + try { + LOG.info("Attempting to update free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName); + + updateSearchWeights(collectionName, attributeName2SearchWeightMap); + + LOG.info("Successfully updated free text request handler {} for collection {}..", FREETEXT_REQUEST_HANDLER, collectionName); + return; + } catch (Throwable t) { + LOG.warn("Error encountered in updating request handler {} for collection {}. Attempting to create one", FREETEXT_REQUEST_HANDLER, collectionName, t); } - SchemaRequest.AddCopyField addCopyFieldRequest = - new SchemaRequest.AddCopyField(srcFieldName, Arrays.asList(mappedCopyFieldName)); - SchemaResponse.UpdateResponse addCopyFieldResponse = null; + try { - addCopyFieldResponse = addCopyFieldRequest.process(solrClient, collectionName); - } catch (SolrServerException | IOException e) { - String msg = String.format("Error encountered in creating the copy field from %s to %s for collection %s.", srcFieldName, mappedCopyFieldName, collectionName); - LOG.error(msg); - throw new RuntimeException(msg, e); + LOG.info("Attempting to create free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName); + + createFreeTextRequestHandler(collectionName, attributeName2SearchWeightMap); + + LOG.info("Successfully created free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName); + } catch (Throwable t) { + String msg = String.format("Error encountered in creating the request handler '%s' for collection '%s'.", FREETEXT_REQUEST_HANDLER, collectionName); + + LOG.error(msg, t); + + throw new RuntimeException(msg, t); } } + + private void updateSearchWeights(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) { + try { + updateFreeTextRequestHandler(collectionName, attributeName2SearchWeightMap); + } catch (Throwable t) { + String msg = String.format("Error encountered in updating the request handler '%s' for collection '%s'", FREETEXT_REQUEST_HANDLER, collectionName); + + LOG.error(msg, t); + + throw new RuntimeException(msg, t); + } + + LOG.info("Updated free text request handler for collection {}.", collectionName); + } + + private String generateSearchWeightString(AtlasGraphManagement management, String indexName, Map<String, Integer> searchWeightsMap) { + StringBuilder searchWeightBuilder = new StringBuilder(); + Set<Map.Entry<String, Integer>> searchWeightFields = searchWeightsMap.entrySet(); + + for (Map.Entry<String, Integer> entry : searchWeightFields) { + AtlasPropertyKey propertyKey = management.getPropertyKey(entry.getKey()); + String indexFieldName = management.getIndexFieldName(indexName, propertyKey); + + searchWeightBuilder.append(" ") + .append(indexFieldName) + .append("^") + .append(entry.getValue().intValue()); + + } + + return searchWeightBuilder.toString(); + } + + private void updateFreeTextRequestHandler(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException { + String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap); + String payLoadString = generatePayLoadForFreeText("update-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString); + + performRequestHandlerAction(collectionName, solrClient, payLoadString); + } + + private void createFreeTextRequestHandler(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException { + String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap); + String payLoadString = generatePayLoadForFreeText("create-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString); + + performRequestHandlerAction(collectionName, solrClient, payLoadString); + } + + @VisibleForTesting + static String generatePayLoadForFreeText(String action, String handlerName, String qfValue) { + return String.format("{" + + " %s : { " + + " 'name' : '%s', " + + " 'class': 'solr.SearchHandler' , " + + " 'defaults': " + "{" + + " 'defType': 'edismax' , " + + " 'rows': 100 , " + + " 'lowercaseOperators': true , " + + " 'qf': '%s' , " + + " 'hl.fl': '*' , " + + " 'hl.requireFieldMatch': true , " + + " 'lowercaseOperators': true , " + + " }" + + " }" + + "}", action, handlerName, qfValue); + } + + private void performRequestHandlerAction(String collectionName, SolrClient solrClient, + String actionPayLoad) throws IOException, SolrServerException { + V2Request v2Request = new V2Request.Builder(String.format("/collections/%s/config", collectionName)) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(actionPayLoad) + .build(); + v2Request.process(solrClient); + } } diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java index 7b5b675..8c1dbd7 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java @@ -261,7 +261,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { public static class AtlasAttributeDef implements Serializable { private static final long serialVersionUID = 1L; public static final int DEFAULT_SEARCHWEIGHT = -1; - public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3; + public static final String SEARCH_WEIGHT_ATTR_NAME = "searchWeight"; public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference"; private final String STRING_TRUE = "true"; diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index dadc3c5..56655a8 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -23,7 +23,6 @@ import com.google.common.base.Preconditions; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; -import org.apache.atlas.discovery.SearchContext; import org.apache.atlas.discovery.SearchIndexer; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.ha.HAConfiguration; @@ -59,8 +58,6 @@ import java.util.List; import java.util.Set; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*; -import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT; -import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT_FOR_STRINGS; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST; import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET; @@ -92,15 +89,27 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang // Added for type lookup when indexing the new typedefs private final AtlasTypeRegistry typeRegistry; - - private final AtlasGraphIndexClient atlasGraphIndexClient; + private final List<IndexChangeListener> indexChangeListeners = new ArrayList<>(); //allows injection of a dummy graph for testing private IAtlasGraphProvider provider; private boolean recomputeIndexedKeys = true; private Set<String> vertexIndexKeys = new HashSet<>(); - private final String[] mappedCopyFieldNames; + + public static boolean isValidSearchWeight(int searchWeight) { + if (searchWeight != -1 ) { + if (searchWeight < 1 || searchWeight > 10) { + return false; + } + } + return true; + } + + public static boolean isStringAttribute(AtlasStructDef.AtlasAttributeDef attributeDef) { + return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeDef.getTypeName()); + } + public enum UniqueKind { NONE, GLOBAL_UNIQUE, PER_TYPE_UNIQUE } @Inject @@ -113,23 +122,19 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang throws IndexException, RepositoryException { this.provider = provider; this.typeRegistry = typeRegistry; - - mappedCopyFieldNames = new String[11]; - for(int i=0; i < mappedCopyFieldNames.length; i++) { - mappedCopyFieldNames[i] = String.format("allt%ds", i); - } - - try { - atlasGraphIndexClient = provider.get().getGraphIndexClient(); - } catch (Exception e) { - LOG.error("Error encountered in creating solr client.", e); - throw new RepositoryException("Error encountered in creating solr client.", e); - } + //make sure solr index follows graph backed index listener + addIndexListener(new SolrIndexHelper(typeRegistry)); if (!HAConfiguration.isHAEnabled(configuration)) { initialize(provider.get()); } + + } + + public void addIndexListener(IndexChangeListener listener) { + indexChangeListeners.add(listener); } + /** * Initialize global indices for JanusGraph on server activation. * @@ -191,7 +196,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang LOG.error("Failed to update indexes for changed typedefs", e); attemptRollback(changedTypeDefs, management); } - + notifyChangeListeners(); } public Set<String> getVertexIndexKeys() { @@ -283,7 +288,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false); createVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false); createVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false); - createVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false, 10); + createVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false); createVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false); createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true); createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true); @@ -356,12 +361,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang LOG.info("Completed deleting indexes for type {}", typeDef.getName()); } - private static boolean isStringAttribute(AtlasStructDef.AtlasAttributeDef attributeDef) { - return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeDef.getTypeName()); - } - private void createIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) { - final String propertyName = AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName()); + final String propertyName = getEncodedPropertyName(typeName, attributeDef); AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality()); boolean isUnique = attributeDef.getIsUnique(); boolean isIndexable = attributeDef.getIsIndexable(); @@ -370,22 +371,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang boolean isArrayType = isArrayType(attribTypeName); boolean isMapType = isMapType(attribTypeName); final String uniqPropName = isUnique ? AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + attributeDef.getName()) : null; - int searchWeight = attributeDef.getSearchWeight(); - - if(attributeDef.getSearchWeight() == DEFAULT_SEARCHWEIGHT) { - if (isStringAttribute(attributeDef)) { - //We will use default search weight of 3 for string attributes. - //this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher. - LOG.info("Applying default search weight of {} for attribute{}.", DEFAULT_SEARCHWEIGHT_FOR_STRINGS, propertyName); - searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS; - } - } - - if(!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) { - String msg = String.format("Invalid search weight '%d' was provided for property %s.", searchWeight, propertyName); - LOG.error(msg); - throw new RuntimeException(msg); - } try { AtlasType atlasType = typeRegistry.getType(typeName); @@ -424,7 +409,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang if (isRelationshipType(atlasType)) { createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false); } else { - createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false, searchWeight); + createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false); if (uniqPropName != null) { createVertexIndex(management, uniqPropName, UniqueKind.PER_TYPE_UNIQUE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, true); @@ -463,6 +448,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang } } + /** + * gets the encoded property name for the attribute passed in. + * @param typeName the type system of the attribute + * @param attributeDef the attribute definition + * @return the encoded property name for the attribute passed in. + */ + public static String getEncodedPropertyName(String typeName, AtlasAttributeDef attributeDef) { + return AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName()); + } + private void createLabelIfNeeded(final AtlasGraphManagement management, final String propertyName, final String attribTypeName) { // If any of the referenced typename is of type Entity or Struct then the edge label needs to be created for (String typeName : AtlasTypeUtil.getReferencedTypeNames(attribTypeName)) { @@ -564,10 +559,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang public void createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass, AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes) { - createVertexIndex(management, propertyName, uniqueKind, propertyClass, cardinality, createCompositeIndex, createCompositeIndexWithTypeAndSuperTypes, -1); - } - private void createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass, - AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes, int searchWeight) { if (propertyName != null) { AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName); @@ -585,16 +576,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang } } - //for now, we will support free text based enhancements for SOLR backed index systems only. - //WARNING--it is suggested that we don't change the search weight once is assigned with a number in the range 1 to 10. - //otherwise, we might end up mapping the index to multiple copy fields...unwanted to space usage. - if( searchWeight != -1 && SearchContext.isIndexSolrBased()) { - String encodedPropertyName = management.getIndexFieldName(VERTEX_INDEX, propertyKey); - String mappedCopyFieldName = mappedCopyFieldNames[searchWeight]; - atlasGraphIndexClient.createCopyField(VERTEX_INDEX, encodedPropertyName, mappedCopyFieldName); - LOG.info("Created copy field from {} to {} for collection {} for property {}.", encodedPropertyName, mappedCopyFieldName, VERTEX_INDEX, propertyName); - } - if (propertyKey != null) { if (createCompositeIndex || uniqueKind == UniqueKind.GLOBAL_UNIQUE || uniqueKind == UniqueKind.PER_TYPE_UNIQUE) { createVertexCompositeIndex(management, propertyClass, propertyKey, uniqueKind == UniqueKind.GLOBAL_UNIQUE); @@ -620,7 +601,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang if (LOG.isDebugEnabled()) { LOG.debug("Creating vertex-centric index for edge label: {} direction: {} for property: {} of type: {} ", - edgeLabel, edgeDirection.name(), propertyName, propertyClass.getName()); + edgeLabel, edgeDirection.name(), propertyName, propertyClass.getName()); } final String indexName = edgeLabel + propertyKey.getName(); @@ -824,12 +805,15 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang LOG.info("Index creation for type {} complete", typeDef.getName()); } - public static boolean isValidSearchWeight(int searchWeight) { - if(searchWeight != -1 ) { - if(searchWeight < 1 || searchWeight > 10) { - return false; + private void notifyChangeListeners() { + for (IndexChangeListener indexChangeListener : indexChangeListeners) { + try { + indexChangeListener.onChange(); + } catch (Throwable t) { + LOG.error("Error encountered in notifying the index change listener {}.", indexChangeListener.getClass().getName(), t); + //we need to throw exception if any of the listeners throw execption. + throw new RuntimeException("Error encountered in notifying the index change listener " + indexChangeListener.getClass().getName(), t); } } - return true; } } diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java new file mode 100644 index 0000000..9cde947 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graph; + +public interface IndexChangeListener { + void onChange(); +} diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java new file mode 100644 index 0000000..20a517f --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.graph; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.discovery.SearchContext; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT; +import static org.apache.atlas.repository.Constants.CLASSIFICATION_TEXT_KEY; + +/** + This is a component that will go through all entity type definitions and create free text index + request handler with SOLR. This is a no op class in non-solr index based deployments. + This component needs to be initialized after type definitions are completely fixed with the needed patches (Ordder 3 initialization). + */ +public class SolrIndexHelper implements IndexChangeListener { + private static final Logger LOG = LoggerFactory.getLogger(SolrIndexHelper.class); + + public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3; + + private final AtlasTypeRegistry typeRegistry; + + + public SolrIndexHelper(AtlasTypeRegistry typeRegistry) { + this.typeRegistry = typeRegistry; + } + + @Override + public void onChange() { + LOG.info("SolrIndexHelper.onChange()"); + + if(!SearchContext.isIndexSolrBased()) { + LOG.warn("Not a Solr based index store. Free text search is not supported"); + + return; + } + + try { + AtlasGraph atlasGraph = AtlasGraphProvider.getGraphInstance(); + AtlasGraphIndexClient atlasGraphIndexClient = atlasGraph.getGraphIndexClient(); + Map<String, Integer> attributeName2SearchWeightMap = getAttributesWithSearchWeights(); + + atlasGraphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, attributeName2SearchWeightMap); + } catch (AtlasException e) { + LOG.error("Error encountered in handling type system change notification.", e); + throw new RuntimeException("Error encountered in handling type system change notification.", e); + } + } + + private Map<String, Integer> getAttributesWithSearchWeights() { + Map<String, Integer> attributesWithSearchWeights = new HashMap<>(); + Collection<AtlasEntityDef> allEntityDefs = typeRegistry.getAllEntityDefs(); + + attributesWithSearchWeights.put(CLASSIFICATION_TEXT_KEY,10); + + if (CollectionUtils.isNotEmpty(allEntityDefs)) { + for (AtlasEntityDef entityDef : allEntityDefs) { + processEntity(attributesWithSearchWeights, entityDef); + } + } + + return attributesWithSearchWeights; + } + + private void processEntity(Map<String, Integer> attributesWithSearchWeights, AtlasEntityDef entityDef) { + for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) { + processAttributeDefinition(attributesWithSearchWeights, entityDef, attributeDef); + } + } + + private void processAttributeDefinition(Map<String, Integer> attributesWithSearchWeights, AtlasEntityDef entityDef, AtlasAttributeDef attributeDef) { + if (GraphBackedSearchIndexer.isStringAttribute(attributeDef)) { + final String attributeName = GraphBackedSearchIndexer.getEncodedPropertyName(entityDef.getName(), attributeDef); + int searchWeight = attributeDef.getSearchWeight(); + + if (searchWeight == DEFAULT_SEARCHWEIGHT) { + //We will use default search weight of 3 for string attributes. + //this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher. + searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS; + } else if (!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) { //validate the value provided in the model. + String msg = String.format("Invalid search weight '%d' for attribute %s.%s", searchWeight, entityDef.getName(), attributeName); + + LOG.error(msg); + + throw new RuntimeException(msg); + } + + LOG.info("Applying search weight {} for attribute {}.{}", searchWeight, entityDef.getName(), attributeName); + + attributesWithSearchWeights.put(attributeName, searchWeight); + } + } +} \ No newline at end of file