Skip to content

A
atlas
Commit b0063fa5 by Sridhar K Committed by Sarath Subramanian
Options

ATLAS-3100 Enhance Full Text Search backend implementation to use SOLR request handler.

Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org>
parent 8fef5353
Showing with 1235 additions and 131 deletions
addons/models/0000-Area0/patches/003-base_model_add_searchWeight.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_0000_033",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "Referenceable",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "qualifiedName",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_0000_034",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "Asset",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "name",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_0000_035",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "Asset",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "description",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_0000_036",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "Asset",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "owner",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_0000_037",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "AtlasServer",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "name",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_0000_038",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "AtlasServer",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "displayName",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_0000_039",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "AtlasServer",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "fullName",
"params": {
"searchWeight": 10
}
}
]
}
addons/models/1000-Hadoop/1090-impala_model.json
...@@ -49,7 +49,8 @@ ...@@ -49,7 +49,8 @@
"cardinality": "SINGLE", "cardinality": "SINGLE",
"isIndexable": false, "isIndexable": false,
"isOptional": false, "isOptional": false,
"isUnique": false "isUnique": false,
"searchWeight": 10
}, },
{ {
"name": "queryPlan", "name": "queryPlan",
...@@ -82,7 +83,8 @@ ...@@ -82,7 +83,8 @@
"isIndexable": false, "isIndexable": false,
"isOptional": true, "isOptional": true,
"includeInNotification": true, "includeInNotification": true,
"isUnique": false "isUnique": false,
"searchWeight": 10
}, },
{ {
"name": "queryGraph", "name": "queryGraph",
...@@ -150,7 +152,8 @@ ...@@ -150,7 +152,8 @@
"cardinality": "SINGLE", "cardinality": "SINGLE",
"isIndexable": false, "isIndexable": false,
"isOptional": false, "isOptional": false,
"isUnique": false "isUnique": false,
"searchWeight": 10
}, },
{ {
"name": "queryGraph", "name": "queryGraph",
......
addons/models/2000-RDBMS/patches/004-add_searchweight.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_2000_127",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "rdbms_type",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_128",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "platform",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_129",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "contact_info",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_130",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "hostname",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_2000_131",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_2000_132",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_db",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "contact_info",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_133",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "name_path",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_2000_134",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_2000_135",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "contact_info",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_136",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_column",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_2000_137",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_index",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}
]
}
addons/models/3000-Cloud/patches/002-cloud_model_add_searchweight.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_3000_138",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_pseudo_dir",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "subject",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_3000_139",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_pseudo_dir",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "objectPrefix",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_3000_140",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_pseudo_dir",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "currentStorageLocation",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_3000_141",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_pseudo_dir",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "subdomain",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_3000_142",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_object",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "partner",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_3000_143",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_object",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "domain",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_3000_144",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_object",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "region",
"params": {
"searchWeight": 8
}
}
]
}
common/src/main/java/org/apache/atlas/repository/Constants.java
...@@ -81,9 +81,11 @@ public final class Constants { ...@@ -81,9 +81,11 @@ public final class Constants {
public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames"); public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames"); public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
public static final String VERSION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version"); public static final String VERSION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version");
public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state"); public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy"); public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy"); public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
/** /**
......
distro/src/conf/solr/schema.xml
...@@ -524,11 +524,37 @@ ...@@ -524,11 +524,37 @@
class="solr.UUIDField" class="solr.UUIDField"
indexed="true" /> indexed="true" />
<fieldType name="freetext" class="solr.TextField" omitNorms="true"
omitTermFreqAndPositions="true">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.EdgeNGramFilterFactory" minGramSize="2" maxGramSize="16" />
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.KeywordTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
<dynamicField name="*_uuid" type="uuid" indexed="true" stored="true"/> <dynamicField name="*_uuid" type="uuid" indexed="true" stored="true"/>
<!-- TTL --> <!-- TTL -->
<field name="ttl" type="string" indexed="true" stored="true" /> <field name="ttl" type="string" indexed="true" stored="true" />
<field name="expire_at" type="date" indexed="true" stored="true" /> <field name="expire_at" type="date" indexed="true" stored="true" />
<field name="timestamp" type="date" indexed="true" stored="true" /> <field name="timestamp" type="date" indexed="true" stored="true" />
<field name="allt1s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt2s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt3s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt4s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt5s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt6s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt7s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt8s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt9s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt10s" type="freetext" multiValued="true" indexed="true" stored="false"/>
</schema> </schema>
distro/src/conf/solr/solrconfig.xml
...@@ -110,9 +110,13 @@ ...@@ -110,9 +110,13 @@
When ManagedIndexSchemaFactory is specified with mutable = true, schema When ManagedIndexSchemaFactory is specified with mutable = true, schema
modification REST API calls will be allowed; otherwise, error responses will be modification REST API calls will be allowed; otherwise, error responses will be
sent back for these requests. sent back for these requests.
<schemaFactory class="ClassicIndexSchemaFactory"/>
--> -->
<schemaFactory class="ClassicIndexSchemaFactory"/> <schemaFactory class="ManagedIndexSchemaFactory">
<bool name="mutable">true</bool>
<str name="managedSchemaResourceName">managed-schema</str>
</schemaFactory>
<!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Index Config - These settings control low-level behavior of indexing Index Config - These settings control low-level behavior of indexing
...@@ -474,6 +478,22 @@ ...@@ -474,6 +478,22 @@
</lst> </lst>
</requestHandler> </requestHandler>
<requestHandler name="/freetext" class="solr.SearchHandler">
<lst name="defaults">
<str name="defType">edismax</str>
<int name="rows">100</int>
<str name="lowercaseOperators">true</str>
<str name="qf">
allt10s^10 allt9s^9 allt8s^8 allt7s^7 allt6s^6 allt5s^5 allt4s^4 allt3s^3 allt2s^2 allt1s^1
</str>
<str name="hl.fl">*</str>
<str name="hl.requireFieldMatch">true</str>
<str name="lowercaseOperators">true</str>
<str name="facet.limit">5</str>
</lst>
</requestHandler>
<!-- <!--
The export request handler is used to export full sorted result sets. The export request handler is used to export full sorted result sets.
Do not change these defaults. Do not change these defaults.
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
*/ */
package org.apache.atlas.repository.graphdb; package org.apache.atlas.repository.graphdb;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression; import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasType;
...@@ -177,6 +178,14 @@ public interface AtlasGraph<V, E> { ...@@ -177,6 +178,14 @@ public interface AtlasGraph<V, E> {
AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString, int offset); AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString, int offset);
/** /**
* Creates an index query.
*
* @param indexQueryParameters the parameterObject containing the information needed for creating the index.
*
*/
AtlasIndexQuery<V, E> indexQuery(GraphIndexQueryParameters indexQueryParameters);
/**
* Gets the management object associated with this graph and opens a transaction * Gets the management object associated with this graph and opens a transaction
* for changes that are made. * for changes that are made.
* @return * @return
...@@ -329,4 +338,26 @@ public interface AtlasGraph<V, E> { ...@@ -329,4 +338,26 @@ public interface AtlasGraph<V, E> {
* @return * @return
*/ */
boolean isMultiProperty(String name); boolean isMultiProperty(String name);
/**
* return the encoded name used for the attribute identified by property key and index name.
* @param propertyKey the property key of attributed
* @param indexName the name of the index containing the property.
* @return the encoded name of the property.
*/
String getIndexFieldName(AtlasPropertyKey propertyKey, String indexName);
/**
* Create Index query parameter for use with atlas graph.
* @param parameterName the name of the parameter that needs to be passed to index layer.
* @param parameterValue the value of the paratemeter that needs to be passed to the index layer.
* @return
*/
AtlasIndexQueryParameter indexQueryParameter(String parameterName, String parameterValue);
/**
* Implementors should return graph index client.
* @return the graph index client
* @throws AtlasException when error encountered in creating the client.
*/
AtlasGraphIndexClient getGraphIndexClient()throws AtlasException;
} }
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb;
/**
* Represents a graph client work with indices used by Jansgraph.
*/
public interface AtlasGraphIndexClient {
/**
* The implementers should create a mapping from source propertyName to mapping field name.
* @param indexName the name of index that needs to be worked with.
* @param sourcePropertyName the name of the source attribute.
* @param targetPropertyName the name of the target attribute to which the mapping is getting done.
*/
void createCopyField(String indexName, String sourcePropertyName, String targetPropertyName);
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
...@@ -157,4 +157,12 @@ public interface AtlasGraphManagement { ...@@ -157,4 +157,12 @@ public interface AtlasGraphManagement {
* @param propertyKey * @param propertyKey
*/ */
void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey); void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
/**
* Gets the index field name for the vertex property.
* @param indexName
* @param propertyKey
* @return the encoded name for the index
*/
String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey);
} }
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb;
import java.util.Iterator;
/**
* Represents an index query parameter for use with AtlasGraph queries.
*/
public interface AtlasIndexQueryParameter {
/**
* @return the name of the parameter for index search.
*/
public String getParameterName();
/**
* @return the value of the parameter for index search.
*/
public String getParameterValue();
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphIndexQueryParameters.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb;
import java.util.List;
/**
* A class wrapping the parameters that need to create a graph index query.
*/
public class GraphIndexQueryParameters {
final String indexName;
final String graphQueryString;
final int offset;
final List<AtlasIndexQueryParameter> indexQueryParameters;
/**
*
* @param indexName the name of the index on which the query is being made
* @param graphQueryString the graph query string.
* @param offset the offset to be used for data fetch.
* @param indexQueryParameters any index system specific parameters for use in query.
*/
public GraphIndexQueryParameters(String indexName, String graphQueryString, int offset, List<AtlasIndexQueryParameter> indexQueryParameters) {
this.indexName = indexName;
this.graphQueryString = graphQueryString;
this.offset = offset;
this.indexQueryParameters = indexQueryParameters;
}
public String getIndexName() {
return indexName;
}
public String getGraphQueryString() {
return graphQueryString;
}
public int getOffset() {
return offset;
}
public List<AtlasIndexQueryParameter> getIndexQueryParameters() {
return indexQueryParameters;
}
}
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
...@@ -25,15 +25,7 @@ import org.apache.atlas.AtlasErrorCode; ...@@ -25,15 +25,7 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression; import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphTraversal;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.repository.graphdb.janus.query.AtlasJanusGraphQuery; import org.apache.atlas.repository.graphdb.janus.query.AtlasJanusGraphQuery;
import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter; import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasType;
...@@ -59,20 +51,18 @@ import org.janusgraph.core.PropertyKey; ...@@ -59,20 +51,18 @@ import org.janusgraph.core.PropertyKey;
import org.janusgraph.core.SchemaViolationException; import org.janusgraph.core.SchemaViolationException;
import org.janusgraph.core.schema.JanusGraphIndex; import org.janusgraph.core.schema.JanusGraphIndex;
import org.janusgraph.core.schema.JanusGraphManagement; import org.janusgraph.core.schema.JanusGraphManagement;
import org.janusgraph.core.schema.Parameter;
import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.graphdb.database.StandardJanusGraph; import org.janusgraph.graphdb.database.StandardJanusGraph;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.script.Bindings; import javax.script.Bindings;
import javax.script.ScriptEngine; import javax.script.ScriptEngine;
import javax.script.ScriptException; import javax.script.ScriptException;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.Collection; import java.util.*;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.StreamSupport; import java.util.stream.StreamSupport;
...@@ -84,12 +74,13 @@ import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase. ...@@ -84,12 +74,13 @@ import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase.
* Janus implementation of AtlasGraph. * Janus implementation of AtlasGraph.
*/ */
public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> { public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraph.class);
private static Configuration APPLICATION_PROPERTIES = null; private static Configuration APPLICATION_PROPERTIES = null;
private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction(); private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
private final Set<String> multiProperties = new HashSet<>(); private final Set<String> multiProperties = new HashSet<>();
private final StandardJanusGraph janusGraph; private final StandardJanusGraph janusGraph;
private final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0];
public AtlasJanusGraph() { public AtlasJanusGraph() {
this(getGraphInstance()); this(getGraphInstance());
...@@ -198,6 +189,21 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE ...@@ -198,6 +189,21 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
} }
@Override @Override
public AtlasIndexQueryParameter indexQueryParameter(String parameterName, String parameterValue) {
return new AtlasJanusIndexQueryParameter(parameterName, parameterValue);
}
@Override
public AtlasGraphIndexClient getGraphIndexClient() throws AtlasException {
try {
return new AtlasJanusGraphSolrIndexClient();
} catch (Exception e) {
LOG.error("Error encountered in creating Graph Index Client.", e);
throw new AtlasException(e);
}
}
@Override
public void commit() { public void commit() {
getGraph().tx().commit(); getGraph().tx().commit();
} }
...@@ -208,15 +214,34 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE ...@@ -208,15 +214,34 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
} }
@Override @Override
public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String fulltextIndex, String graphQuery) { public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String indexName, String graphQuery) {
return indexQuery(fulltextIndex, graphQuery, 0); return indexQuery(indexName, graphQuery, 0, null);
} }
@Override @Override
public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String fulltextIndex, String graphQuery, int offset) { public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String indexName, String graphQuery, int offset) {
return indexQuery(indexName, graphQuery, offset, null);
}
/**
* Creates an index query.
*
* @param indexQueryParameters the parameterObject containing the information needed for creating the index.
*
*/
public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(GraphIndexQueryParameters indexQueryParameters) {
return indexQuery(indexQueryParameters.getIndexName(), indexQueryParameters.getGraphQueryString(), indexQueryParameters.getOffset(), indexQueryParameters.getIndexQueryParameters());
}
private AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String indexName, String graphQuery, int offset, List<AtlasIndexQueryParameter> indexQueryParameterList) {
String prefix = getIndexQueryPrefix(); String prefix = getIndexQueryPrefix();
JanusGraphIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery).setElementIdentifier(prefix).offset(offset); JanusGraphIndexQuery query = getGraph().indexQuery(indexName, graphQuery).setElementIdentifier(prefix).offset(offset);
if(indexQueryParameterList != null && indexQueryParameterList.size() > 0) {
for(AtlasIndexQueryParameter indexQueryParameter: indexQueryParameterList) {
query = query.addParameter(new Parameter(indexQueryParameter.getParameterName(), indexQueryParameter.getParameterValue()));
}
}
return new AtlasJanusIndexQuery(this, query); return new AtlasJanusIndexQuery(this, query);
} }
...@@ -497,4 +522,9 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE ...@@ -497,4 +522,9 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
return convertGremlinValue(input); return convertGremlinValue(input);
} }
} }
public String getIndexFieldName(AtlasPropertyKey propertyKey, String indexName) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey, EMPTY_PARAMETER_ARRAY, indexName);
}
} }
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
...@@ -200,6 +200,14 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement { ...@@ -200,6 +200,14 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
JanusGraphIndex vertexIndex = management.getGraphIndex(indexName); JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
management.addIndexKey(vertexIndex, janusKey); management.addIndexKey(vertexIndex, janusKey);
String encodedName = graph.getIndexFieldName(propertyKey, vertexIndex.getBackingIndex());
LOG.info("property '{}' is encoded to '{}'.", propertyKey.getName(), encodedName);
}
@Override
public String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey) {
JanusGraphIndex index = management.getGraphIndex(indexName);
return graph.getIndexFieldName(propertyKey, index.getBackingIndex());
} }
@Override @Override
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.janus;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
public class AtlasJanusGraphSolrIndexClient implements AtlasGraphIndexClient {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphSolrIndexClient.class);
private final SolrClient solrClient;
public AtlasJanusGraphSolrIndexClient() throws Exception {
//well, this is temp hack to get solr client using same settings as that of Janus Graph
solrClient = Solr6Index.getSolrClient();
if(solrClient == null) {
LOG.warn("The indexing system is not solr based. Non SOLR based indexing systems are not supported yet.");
}
}
@Override
public void createCopyField(String collectionName, String srcFieldName, String mappedCopyFieldName) {
if(solrClient == null) {
LOG.error("The indexing system is not solr based. Copy fields can not be created in non SOLR based indexing systems. This request will be treated as no op.");
return;
}
SchemaRequest.AddCopyField addCopyFieldRequest =
new SchemaRequest.AddCopyField(srcFieldName, Arrays.asList(mappedCopyFieldName));
SchemaResponse.UpdateResponse addCopyFieldResponse = null;
try {
addCopyFieldResponse = addCopyFieldRequest.process(solrClient, collectionName);
} catch (SolrServerException | IOException e) {
String msg = String.format("Error encountered in creating the copy field from %s to %s for collection %s.", srcFieldName, mappedCopyFieldName, collectionName);
LOG.error(msg);
throw new RuntimeException(msg, e);
}
}
}
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQueryParameter.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.janus;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQueryParameter;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.janusgraph.core.JanusGraphIndexQuery;
import org.janusgraph.core.JanusGraphVertex;
import java.util.Iterator;
import java.util.List;
/**
* Janus implementation of AtlasIndexQueryParameter.
*/
public class AtlasJanusIndexQueryParameter implements AtlasIndexQueryParameter {
private final String parameterName;
private final String parameterValue;
public AtlasJanusIndexQueryParameter(String parameterName, String parameterValue) {
this.parameterName = parameterName;
this.parameterValue = parameterValue;
}
@Override
public String getParameterName() {
return parameterName;
}
@Override
public String getParameterValue() {
return parameterValue;
}
}
graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
...@@ -135,9 +135,9 @@ public class Solr6Index implements IndexProvider { ...@@ -135,9 +135,9 @@ public class Solr6Index implements IndexProvider {
private static final Logger logger = LoggerFactory.getLogger(Solr6Index.class); private static final Logger logger = LoggerFactory.getLogger(Solr6Index.class);
private static final String DEFAULT_ID_FIELD = "id"; private static final String DEFAULT_ID_FIELD = "id";
private static final char CHROOT_START_CHAR = '/'; private static final char CHROOT_START_CHAR = '/';
private static Configuration config;
private enum Mode { private enum Mode {
HTTP, CLOUD; HTTP, CLOUD;
...@@ -181,7 +181,7 @@ public class Solr6Index implements IndexProvider { ...@@ -181,7 +181,7 @@ public class Solr6Index implements IndexProvider {
public Solr6Index(final Configuration config) throws BackendException { public Solr6Index(final Configuration config) throws BackendException {
// Add Kerberos-enabled SolrHttpClientBuilder // Add Kerberos-enabled SolrHttpClientBuilder
HttpClientUtil.setHttpClientBuilder(new Krb5HttpClientBuilder().getBuilder()); HttpClientUtil.setHttpClientBuilder(new Krb5HttpClientBuilder().getBuilder());
initConfiguration(config);
Preconditions.checkArgument(config!=null); Preconditions.checkArgument(config!=null);
configuration = config; configuration = config;
mode = Mode.parse(config.get(SOLR_MODE)); mode = Mode.parse(config.get(SOLR_MODE));
...@@ -200,23 +200,39 @@ public class Solr6Index implements IndexProvider { ...@@ -200,23 +200,39 @@ public class Solr6Index implements IndexProvider {
logger.debug("KERBEROS_ENABLED name is " + KERBEROS_ENABLED.getName() + " and it is" + (KERBEROS_ENABLED.isOption() ? " " : " not") + " an option."); logger.debug("KERBEROS_ENABLED name is " + KERBEROS_ENABLED.getName() + " and it is" + (KERBEROS_ENABLED.isOption() ? " " : " not") + " an option.");
logger.debug("KERBEROS_ENABLED type is " + KERBEROS_ENABLED.getType().name()); logger.debug("KERBEROS_ENABLED type is " + KERBEROS_ENABLED.getType().name());
} }
solrClient = getSolrClient();
}
private static void initConfiguration(Configuration config) {
if(Solr6Index.config == null) {
Solr6Index.config = config;
}
}
public static SolrClient getSolrClient() {
final ModifiableSolrParams clientParams = new ModifiableSolrParams(); final ModifiableSolrParams clientParams = new ModifiableSolrParams();
SolrClient solrClient = null;
if(Solr6Index.config == null) {
logger.error("The solr client is not being used for the indexing purposes.");
return null;
}
Configuration config = Solr6Index.config;
Mode mode = Mode.parse(config.get(SOLR_MODE));
switch (mode) { switch (mode) {
case CLOUD: case CLOUD:
/* ATLAS-2920: Update JanusGraph Solr clients to use all zookeeper entries – start */
final List<String> zookeeperUrls = getZookeeperURLs(config);
/* ATLAS-2920: end */
final CloudSolrClient cloudServer = new CloudSolrClient.Builder() final CloudSolrClient cloudServer = new CloudSolrClient.Builder()
.withLBHttpSolrClientBuilder( .withLBHttpSolrClientBuilder(
new LBHttpSolrClient.Builder() new LBHttpSolrClient.Builder()
.withHttpSolrClientBuilder(new HttpSolrClient.Builder().withInvariantParams(clientParams)) .withHttpSolrClientBuilder(new HttpSolrClient.Builder().withInvariantParams(clientParams))
.withBaseSolrUrls(config.get(HTTP_URLS)) .withBaseSolrUrls(config.get(HTTP_URLS))
) )
.withZkHost(zookeeperUrls) .withZkHost(getZookeeperURLs(config))
.sendUpdatesOnlyToShardLeaders() .sendUpdatesOnlyToShardLeaders()
.build(); .build();
cloudServer.connect(); cloudServer.connect();
solrClient = cloudServer; solrClient = cloudServer;
logger.info("Created solr client using Cloud based configuration.");
break; break;
case HTTP: case HTTP:
clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION, config.get(HTTP_ALLOW_COMPRESSION).toString()); clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION, config.get(HTTP_ALLOW_COMPRESSION).toString());
...@@ -228,12 +244,12 @@ public class Solr6Index implements IndexProvider { ...@@ -228,12 +244,12 @@ public class Solr6Index implements IndexProvider {
.withHttpClient(client) .withHttpClient(client)
.withBaseSolrUrls(config.get(HTTP_URLS)) .withBaseSolrUrls(config.get(HTTP_URLS))
.build(); .build();
logger.info("Created solr client using HTTP based configuration.");
break; break;
default: default:
throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode); throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
} }
return solrClient;
} }
private void configureSolrClientsForKerberos() throws PermanentBackendException { private void configureSolrClientsForKerberos() throws PermanentBackendException {
...@@ -1146,7 +1162,7 @@ public class Solr6Index implements IndexProvider { ...@@ -1146,7 +1162,7 @@ public class Solr6Index implements IndexProvider {
} }
/* ATLAS-2920: Update JanusGraph Solr clients to use all zookeeper entries – start */ /* ATLAS-2920: Update JanusGraph Solr clients to use all zookeeper entries – start */
private List<String> getZookeeperURLs(Configuration config) { private static List<String> getZookeeperURLs(Configuration config) {
List<String> ret = null; List<String> ret = null;
String[] zkHosts = config.get(ZOOKEEPER_URLS); String[] zkHosts = config.get(ZOOKEEPER_URLS);
......
intg/src/main/java/org/apache/atlas/ApplicationProperties.java
...@@ -42,20 +42,22 @@ public final class ApplicationProperties extends PropertiesConfiguration { ...@@ -42,20 +42,22 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static final String ATLAS_CONFIGURATION_DIRECTORY_PROPERTY = "atlas.conf"; public static final String ATLAS_CONFIGURATION_DIRECTORY_PROPERTY = "atlas.conf";
private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class); private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class);
public static final String APPLICATION_PROPERTIES = "atlas-application.properties"; public static final String APPLICATION_PROPERTIES = "atlas-application.properties";
public static final String GRAPHDB_BACKEND_CONF = "atlas.graphdb.backend"; public static final String GRAPHDB_BACKEND_CONF = "atlas.graphdb.backend";
public static final String STORAGE_BACKEND_CONF = "atlas.graph.storage.backend"; public static final String STORAGE_BACKEND_CONF = "atlas.graph.storage.backend";
public static final String INDEX_BACKEND_CONF = "atlas.graph.index.search.backend"; public static final String INDEX_BACKEND_CONF = "atlas.graph.index.search.backend";
public static final String INDEX_MAP_NAME_CONF = "atlas.graph.index.search.map-name"; public static final String INDEX_MAP_NAME_CONF = "atlas.graph.index.search.map-name";
public static final String SOLR_WAIT_SEARCHER_CONF = "atlas.graph.index.search.solr.wait-searcher"; public static final String FREE_TEXT_INDEX_ENABLED = "atlas.freetext.index.enabled";
public static final String GRAPHBD_BACKEND_JANUS = "janus"; public static final String SOLR_WAIT_SEARCHER_CONF = "atlas.graph.index.search.solr.wait-searcher";
public static final String STORAGE_BACKEND_HBASE = "hbase"; public static final String GRAPHBD_BACKEND_JANUS = "janus";
public static final String STORAGE_BACKEND_HBASE2 = "hbase2"; public static final String STORAGE_BACKEND_HBASE = "hbase";
public static final String INDEX_BACKEND_SOLR = "solr"; public static final String STORAGE_BACKEND_HBASE2 = "hbase2";
public static final String DEFAULT_GRAPHDB_BACKEND = GRAPHBD_BACKEND_JANUS; public static final String INDEX_BACKEND_SOLR = "solr";
public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true; public static final String DEFAULT_GRAPHDB_BACKEND = GRAPHBD_BACKEND_JANUS;
public static final boolean DEFAULT_INDEX_MAP_NAME = false; public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true;
public static final boolean DEFAULT_INDEX_MAP_NAME = false;
public static final boolean DEFAULT_FREE_TEXT_INDEX_ENABLED = false;
public static final SimpleEntry<String, String> DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true"); public static final SimpleEntry<String, String> DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true");
public static final SimpleEntry<String, String> DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20"); public static final SimpleEntry<String, String> DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20");
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
...@@ -259,10 +259,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -259,10 +259,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
@XmlRootElement @XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY) @XmlAccessorType(XmlAccessType.PROPERTY)
public static class AtlasAttributeDef implements Serializable { public static class AtlasAttributeDef implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public static final int DEFAULT_SEARCHWEIGHT = -1;
public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3;
public static final String SEARCH_WEIGHT_ATTR_NAME = "searchWeight";
public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference"; public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference";
private final String STRING_TRUE = "true"; private final String STRING_TRUE = "true";
/** /**
* single-valued attribute or multi-valued attribute. * single-valued attribute or multi-valued attribute.
...@@ -282,6 +284,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -282,6 +284,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
private boolean includeInNotification; private boolean includeInNotification;
private String defaultValue; private String defaultValue;
private String description; private String description;
private int searchWeight = DEFAULT_SEARCHWEIGHT;
private List<AtlasConstraintDef> constraints; private List<AtlasConstraintDef> constraints;
private Map<String, String> options; private Map<String, String> options;
...@@ -289,21 +292,36 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -289,21 +292,36 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
public AtlasAttributeDef() { this(null, null); } public AtlasAttributeDef() { this(null, null); }
public AtlasAttributeDef(String name, String typeName) { public AtlasAttributeDef(String name, String typeName) {
this(name, typeName, false, Cardinality.SINGLE); this(name, typeName, DEFAULT_SEARCHWEIGHT);
}
public AtlasAttributeDef(String name, String typeName, int searchWeight) {
this(name, typeName, false, Cardinality.SINGLE, searchWeight);
} }
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality) { public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality) {
this(name, typeName, isOptional, cardinality, COUNT_NOT_SET, COUNT_NOT_SET, false, false, false, null); this(name, typeName, isOptional, cardinality, DEFAULT_SEARCHWEIGHT);
} }
private AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, int searchWeight) {
this(name, typeName, isOptional, cardinality, COUNT_NOT_SET, COUNT_NOT_SET, false, false, false, null, searchWeight);
}
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, List<AtlasConstraintDef> constraints) { int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, List<AtlasConstraintDef> constraints) {
this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, null, constraints, null, null); this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, constraints, DEFAULT_SEARCHWEIGHT);
}
private AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, List<AtlasConstraintDef> constraints, int searchWeight) {
this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, null, constraints, null, null, searchWeight);
} }
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, String defaultValue, int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, String defaultValue,
List<AtlasConstraintDef> constraints, Map<String,String> options, String description) { List<AtlasConstraintDef> constraints, Map<String,String> options, String description, int searchWeight) {
setName(name); setName(name);
setTypeName(typeName); setTypeName(typeName);
setIsOptional(isOptional); setIsOptional(isOptional);
...@@ -317,6 +335,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -317,6 +335,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
setConstraints(constraints); setConstraints(constraints);
setOptions(options); setOptions(options);
setDescription(description); setDescription(description);
setSearchWeight(searchWeight);
} }
public AtlasAttributeDef(AtlasAttributeDef other) { public AtlasAttributeDef(AtlasAttributeDef other) {
...@@ -334,9 +353,18 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -334,9 +353,18 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
setConstraints(other.getConstraints()); setConstraints(other.getConstraints());
setOptions(other.getOptions()); setOptions(other.getOptions());
setDescription((other.getDescription())); setDescription((other.getDescription()));
setSearchWeight(other.getSearchWeight());
} }
} }
public int getSearchWeight() {
return searchWeight;
}
public void setSearchWeight(int searchWeight) {
this.searchWeight = searchWeight;
}
public String getName() { public String getName() {
return name; return name;
} }
...@@ -481,6 +509,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -481,6 +509,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
sb.append(", includeInNotification=").append(includeInNotification); sb.append(", includeInNotification=").append(includeInNotification);
sb.append(", defaultValue=").append(defaultValue); sb.append(", defaultValue=").append(defaultValue);
sb.append(", options='").append(options).append('\''); sb.append(", options='").append(options).append('\'');
sb.append(", searchWeight='").append(searchWeight).append('\'');
sb.append(", constraints=["); sb.append(", constraints=[");
if (CollectionUtils.isNotEmpty(constraints)) { if (CollectionUtils.isNotEmpty(constraints)) {
int i = 0; int i = 0;
...@@ -515,12 +544,13 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { ...@@ -515,12 +544,13 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
Objects.equals(defaultValue, that.defaultValue) && Objects.equals(defaultValue, that.defaultValue) &&
Objects.equals(description, that.description) && Objects.equals(description, that.description) &&
Objects.equals(constraints, that.constraints) && Objects.equals(constraints, that.constraints) &&
Objects.equals(options, that.options); Objects.equals(options, that.options) &&
Objects.equals(searchWeight, that.searchWeight);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description); return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description, searchWeight);
} }
@Override @Override
......
intg/src/main/java/org/apache/atlas/v1/model/typedef/AttributeDefinition.java
...@@ -32,6 +32,7 @@ import java.util.Objects; ...@@ -32,6 +32,7 @@ import java.util.Objects;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) @JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE)
...@@ -40,7 +41,6 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ ...@@ -40,7 +41,6 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
@XmlAccessorType(XmlAccessType.PROPERTY) @XmlAccessorType(XmlAccessType.PROPERTY)
public class AttributeDefinition implements Serializable { public class AttributeDefinition implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private String name; private String name;
private String dataTypeName; private String dataTypeName;
private Multiplicity multiplicity; private Multiplicity multiplicity;
...@@ -51,22 +51,34 @@ public class AttributeDefinition implements Serializable { ...@@ -51,22 +51,34 @@ public class AttributeDefinition implements Serializable {
private String defaultValue; private String defaultValue;
private String description; private String description;
private Map<String, String> options; private Map<String, String> options;
private int searchWeight = DEFAULT_SEARCHWEIGHT;
public AttributeDefinition() { public AttributeDefinition() {
} }
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity) { public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity) {
this(name, dataTypeName, multiplicity, false, false, true, null, null); this(name, dataTypeName, multiplicity, false, false, true, null, null, DEFAULT_SEARCHWEIGHT);
} }
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite, public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
String reverseAttributeName) { String reverseAttributeName) {
this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName, null); this(name, dataTypeName, multiplicity, isComposite, reverseAttributeName, DEFAULT_SEARCHWEIGHT);
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
String reverseAttributeName, int searchWeight) {
this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName, null, searchWeight);
} }
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite, public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
boolean isUnique, boolean isIndexable, String reverseAttributeName, boolean isUnique, boolean isIndexable, String reverseAttributeName,
Map<String, String> options) { Map<String, String> options) {
this(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable,reverseAttributeName, options, DEFAULT_SEARCHWEIGHT);
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
boolean isUnique, boolean isIndexable, String reverseAttributeName,
Map<String, String> options, int searchWeight) {
this.name = name; this.name = name;
this.dataTypeName = dataTypeName; this.dataTypeName = dataTypeName;
this.multiplicity = multiplicity; this.multiplicity = multiplicity;
...@@ -75,6 +87,7 @@ public class AttributeDefinition implements Serializable { ...@@ -75,6 +87,7 @@ public class AttributeDefinition implements Serializable {
this.isIndexable = isIndexable; this.isIndexable = isIndexable;
this.reverseAttributeName = reverseAttributeName; this.reverseAttributeName = reverseAttributeName;
this.options = options; this.options = options;
this.searchWeight = searchWeight;
} }
...@@ -183,12 +196,21 @@ public class AttributeDefinition implements Serializable { ...@@ -183,12 +196,21 @@ public class AttributeDefinition implements Serializable {
Objects.equals(defaultValue, that.defaultValue) && Objects.equals(defaultValue, that.defaultValue) &&
Objects.equals(description, that.description) && Objects.equals(description, that.description) &&
Objects.equals(reverseAttributeName, that.reverseAttributeName) && Objects.equals(reverseAttributeName, that.reverseAttributeName) &&
Objects.equals(options, that.options); Objects.equals(options, that.options) &&
Objects.equals(searchWeight, that.searchWeight);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable, return Objects.hash(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable,
reverseAttributeName, defaultValue, description, options); reverseAttributeName, defaultValue, description, options, searchWeight);
} }
public void setSearchWeight(int searchWeight) {
this.searchWeight = searchWeight;
}
public int getSearchWeight() {
return searchWeight;
}
} }
repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
/**
* This class is equivalent to legacy FullTextSearchProcessor--except that it uses a better search techniques using SOLR
* than going through Janus Graph index apis.
*/
public class FreeTextSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(FreeTextSearchProcessor.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("FreeTextSearchProcessor");
public static final String SOLR_QT_PARAMETER = "qt";
public static final String SOLR_REQUEST_HANDLER_NAME = "/freetext";
private static final int MAX_TYPES_STRING_SIZE = 1000;
private final AtlasIndexQuery indexQuery;
public FreeTextSearchProcessor(SearchContext context) {
super(context);
SearchParameters searchParameters = context.getSearchParameters();
StringBuilder queryString = new StringBuilder();
queryString.append(searchParameters.getQuery());
String queryFields = null;
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
if (context.getEntityType() != null) {
String typeString = context.getEntityType().getTypeAndAllSubTypesQryStr();
if (typeString.length() > MAX_TYPES_STRING_SIZE) {
LOG.info("Dropping the use of types string optimization as there are too many types {} for select type {}.", typeString, context.getEntityType().getTypeName());
} else {
LOG.debug("Using the use of types string optimization as there are too many types {} for select type {}.", typeString, context.getEntityType().getTypeName());
final Set<String> types = context.getEntityType().getTypeAndAllSubTypes();
final AtlasGraphManagement managementSystem = context.getGraph().getManagementSystem();
AtlasPropertyKey entityTypeNamePropertyKey = managementSystem.getPropertyKey(AtlasGraphUtilsV2.encodePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY));
String encodedPropertyName = managementSystem.getIndexFieldName(Constants.VERTEX_INDEX, entityTypeNamePropertyKey);
StringBuilder typesStringBuilder = new StringBuilder();
for(String typeName: types) {
typesStringBuilder.append(" ").append(typeName);
}
//append the query with type and substypes listed in it
String typesString = typesStringBuilder.toString();
queryString.append(" AND +").append(encodedPropertyName).append(":[");
queryString.append(typesStringBuilder.toString());
queryString.append("]");
}
}
//just use the query string as is
LOG.debug("Using query string '{}'.", queryString);
indexQuery = context.getGraph().indexQuery(prepareGraphIndexQueryParameters(context, queryString));
}
private GraphIndexQueryParameters prepareGraphIndexQueryParameters(SearchContext context, StringBuilder queryString) {
List<AtlasIndexQueryParameter> parameters = new ArrayList<AtlasIndexQueryParameter>();
parameters.add(context.getGraph().indexQueryParameter(SOLR_QT_PARAMETER, SOLR_REQUEST_HANDLER_NAME));
return new GraphIndexQueryParameters(Constants.VERTEX_INDEX, queryString.toString(), 0, parameters);
}
@Override
public List<AtlasVertex> execute() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> FullTextSearchProcessorUsingFreeText.execute({})", context);
}
List<AtlasVertex> ret = new ArrayList<>();
AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FullTextSearchProcessorUsingFreeText.execute(" + context + ")");
}
try {
final int startIdx = context.getSearchParameters().getOffset();
final int limit = context.getSearchParameters().getLimit();
final boolean activeOnly = context.getSearchParameters().getExcludeDeletedEntities();
// query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
// have been dropped: like vertices of non-entity or non-active-entity
//
// first 'startIdx' number of entries will be ignored
int qryOffset = 0;
int resultIdx = qryOffset;
final List<AtlasVertex> entityVertices = new ArrayList<>();
try {
for (; ret.size() < limit; qryOffset += limit) {
entityVertices.clear();
if (context.terminateSearch()) {
LOG.warn("query terminated: {}", context.getSearchParameters());
break;
}
Iterator<AtlasIndexQuery.Result> idxQueryResult = indexQuery.vertices(qryOffset, limit);
final boolean isLastResultPage;
int resultCount = 0;
while (idxQueryResult.hasNext()) {
AtlasVertex vertex = idxQueryResult.next().getVertex();
resultCount++;
// skip non-entity vertices
if (!AtlasGraphUtilsV2.isEntityVertex(vertex)) {
if (LOG.isDebugEnabled()) {
LOG.debug("FullTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId());
}
continue;
}
if (activeOnly && AtlasGraphUtilsV2.getState(vertex) != AtlasEntity.Status.ACTIVE) {
continue;
}
entityVertices.add(vertex);
}
isLastResultPage = resultCount < limit;
super.filter(entityVertices);
resultIdx = collectResultVertices(ret, startIdx, limit, resultIdx, entityVertices);
if (isLastResultPage) {
break;
}
}
} catch (Throwable t) {
throw t;
}
} finally {
AtlasPerfTracer.log(perf);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== FullTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
}
return ret;
}
}
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
...@@ -18,7 +18,9 @@ ...@@ -18,7 +18,9 @@
package org.apache.atlas.discovery; package org.apache.atlas.discovery;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria; import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
...@@ -39,6 +41,8 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute; ...@@ -39,6 +41,8 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
...@@ -56,6 +60,7 @@ import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIF ...@@ -56,6 +60,7 @@ import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIF
* possible chaining of processor(s) * possible chaining of processor(s)
*/ */
public class SearchContext { public class SearchContext {
private static final Logger LOG = LoggerFactory.getLogger(SearchContext.class);
private final SearchParameters searchParameters; private final SearchParameters searchParameters;
private final AtlasTypeRegistry typeRegistry; private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph; private final AtlasGraph graph;
...@@ -65,11 +70,21 @@ public class SearchContext { ...@@ -65,11 +70,21 @@ public class SearchContext {
private final AtlasClassificationType classificationType; private final AtlasClassificationType classificationType;
private SearchProcessor searchProcessor; private SearchProcessor searchProcessor;
private boolean terminateSearch = false; private boolean terminateSearch = false;
private static boolean isIndexSolrBased = false;
public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS)); public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS)); public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS)); public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
static {
try {
isIndexSolrBased = ApplicationProperties.INDEX_BACKEND_SOLR.equalsIgnoreCase(ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF));
} catch (AtlasException e) {
String msg = String.format("Error encountered in verifying the backend index mode.");
LOG.error(msg, e);
throw new RuntimeException(msg, e);
};
}
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException { public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
String classificationName = searchParameters.getClassification(); String classificationName = searchParameters.getClassification();
...@@ -109,13 +124,20 @@ public class SearchContext { ...@@ -109,13 +124,20 @@ public class SearchContext {
} }
if (needFullTextProcessor()) { if (needFullTextProcessor()) {
addProcessor(new FullTextSearchProcessor(this)); if(!isFreeTextIndexEnabled()) {
LOG.info("Using Full Text index based search.");
addProcessor(new FullTextSearchProcessor(this));
}else {
LOG.info("Using Free Text index based search.");
addProcessor(new FreeTextSearchProcessor(this));
}
} }
if (needClassificationProcessor()) { if (needClassificationProcessor()) {
addProcessor(new ClassificationSearchProcessor(this)); addProcessor(new ClassificationSearchProcessor(this));
} }
if (needEntityProcessor()) { if (needEntityProcessor()) {
addProcessor(new EntitySearchProcessor(this)); addProcessor(new EntitySearchProcessor(this));
} }
...@@ -262,4 +284,18 @@ public class SearchContext { ...@@ -262,4 +284,18 @@ public class SearchContext {
private AtlasEntityType getTermEntityType() { private AtlasEntityType getTermEntityType() {
return typeRegistry.getEntityTypeByName(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ENTITY_TYPE); return typeRegistry.getEntityTypeByName(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ENTITY_TYPE);
} }
public static boolean isFreeTextIndexEnabled() {
try {
return isIndexSolrBased() && ApplicationProperties.get().getBoolean(ApplicationProperties.FREE_TEXT_INDEX_ENABLED, ApplicationProperties.DEFAULT_FREE_TEXT_INDEX_ENABLED);
} catch (AtlasException e) {
String msg = String.format("Error encountered in fetching the configuration %s.", ApplicationProperties.FREE_TEXT_INDEX_ENABLED);
LOG.error(msg, e);
throw new RuntimeException(msg, e);
}
}
public static boolean isIndexSolrBased() {
return isIndexSolrBased;
}
} }
repository/src/main/java/org/apache/atlas/repository/converters/TypeConverterUtil.java
...@@ -307,7 +307,7 @@ public final class TypeConverterUtil { ...@@ -307,7 +307,7 @@ public final class TypeConverterUtil {
} }
public static AtlasAttributeDef toAtlasAttributeDef(final AttributeDefinition attrDefinition) { public static AtlasAttributeDef toAtlasAttributeDef(final AttributeDefinition attrDefinition) {
AtlasAttributeDef ret = new AtlasAttributeDef(attrDefinition.getName(), attrDefinition.getDataTypeName()); AtlasAttributeDef ret = new AtlasAttributeDef(attrDefinition.getName(), attrDefinition.getDataTypeName(), attrDefinition.getSearchWeight());
ret.setIsIndexable(attrDefinition.getIsIndexable()); ret.setIsIndexable(attrDefinition.getIsIndexable());
ret.setIsUnique(attrDefinition.getIsUnique()); ret.setIsUnique(attrDefinition.getIsUnique());
...@@ -362,7 +362,7 @@ public final class TypeConverterUtil { ...@@ -362,7 +362,7 @@ public final class TypeConverterUtil {
AttributeDefinition oldAttrDef = AtlasStructDefStoreV2.toAttributeDefinition(attribute); AttributeDefinition oldAttrDef = AtlasStructDefStoreV2.toAttributeDefinition(attribute);
ret.add(new AttributeDefinition(oldAttrDef.getName(), oldAttrDef.getDataTypeName(), new Multiplicity(oldAttrDef.getMultiplicity()), oldAttrDef.getIsComposite(), oldAttrDef.getIsUnique(), oldAttrDef.getIsIndexable(), oldAttrDef.getReverseAttributeName(), oldAttrDef.getOptions())); ret.add(new AttributeDefinition(oldAttrDef.getName(), oldAttrDef.getDataTypeName(), new Multiplicity(oldAttrDef.getMultiplicity()), oldAttrDef.getIsComposite(), oldAttrDef.getIsUnique(), oldAttrDef.getIsIndexable(), oldAttrDef.getReverseAttributeName(), oldAttrDef.getOptions(), oldAttrDef.getSearchWeight()));
} }
} }
......
repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
...@@ -106,7 +106,7 @@ public class FullTextMapperV2 { ...@@ -106,7 +106,7 @@ public class FullTextMapperV2 {
sb.append(classification.getTypeName()).append(FULL_TEXT_DELIMITER); sb.append(classification.getTypeName()).append(FULL_TEXT_DELIMITER);
mapAttributes(classificationType, classification.getAttributes(), entityWithExtInfo, sb, new HashSet<String>(), excludeAttributes); mapAttributes(classificationType, classification.getAttributes(), entityWithExtInfo, sb, new HashSet<String>(), excludeAttributes, false); //false because of full text context.
} }
} }
...@@ -138,7 +138,7 @@ public class FullTextMapperV2 { ...@@ -138,7 +138,7 @@ public class FullTextMapperV2 {
if (entity != null) { if (entity != null) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
map(entity, entityExtInfo, sb, new HashSet<String>()); map(entity, entityExtInfo, sb, new HashSet<String>(), false);
ret = sb.toString(); ret = sb.toString();
} }
...@@ -150,7 +150,22 @@ public class FullTextMapperV2 { ...@@ -150,7 +150,22 @@ public class FullTextMapperV2 {
return ret; return ret;
} }
private void map(AtlasEntity entity, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException { public String getClassificationTextForEntity(AtlasEntity entity) throws AtlasBaseException {
String ret = null;
if (entity != null) {
StringBuilder sb = new StringBuilder();
map(entity, null, sb, new HashSet<String>(), true);
ret = sb.toString();
}
if (LOG.isDebugEnabled()) {
LOG.info("FullTextMapperV2.getClassificationTextForEntity({}): {}", entity.getGuid(), ret);
}
return ret;
}
private void map(AtlasEntity entity, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids, boolean isClassificationOnly) throws AtlasBaseException {
if (entity == null || processedGuids.contains(entity.getGuid())) { if (entity == null || processedGuids.contains(entity.getGuid())) {
return; return;
} }
...@@ -159,10 +174,11 @@ public class FullTextMapperV2 { ...@@ -159,10 +174,11 @@ public class FullTextMapperV2 {
final Set<String> excludeAttributes = getExcludeAttributesForIndexText(entity.getTypeName()); final Set<String> excludeAttributes = getExcludeAttributesForIndexText(entity.getTypeName());
processedGuids.add(entity.getGuid()); processedGuids.add(entity.getGuid());
if(!isClassificationOnly) {
sb.append(entity.getTypeName()).append(FULL_TEXT_DELIMITER);
sb.append(entity.getTypeName()).append(FULL_TEXT_DELIMITER); mapAttributes(entityType, entity.getAttributes(), entityExtInfo, sb, processedGuids, excludeAttributes, isClassificationOnly);
}
mapAttributes(entityType, entity.getAttributes(), entityExtInfo, sb, processedGuids, excludeAttributes);
final List<AtlasClassification> classifications = entity.getClassifications(); final List<AtlasClassification> classifications = entity.getClassifications();
if (CollectionUtils.isNotEmpty(classifications)) { if (CollectionUtils.isNotEmpty(classifications)) {
...@@ -173,13 +189,13 @@ public class FullTextMapperV2 { ...@@ -173,13 +189,13 @@ public class FullTextMapperV2 {
sb.append(classification.getTypeName()).append(FULL_TEXT_DELIMITER); sb.append(classification.getTypeName()).append(FULL_TEXT_DELIMITER);
mapAttributes(classificationType, classification.getAttributes(), entityExtInfo, sb, processedGuids, excludeClassificationAttributes); mapAttributes(classificationType, classification.getAttributes(), entityExtInfo, sb, processedGuids, excludeClassificationAttributes, isClassificationOnly);
} }
} }
} }
private void mapAttributes(AtlasStructType structType, Map<String, Object> attributes, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, private void mapAttributes(AtlasStructType structType, Map<String, Object> attributes, AtlasEntityExtInfo entityExtInfo, StringBuilder sb,
Set<String> processedGuids, Set<String> excludeAttributes) throws AtlasBaseException { Set<String> processedGuids, Set<String> excludeAttributes, boolean isClassificationOnly) throws AtlasBaseException {
if (MapUtils.isEmpty(attributes)) { if (MapUtils.isEmpty(attributes)) {
return; return;
} }
...@@ -212,32 +228,32 @@ public class FullTextMapperV2 { ...@@ -212,32 +228,32 @@ public class FullTextMapperV2 {
sb.append(attribKey).append(FULL_TEXT_DELIMITER); sb.append(attribKey).append(FULL_TEXT_DELIMITER);
mapAttribute(attrValue, entityExtInfo, sb, processedGuids); mapAttribute(attrValue, entityExtInfo, sb, processedGuids, isClassificationOnly);
} }
} }
private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException { private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids, boolean isClassificationOnly) throws AtlasBaseException {
if (value instanceof AtlasObjectId) { if (value instanceof AtlasObjectId) {
if (followReferences && entityExtInfo != null) { if (followReferences && entityExtInfo != null) {
AtlasObjectId objectId = (AtlasObjectId) value; AtlasObjectId objectId = (AtlasObjectId) value;
AtlasEntity entity = entityExtInfo.getEntity(objectId.getGuid()); AtlasEntity entity = entityExtInfo.getEntity(objectId.getGuid());
if (entity != null) { if (entity != null) {
map(entity, entityExtInfo, sb, processedGuids); map(entity, entityExtInfo, sb, processedGuids, isClassificationOnly);
} }
} }
} else if (value instanceof List) { } else if (value instanceof List) {
List valueList = (List) value; List valueList = (List) value;
for (Object listElement : valueList) { for (Object listElement : valueList) {
mapAttribute(listElement, entityExtInfo, sb, processedGuids); mapAttribute(listElement, entityExtInfo, sb, processedGuids, isClassificationOnly);
} }
} else if (value instanceof Map) { } else if (value instanceof Map) {
Map valueMap = (Map) value; Map valueMap = (Map) value;
for (Object key : valueMap.keySet()) { for (Object key : valueMap.keySet()) {
mapAttribute(key, entityExtInfo, sb, processedGuids); mapAttribute(key, entityExtInfo, sb, processedGuids, isClassificationOnly);
mapAttribute(valueMap.get(key), entityExtInfo, sb, processedGuids); mapAttribute(valueMap.get(key), entityExtInfo, sb, processedGuids, isClassificationOnly);
} }
} else if (value instanceof Enum) { } else if (value instanceof Enum) {
Enum enumValue = (Enum) value; Enum enumValue = (Enum) value;
...@@ -248,7 +264,7 @@ public class FullTextMapperV2 { ...@@ -248,7 +264,7 @@ public class FullTextMapperV2 {
for (Map.Entry<String, Object> entry : atlasStruct.getAttributes().entrySet()) { for (Map.Entry<String, Object> entry : atlasStruct.getAttributes().entrySet()) {
sb.append(entry.getKey()).append(FULL_TEXT_DELIMITER); sb.append(entry.getKey()).append(FULL_TEXT_DELIMITER);
mapAttribute(entry.getValue(), entityExtInfo, sb, processedGuids); mapAttribute(entry.getValue(), entityExtInfo, sb, processedGuids, isClassificationOnly);
} }
} else { } else {
sb.append(String.valueOf(value)).append(FULL_TEXT_DELIMITER); sb.append(String.valueOf(value)).append(FULL_TEXT_DELIMITER);
......
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
...@@ -40,6 +40,8 @@ import org.apache.atlas.model.typedef.AtlasRelationshipEndDef; ...@@ -40,6 +40,8 @@ import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.patches.AtlasPatchRegistry; import org.apache.atlas.repository.patches.AtlasPatchRegistry;
import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.store.AtlasTypeDefStore;
...@@ -66,11 +68,7 @@ import javax.xml.bind.annotation.XmlRootElement; ...@@ -66,11 +68,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.io.File; import java.io.File;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.util.Arrays; import java.util.*;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
...@@ -425,7 +423,8 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler { ...@@ -425,7 +423,8 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
new UpdateAttributePatchHandler(typeDefStore, typeRegistry), new UpdateAttributePatchHandler(typeDefStore, typeRegistry),
new RemoveLegacyRefAttributesPatchHandler(typeDefStore, typeRegistry), new RemoveLegacyRefAttributesPatchHandler(typeDefStore, typeRegistry),
new UpdateTypeDefOptionsPatchHandler(typeDefStore, typeRegistry), new UpdateTypeDefOptionsPatchHandler(typeDefStore, typeRegistry),
new SetServiceTypePatchHandler(typeDefStore, typeRegistry) new SetServiceTypePatchHandler(typeDefStore, typeRegistry),
new UpdateAttributeMetadataHandler(typeDefStore, typeRegistry)
}; };
Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>(); Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
...@@ -506,6 +505,7 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler { ...@@ -506,6 +505,7 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
private List<AtlasAttributeDef> attributeDefs; private List<AtlasAttributeDef> attributeDefs;
private Map<String, String> typeDefOptions; private Map<String, String> typeDefOptions;
private String serviceType; private String serviceType;
private String attributeName;
public String getId() { public String getId() {
return id; return id;
...@@ -586,6 +586,10 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler { ...@@ -586,6 +586,10 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
public void setServiceType(String serviceType) { public void setServiceType(String serviceType) {
this.serviceType = serviceType; this.serviceType = serviceType;
} }
public String getAttributeName() { return attributeName; }
public void setAttributeName(String attributeName) { this.attributeName = attributeName; }
} }
/** /**
...@@ -962,4 +966,92 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler { ...@@ -962,4 +966,92 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
return ret; return ret;
} }
} }
class UpdateAttributeMetadataHandler extends PatchHandler {
public UpdateAttributeMetadataHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry, new String[] { "UPDATE_ATTRIBUTE_METADATA" });
}
@Override
public PatchStatus applyPatch(TypeDefPatch patch) throws AtlasBaseException {
String typeName = patch.getTypeName();
AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName);
PatchStatus ret;
if (typeDef == null) {
throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
}
if (isPatchApplicable(patch, typeDef)) {
String attributeNameFromPatch = patch.getAttributeName();
if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = new AtlasEntityDef((AtlasEntityDef)typeDef);
updateAttributeMetadata(patch, entityDef.getAttributeDefs());
entityDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateEntityDefByName(typeName, entityDef);
ret = APPLIED;
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef)typeDef);
updateAttributeMetadata(patch, updatedDef.getAttributeDefs());
updatedDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateStructDefByName(typeName, updatedDef);
ret = APPLIED;
} else {
throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE, patch.getAction(), typeDef.getClass().getSimpleName());
}
} else {
LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
ret = SKIPPED;
}
return ret;
}
private void updateAttributeMetadata(TypeDefPatch patch, List<AtlasAttributeDef> attributeDefsFromEntity) {
for(AtlasAttributeDef attributeDef: attributeDefsFromEntity) {
if(attributeDef.getName().equalsIgnoreCase(patch.getAttributeName())) {
updateAttribute(attributeDef, patch.getParams());
}
}
}
private void updateAttribute(AtlasAttributeDef atlasAttributeDef, Map<String, Object> params) {
if(!params.isEmpty()) {
for(Map.Entry<String, Object> entry: params.entrySet()) {
try {
if (AtlasAttributeDef.SEARCH_WEIGHT_ATTR_NAME.equalsIgnoreCase(entry.getKey())) {
Number number = (Number) entry.getValue();
int searchWeight = number.intValue();
if(!GraphBackedSearchIndexer.isValidSearchWeight(number.intValue())) {
String msg = String.format("Invalid search weight '%d' was provided for property %s.", searchWeight, atlasAttributeDef.getName());
LOG.error(msg);
throw new RuntimeException(msg);
}
atlasAttributeDef.setSearchWeight(searchWeight);
LOG.info("Updating Model attribute {}'s property{} to {}.", atlasAttributeDef.getName(), entry.getKey(), entry.getValue());
} else {
//sanity exception
//more attributes can be added as needed.
String msg = String.format("Received unknown property{} for attribute {}'s ", entry.getKey(), atlasAttributeDef.getName());
LOG.error(msg);
throw new RuntimeException(msg);
}
} catch (Exception e) {
String msg = String.format("Error encountered in updating Model attribute %s's property '%s' to %s.", atlasAttributeDef.getName(), entry.getKey(), entry.getValue().toString());
LOG.info(msg, e);
throw new RuntimeException(msg, e);
}
}
}
}
}
} }
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
...@@ -539,6 +539,7 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe ...@@ -539,6 +539,7 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
attribInfo.put("reverseAttributeName", attribute.getInverseRefAttributeName()); attribInfo.put("reverseAttributeName", attribute.getInverseRefAttributeName());
attribInfo.put("defaultValue", attributeDef.getDefaultValue()); attribInfo.put("defaultValue", attributeDef.getDefaultValue());
attribInfo.put("description", attributeDef.getDescription()); attribInfo.put("description", attributeDef.getDescription());
attribInfo.put("searchWeight", attributeDef.getSearchWeight());
if(attributeDef.getOptions() != null) { if(attributeDef.getOptions() != null) {
attribInfo.put("options", AtlasType.toJson(attributeDef.getOptions())); attribInfo.put("options", AtlasType.toJson(attributeDef.getOptions()));
...@@ -627,6 +628,12 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe ...@@ -627,6 +628,12 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
ret.setValuesMaxCount(maxCount.intValue()); ret.setValuesMaxCount(maxCount.intValue());
} }
Number searchWeight = (Number) attribInfo.get("searchWeight");
if( searchWeight != null ) {
ret.setSearchWeight(searchWeight.intValue());
} else {
ret.setSearchWeight(-1);
}
return ret; return ret;
} }
...@@ -644,7 +651,9 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe ...@@ -644,7 +651,9 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
ret.setReverseAttributeName(attribute.getInverseRefAttributeName()); ret.setReverseAttributeName(attribute.getInverseRefAttributeName());
ret.setDescription(attrDef.getDescription()); ret.setDescription(attrDef.getDescription());
ret.setDefaultValue(attrDef.getDefaultValue()); ret.setDefaultValue(attrDef.getDefaultValue());
ret.setSearchWeight(attrDef.getSearchWeight());
return ret; return ret;
} }
} }
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
...@@ -38,6 +38,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; ...@@ -38,6 +38,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.converters.AtlasInstanceConverter; import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.graph.FullTextMapperV2;
import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasGraph;
...@@ -79,28 +80,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DE ...@@ -79,28 +80,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DE
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET; import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VALIDITY_PERIODS_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY;
import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.HOME_ID_KEY;
import static org.apache.atlas.repository.Constants.IS_PROXY_KEY;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.SUPER_TYPES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship; import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
...@@ -136,11 +116,12 @@ public class EntityGraphMapper { ...@@ -136,11 +116,12 @@ public class EntityGraphMapper {
private final AtlasEntityChangeNotifier entityChangeNotifier; private final AtlasEntityChangeNotifier entityChangeNotifier;
private final AtlasInstanceConverter instanceConverter; private final AtlasInstanceConverter instanceConverter;
private final EntityGraphRetriever entityRetriever; private final EntityGraphRetriever entityRetriever;
private final FullTextMapperV2 fullTextMapperV2;
@Inject @Inject
public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph, public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph,
AtlasRelationshipStore relationshipStore, AtlasEntityChangeNotifier entityChangeNotifier, AtlasRelationshipStore relationshipStore, AtlasEntityChangeNotifier entityChangeNotifier,
AtlasInstanceConverter instanceConverter) { AtlasInstanceConverter instanceConverter, FullTextMapperV2 fullTextMapperV2) {
this.deleteDelegate = deleteDelegate; this.deleteDelegate = deleteDelegate;
this.typeRegistry = typeRegistry; this.typeRegistry = typeRegistry;
this.graph = atlasGraph; this.graph = atlasGraph;
...@@ -148,6 +129,7 @@ public class EntityGraphMapper { ...@@ -148,6 +129,7 @@ public class EntityGraphMapper {
this.entityChangeNotifier = entityChangeNotifier; this.entityChangeNotifier = entityChangeNotifier;
this.instanceConverter = instanceConverter; this.instanceConverter = instanceConverter;
this.entityRetriever = new EntityGraphRetriever(typeRegistry); this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.fullTextMapperV2 = fullTextMapperV2;
} }
public AtlasVertex createVertex(AtlasEntity entity) { public AtlasVertex createVertex(AtlasEntity entity) {
...@@ -1588,6 +1570,8 @@ public class EntityGraphMapper { ...@@ -1588,6 +1570,8 @@ public class EntityGraphMapper {
AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid); AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
List<AtlasClassification> addedClassifications = StringUtils.equals(entityGuid, guid) ? addClassifications : propagations.get(vertex); List<AtlasClassification> addedClassifications = StringUtils.equals(entityGuid, guid) ? addClassifications : propagations.get(vertex);
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
if (CollectionUtils.isNotEmpty(addedClassifications)) { if (CollectionUtils.isNotEmpty(addedClassifications)) {
entityChangeNotifier.onClassificationAddedToEntity(entity, addedClassifications); entityChangeNotifier.onClassificationAddedToEntity(entity, addedClassifications);
} }
...@@ -1691,10 +1675,12 @@ public class EntityGraphMapper { ...@@ -1691,10 +1675,12 @@ public class EntityGraphMapper {
updateModificationMetadata(entityVertex); updateModificationMetadata(entityVertex);
for (Map.Entry<AtlasVertex, List<AtlasClassification>> entry : removedClassifications.entrySet()) { for (Map.Entry<AtlasVertex, List<AtlasClassification>> entry : removedClassifications.entrySet()) {
String guid = GraphHelper.getGuid(entry.getKey()); AtlasVertex vertex = entry.getKey();
String guid = GraphHelper.getGuid(vertex);
List<AtlasClassification> deletedClassificationNames = entry.getValue(); List<AtlasClassification> deletedClassificationNames = entry.getValue();
AtlasEntity entity = instanceConverter.getAndCacheEntity(guid); AtlasEntity entity = instanceConverter.getAndCacheEntity(guid);
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassificationNames); entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassificationNames);
} }
} }
...@@ -1854,6 +1840,7 @@ public class EntityGraphMapper { ...@@ -1854,6 +1840,7 @@ public class EntityGraphMapper {
AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid); AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
if (isActive(entity)) { if (isActive(entity)) {
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
entityChangeNotifier.onClassificationUpdatedToEntity(entity, updatedClassifications); entityChangeNotifier.onClassificationUpdatedToEntity(entity, updatedClassifications);
} }
} }
...@@ -1866,6 +1853,7 @@ public class EntityGraphMapper { ...@@ -1866,6 +1853,7 @@ public class EntityGraphMapper {
AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid); AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
if (isActive(entity)) { if (isActive(entity)) {
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
entityChangeNotifier.onClassificationDeletedFromEntity(entity, removedClassifications); entityChangeNotifier.onClassificationDeletedFromEntity(entity, removedClassifications);
} }
} }
......
webapp/pom.xml
...@@ -137,6 +137,12 @@ ...@@ -137,6 +137,12 @@
<groupId>org.apache.atlas</groupId> <groupId>org.apache.atlas</groupId>
<artifactId>atlas-janusgraph-hbase2</artifactId> <artifactId>atlas-janusgraph-hbase2</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.noggit</groupId>
<artifactId>noggit</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment