Skip to content

A
atlas
Commit b0063fa5 by Sridhar K Committed by Sarath Subramanian
Options

ATLAS-3100 Enhance Full Text Search backend implementation to use SOLR request handler.

Signed-off-by: 's avatarSarath Subramanian <sarath@apache.org>
parent 8fef5353
Showing with 1647 additions and 131 deletions
addons/models/0000-Area0/patches/003-base_model_add_searchWeight.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_0000_033",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "Referenceable",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "qualifiedName",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_0000_034",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "Asset",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "name",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_0000_035",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "Asset",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "description",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_0000_036",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "Asset",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "owner",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_0000_037",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "AtlasServer",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "name",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_0000_038",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "AtlasServer",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "displayName",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_0000_039",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "AtlasServer",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "fullName",
"params": {
"searchWeight": 10
}
}
]
}
addons/models/1000-Hadoop/1090-impala_model.json
......@@ -49,7 +49,8 @@
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
"isUnique": false,
"searchWeight": 10
},
{
"name": "queryPlan",
......@@ -82,7 +83,8 @@
"isIndexable": false,
"isOptional": true,
"includeInNotification": true,
"isUnique": false
"isUnique": false,
"searchWeight": 10
},
{
"name": "queryGraph",
......@@ -150,7 +152,8 @@
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
"isUnique": false,
"searchWeight": 10
},
{
"name": "queryGraph",
......
addons/models/1000-Hadoop/patches/009-add_searchweight.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_1000_101",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_process",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "userName",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_102",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_process",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "queryText",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_103",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_process",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "clusterName",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_1000_104",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_105",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "viewOriginalText",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_106",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "viewExpandedText",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_107",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_storagedesc",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "location",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_1000_108",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_db",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "location",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_1000_109",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_db",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "clusterName",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_110",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hive_column",
"applyToVersion": "1.4",
"updateToVersion": "1.5",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_111",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "sqoop_process",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "userName",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_112",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "falcon_cluster",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "colo",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_1000_113",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "falcon_feed",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "groups",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_114",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "falcon_process",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "pipelines",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_1000_115",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hbase_namespace",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "clusterName",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_1000_116",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hbase_table",
"applyToVersion": "1.4",
"updateToVersion": "1.5",
"attributeName": "uri",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_1000_117",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "avro_field",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "doc",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_118",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "avro_record",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "doc",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_119",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "avro_record",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "avro_notation",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_1000_120",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "avro_record",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "avroClass",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_1000_121",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "avro_schema",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "namespace",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_1000_122",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "kafka_topic",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "topic",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_1000_123",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "kafka_topic",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "uri",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_1000_124",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "kafka_topic",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "contactInfo",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_1000_125",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "kafka_topic",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "type",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_1000_126",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "kafka_topic",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "keyClassname",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_127",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "jms_topic",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "topic",
"params": {
"searchWeight": 10
}
} , {
"id": "TYPEDEF_PATCH_1000_128",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "jms_topic",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "uri",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_1000_129",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "storm_topology",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "clusterName",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_1000_130",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "storm_node",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "name",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_1000_131",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "storm_node",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "description",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_1000_132",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "storm_node",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "driverClass",
"params": {
"searchWeight": 8
}
} , {
"id": "TYPEDEF_PATCH_1000_133",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "fs_path",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "path",
"params": {
"searchWeight": 5
}
} , {
"id": "TYPEDEF_PATCH_1000_134",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "fs_path",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "group",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_135",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "fs_path",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "posixPermissions",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_1000_136",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hdfs_path",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "clusterName",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_1000_137",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "hdfs_path",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "nameServiceId",
"params": {
"searchWeight": 10
}
}
]
}
addons/models/2000-RDBMS/patches/004-add_searchweight.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_2000_127",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "rdbms_type",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_128",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "platform",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_129",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "contact_info",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_130",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "hostname",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_2000_131",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_instance",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_2000_132",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_db",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "contact_info",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_133",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "name_path",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_2000_134",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_2000_135",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_table",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "contact_info",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_2000_136",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_column",
"applyToVersion": "1.2",
"updateToVersion": "1.3",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}, {
"id": "TYPEDEF_PATCH_2000_137",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "rdbms_index",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "comment",
"params": {
"searchWeight": 9
}
}
]
}
addons/models/3000-Cloud/patches/002-cloud_model_add_searchweight.json 0 → 100644
{
"patches": [
{
"id": "TYPEDEF_PATCH_3000_138",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_pseudo_dir",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "subject",
"params": {
"searchWeight": 10
}
}, {
"id": "TYPEDEF_PATCH_3000_139",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_pseudo_dir",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "objectPrefix",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_3000_140",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_pseudo_dir",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "currentStorageLocation",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_3000_141",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_pseudo_dir",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "subdomain",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_3000_142",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_object",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "partner",
"params": {
"searchWeight": 5
}
}, {
"id": "TYPEDEF_PATCH_3000_143",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_object",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "domain",
"params": {
"searchWeight": 8
}
}, {
"id": "TYPEDEF_PATCH_3000_144",
"description": "Adding search Weights",
"action": "UPDATE_ATTRIBUTE_METADATA",
"typeName": "aws_s3_object",
"applyToVersion": "1.1",
"updateToVersion": "1.2",
"attributeName": "region",
"params": {
"searchWeight": 8
}
}
]
}
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -81,9 +81,11 @@ public final class Constants {
public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
public static final String VERSION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version");
public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
public static final String VERSION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version");
public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
/**
......
distro/src/conf/solr/schema.xml
......@@ -524,11 +524,37 @@
class="solr.UUIDField"
indexed="true" />
<fieldType name="freetext" class="solr.TextField" omitNorms="true"
omitTermFreqAndPositions="true">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.EdgeNGramFilterFactory" minGramSize="2" maxGramSize="16" />
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.KeywordTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
<dynamicField name="*_uuid" type="uuid" indexed="true" stored="true"/>
<!-- TTL -->
<field name="ttl" type="string" indexed="true" stored="true" />
<field name="expire_at" type="date" indexed="true" stored="true" />
<field name="timestamp" type="date" indexed="true" stored="true" />
<field name="allt1s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt2s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt3s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt4s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt5s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt6s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt7s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt8s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt9s" type="freetext" multiValued="true" indexed="true" stored="false"/>
<field name="allt10s" type="freetext" multiValued="true" indexed="true" stored="false"/>
</schema>
distro/src/conf/solr/solrconfig.xml
......@@ -110,9 +110,13 @@
When ManagedIndexSchemaFactory is specified with mutable = true, schema
modification REST API calls will be allowed; otherwise, error responses will be
sent back for these requests.
<schemaFactory class="ClassicIndexSchemaFactory"/>
-->
<schemaFactory class="ClassicIndexSchemaFactory"/>
<schemaFactory class="ManagedIndexSchemaFactory">
<bool name="mutable">true</bool>
<str name="managedSchemaResourceName">managed-schema</str>
</schemaFactory>
<!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Index Config - These settings control low-level behavior of indexing
......@@ -474,6 +478,22 @@
</lst>
</requestHandler>
<requestHandler name="/freetext" class="solr.SearchHandler">
<lst name="defaults">
<str name="defType">edismax</str>
<int name="rows">100</int>
<str name="lowercaseOperators">true</str>
<str name="qf">
allt10s^10 allt9s^9 allt8s^8 allt7s^7 allt6s^6 allt5s^5 allt4s^4 allt3s^3 allt2s^2 allt1s^1
</str>
<str name="hl.fl">*</str>
<str name="hl.requireFieldMatch">true</str>
<str name="lowercaseOperators">true</str>
<str name="facet.limit">5</str>
</lst>
</requestHandler>
<!--
The export request handler is used to export full sorted result sets.
Do not change these defaults.
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
......@@ -17,6 +17,7 @@
*/
package org.apache.atlas.repository.graphdb;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.type.AtlasType;
......@@ -177,6 +178,14 @@ public interface AtlasGraph<V, E> {
AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString, int offset);
/**
* Creates an index query.
*
* @param indexQueryParameters the parameterObject containing the information needed for creating the index.
*
*/
AtlasIndexQuery<V, E> indexQuery(GraphIndexQueryParameters indexQueryParameters);
/**
* Gets the management object associated with this graph and opens a transaction
* for changes that are made.
* @return
......@@ -329,4 +338,26 @@ public interface AtlasGraph<V, E> {
* @return
*/
boolean isMultiProperty(String name);
/**
* return the encoded name used for the attribute identified by property key and index name.
* @param propertyKey the property key of attributed
* @param indexName the name of the index containing the property.
* @return the encoded name of the property.
*/
String getIndexFieldName(AtlasPropertyKey propertyKey, String indexName);
/**
* Create Index query parameter for use with atlas graph.
* @param parameterName the name of the parameter that needs to be passed to index layer.
* @param parameterValue the value of the paratemeter that needs to be passed to the index layer.
* @return
*/
AtlasIndexQueryParameter indexQueryParameter(String parameterName, String parameterValue);
/**
* Implementors should return graph index client.
* @return the graph index client
* @throws AtlasException when error encountered in creating the client.
*/
AtlasGraphIndexClient getGraphIndexClient()throws AtlasException;
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb;
/**
* Represents a graph client work with indices used by Jansgraph.
*/
public interface AtlasGraphIndexClient {
/**
* The implementers should create a mapping from source propertyName to mapping field name.
* @param indexName the name of index that needs to be worked with.
* @param sourcePropertyName the name of the source attribute.
* @param targetPropertyName the name of the target attribute to which the mapping is getting done.
*/
void createCopyField(String indexName, String sourcePropertyName, String targetPropertyName);
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
......@@ -157,4 +157,12 @@ public interface AtlasGraphManagement {
* @param propertyKey
*/
void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
/**
* Gets the index field name for the vertex property.
* @param indexName
* @param propertyKey
* @return the encoded name for the index
*/
String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey);
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb;
import java.util.Iterator;
/**
* Represents an index query parameter for use with AtlasGraph queries.
*/
public interface AtlasIndexQueryParameter {
/**
* @return the name of the parameter for index search.
*/
public String getParameterName();
/**
* @return the value of the parameter for index search.
*/
public String getParameterValue();
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphIndexQueryParameters.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb;
import java.util.List;
/**
* A class wrapping the parameters that need to create a graph index query.
*/
public class GraphIndexQueryParameters {
final String indexName;
final String graphQueryString;
final int offset;
final List<AtlasIndexQueryParameter> indexQueryParameters;
/**
*
* @param indexName the name of the index on which the query is being made
* @param graphQueryString the graph query string.
* @param offset the offset to be used for data fetch.
* @param indexQueryParameters any index system specific parameters for use in query.
*/
public GraphIndexQueryParameters(String indexName, String graphQueryString, int offset, List<AtlasIndexQueryParameter> indexQueryParameters) {
this.indexName = indexName;
this.graphQueryString = graphQueryString;
this.offset = offset;
this.indexQueryParameters = indexQueryParameters;
}
public String getIndexName() {
return indexName;
}
public String getGraphQueryString() {
return graphQueryString;
}
public int getOffset() {
return offset;
}
public List<AtlasIndexQueryParameter> getIndexQueryParameters() {
return indexQueryParameters;
}
}
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
......@@ -25,15 +25,7 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasGraphTraversal;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.graphdb.janus.query.AtlasJanusGraphQuery;
import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
import org.apache.atlas.type.AtlasType;
......@@ -59,20 +51,18 @@ import org.janusgraph.core.PropertyKey;
import org.janusgraph.core.SchemaViolationException;
import org.janusgraph.core.schema.JanusGraphIndex;
import org.janusgraph.core.schema.JanusGraphManagement;
import org.janusgraph.core.schema.Parameter;
import org.janusgraph.diskstorage.BackendException;
import org.janusgraph.graphdb.database.StandardJanusGraph;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
......@@ -84,12 +74,13 @@ import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase.
* Janus implementation of AtlasGraph.
*/
public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraph.class);
private static Configuration APPLICATION_PROPERTIES = null;
private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
private final Set<String> multiProperties = new HashSet<>();
private final StandardJanusGraph janusGraph;
private final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0];
public AtlasJanusGraph() {
this(getGraphInstance());
......@@ -198,6 +189,21 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
@Override
public AtlasIndexQueryParameter indexQueryParameter(String parameterName, String parameterValue) {
return new AtlasJanusIndexQueryParameter(parameterName, parameterValue);
}
@Override
public AtlasGraphIndexClient getGraphIndexClient() throws AtlasException {
try {
return new AtlasJanusGraphSolrIndexClient();
} catch (Exception e) {
LOG.error("Error encountered in creating Graph Index Client.", e);
throw new AtlasException(e);
}
}
@Override
public void commit() {
getGraph().tx().commit();
}
......@@ -208,15 +214,34 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
@Override
public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String fulltextIndex, String graphQuery) {
return indexQuery(fulltextIndex, graphQuery, 0);
public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String indexName, String graphQuery) {
return indexQuery(indexName, graphQuery, 0, null);
}
@Override
public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String fulltextIndex, String graphQuery, int offset) {
public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String indexName, String graphQuery, int offset) {
return indexQuery(indexName, graphQuery, offset, null);
}
/**
* Creates an index query.
*
* @param indexQueryParameters the parameterObject containing the information needed for creating the index.
*
*/
public AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(GraphIndexQueryParameters indexQueryParameters) {
return indexQuery(indexQueryParameters.getIndexName(), indexQueryParameters.getGraphQueryString(), indexQueryParameters.getOffset(), indexQueryParameters.getIndexQueryParameters());
}
private AtlasIndexQuery<AtlasJanusVertex, AtlasJanusEdge> indexQuery(String indexName, String graphQuery, int offset, List<AtlasIndexQueryParameter> indexQueryParameterList) {
String prefix = getIndexQueryPrefix();
JanusGraphIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery).setElementIdentifier(prefix).offset(offset);
JanusGraphIndexQuery query = getGraph().indexQuery(indexName, graphQuery).setElementIdentifier(prefix).offset(offset);
if(indexQueryParameterList != null && indexQueryParameterList.size() > 0) {
for(AtlasIndexQueryParameter indexQueryParameter: indexQueryParameterList) {
query = query.addParameter(new Parameter(indexQueryParameter.getParameterName(), indexQueryParameter.getParameterValue()));
}
}
return new AtlasJanusIndexQuery(this, query);
}
......@@ -497,4 +522,9 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
return convertGremlinValue(input);
}
}
public String getIndexFieldName(AtlasPropertyKey propertyKey, String indexName) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey, EMPTY_PARAMETER_ARRAY, indexName);
}
}
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
......@@ -200,6 +200,14 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
management.addIndexKey(vertexIndex, janusKey);
String encodedName = graph.getIndexFieldName(propertyKey, vertexIndex.getBackingIndex());
LOG.info("property '{}' is encoded to '{}'.", propertyKey.getName(), encodedName);
}
@Override
public String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey) {
JanusGraphIndex index = management.getGraphIndex(indexName);
return graph.getIndexFieldName(propertyKey, index.getBackingIndex());
}
@Override
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.janus;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
public class AtlasJanusGraphSolrIndexClient implements AtlasGraphIndexClient {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphSolrIndexClient.class);
private final SolrClient solrClient;
public AtlasJanusGraphSolrIndexClient() throws Exception {
//well, this is temp hack to get solr client using same settings as that of Janus Graph
solrClient = Solr6Index.getSolrClient();
if(solrClient == null) {
LOG.warn("The indexing system is not solr based. Non SOLR based indexing systems are not supported yet.");
}
}
@Override
public void createCopyField(String collectionName, String srcFieldName, String mappedCopyFieldName) {
if(solrClient == null) {
LOG.error("The indexing system is not solr based. Copy fields can not be created in non SOLR based indexing systems. This request will be treated as no op.");
return;
}
SchemaRequest.AddCopyField addCopyFieldRequest =
new SchemaRequest.AddCopyField(srcFieldName, Arrays.asList(mappedCopyFieldName));
SchemaResponse.UpdateResponse addCopyFieldResponse = null;
try {
addCopyFieldResponse = addCopyFieldRequest.process(solrClient, collectionName);
} catch (SolrServerException | IOException e) {
String msg = String.format("Error encountered in creating the copy field from %s to %s for collection %s.", srcFieldName, mappedCopyFieldName, collectionName);
LOG.error(msg);
throw new RuntimeException(msg, e);
}
}
}
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQueryParameter.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb.janus;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQueryParameter;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.janusgraph.core.JanusGraphIndexQuery;
import org.janusgraph.core.JanusGraphVertex;
import java.util.Iterator;
import java.util.List;
/**
* Janus implementation of AtlasIndexQueryParameter.
*/
public class AtlasJanusIndexQueryParameter implements AtlasIndexQueryParameter {
private final String parameterName;
private final String parameterValue;
public AtlasJanusIndexQueryParameter(String parameterName, String parameterValue) {
this.parameterName = parameterName;
this.parameterValue = parameterValue;
}
@Override
public String getParameterName() {
return parameterName;
}
@Override
public String getParameterValue() {
return parameterValue;
}
}
graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
......@@ -135,9 +135,9 @@ public class Solr6Index implements IndexProvider {
private static final Logger logger = LoggerFactory.getLogger(Solr6Index.class);
private static final String DEFAULT_ID_FIELD = "id";
private static final char CHROOT_START_CHAR = '/';
private static Configuration config;
private enum Mode {
HTTP, CLOUD;
......@@ -181,7 +181,7 @@ public class Solr6Index implements IndexProvider {
public Solr6Index(final Configuration config) throws BackendException {
// Add Kerberos-enabled SolrHttpClientBuilder
HttpClientUtil.setHttpClientBuilder(new Krb5HttpClientBuilder().getBuilder());
initConfiguration(config);
Preconditions.checkArgument(config!=null);
configuration = config;
mode = Mode.parse(config.get(SOLR_MODE));
......@@ -200,23 +200,39 @@ public class Solr6Index implements IndexProvider {
logger.debug("KERBEROS_ENABLED name is " + KERBEROS_ENABLED.getName() + " and it is" + (KERBEROS_ENABLED.isOption() ? " " : " not") + " an option.");
logger.debug("KERBEROS_ENABLED type is " + KERBEROS_ENABLED.getType().name());
}
solrClient = getSolrClient();
}
private static void initConfiguration(Configuration config) {
if(Solr6Index.config == null) {
Solr6Index.config = config;
}
}
public static SolrClient getSolrClient() {
final ModifiableSolrParams clientParams = new ModifiableSolrParams();
SolrClient solrClient = null;
if(Solr6Index.config == null) {
logger.error("The solr client is not being used for the indexing purposes.");
return null;
}
Configuration config = Solr6Index.config;
Mode mode = Mode.parse(config.get(SOLR_MODE));
switch (mode) {
case CLOUD:
/* ATLAS-2920: Update JanusGraph Solr clients to use all zookeeper entries – start */
final List<String> zookeeperUrls = getZookeeperURLs(config);
/* ATLAS-2920: end */
final CloudSolrClient cloudServer = new CloudSolrClient.Builder()
.withLBHttpSolrClientBuilder(
new LBHttpSolrClient.Builder()
.withHttpSolrClientBuilder(new HttpSolrClient.Builder().withInvariantParams(clientParams))
.withBaseSolrUrls(config.get(HTTP_URLS))
)
.withZkHost(zookeeperUrls)
.withZkHost(getZookeeperURLs(config))
.sendUpdatesOnlyToShardLeaders()
.build();
cloudServer.connect();
solrClient = cloudServer;
logger.info("Created solr client using Cloud based configuration.");
break;
case HTTP:
clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION, config.get(HTTP_ALLOW_COMPRESSION).toString());
......@@ -228,12 +244,12 @@ public class Solr6Index implements IndexProvider {
.withHttpClient(client)
.withBaseSolrUrls(config.get(HTTP_URLS))
.build();
logger.info("Created solr client using HTTP based configuration.");
break;
default:
throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
}
return solrClient;
}
private void configureSolrClientsForKerberos() throws PermanentBackendException {
......@@ -1146,7 +1162,7 @@ public class Solr6Index implements IndexProvider {
}
/* ATLAS-2920: Update JanusGraph Solr clients to use all zookeeper entries – start */
private List<String> getZookeeperURLs(Configuration config) {
private static List<String> getZookeeperURLs(Configuration config) {
List<String> ret = null;
String[] zkHosts = config.get(ZOOKEEPER_URLS);
......
intg/src/main/java/org/apache/atlas/ApplicationProperties.java
......@@ -42,20 +42,22 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static final String ATLAS_CONFIGURATION_DIRECTORY_PROPERTY = "atlas.conf";
private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class);
public static final String APPLICATION_PROPERTIES = "atlas-application.properties";
public static final String GRAPHDB_BACKEND_CONF = "atlas.graphdb.backend";
public static final String STORAGE_BACKEND_CONF = "atlas.graph.storage.backend";
public static final String INDEX_BACKEND_CONF = "atlas.graph.index.search.backend";
public static final String INDEX_MAP_NAME_CONF = "atlas.graph.index.search.map-name";
public static final String SOLR_WAIT_SEARCHER_CONF = "atlas.graph.index.search.solr.wait-searcher";
public static final String GRAPHBD_BACKEND_JANUS = "janus";
public static final String STORAGE_BACKEND_HBASE = "hbase";
public static final String STORAGE_BACKEND_HBASE2 = "hbase2";
public static final String INDEX_BACKEND_SOLR = "solr";
public static final String DEFAULT_GRAPHDB_BACKEND = GRAPHBD_BACKEND_JANUS;
public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true;
public static final boolean DEFAULT_INDEX_MAP_NAME = false;
public static final String APPLICATION_PROPERTIES = "atlas-application.properties";
public static final String GRAPHDB_BACKEND_CONF = "atlas.graphdb.backend";
public static final String STORAGE_BACKEND_CONF = "atlas.graph.storage.backend";
public static final String INDEX_BACKEND_CONF = "atlas.graph.index.search.backend";
public static final String INDEX_MAP_NAME_CONF = "atlas.graph.index.search.map-name";
public static final String FREE_TEXT_INDEX_ENABLED = "atlas.freetext.index.enabled";
public static final String SOLR_WAIT_SEARCHER_CONF = "atlas.graph.index.search.solr.wait-searcher";
public static final String GRAPHBD_BACKEND_JANUS = "janus";
public static final String STORAGE_BACKEND_HBASE = "hbase";
public static final String STORAGE_BACKEND_HBASE2 = "hbase2";
public static final String INDEX_BACKEND_SOLR = "solr";
public static final String DEFAULT_GRAPHDB_BACKEND = GRAPHBD_BACKEND_JANUS;
public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true;
public static final boolean DEFAULT_INDEX_MAP_NAME = false;
public static final boolean DEFAULT_FREE_TEXT_INDEX_ENABLED = false;
public static final SimpleEntry<String, String> DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true");
public static final SimpleEntry<String, String> DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20");
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
......@@ -259,10 +259,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
public static class AtlasAttributeDef implements Serializable {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;
public static final int DEFAULT_SEARCHWEIGHT = -1;
public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3;
public static final String SEARCH_WEIGHT_ATTR_NAME = "searchWeight";
public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference";
private final String STRING_TRUE = "true";
private final String STRING_TRUE = "true";
/**
* single-valued attribute or multi-valued attribute.
......@@ -282,6 +284,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
private boolean includeInNotification;
private String defaultValue;
private String description;
private int searchWeight = DEFAULT_SEARCHWEIGHT;
private List<AtlasConstraintDef> constraints;
private Map<String, String> options;
......@@ -289,21 +292,36 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
public AtlasAttributeDef() { this(null, null); }
public AtlasAttributeDef(String name, String typeName) {
this(name, typeName, false, Cardinality.SINGLE);
this(name, typeName, DEFAULT_SEARCHWEIGHT);
}
public AtlasAttributeDef(String name, String typeName, int searchWeight) {
this(name, typeName, false, Cardinality.SINGLE, searchWeight);
}
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality) {
this(name, typeName, isOptional, cardinality, COUNT_NOT_SET, COUNT_NOT_SET, false, false, false, null);
this(name, typeName, isOptional, cardinality, DEFAULT_SEARCHWEIGHT);
}
private AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, int searchWeight) {
this(name, typeName, isOptional, cardinality, COUNT_NOT_SET, COUNT_NOT_SET, false, false, false, null, searchWeight);
}
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, List<AtlasConstraintDef> constraints) {
this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, null, constraints, null, null);
this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, constraints, DEFAULT_SEARCHWEIGHT);
}
private AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, List<AtlasConstraintDef> constraints, int searchWeight) {
this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, null, constraints, null, null, searchWeight);
}
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, boolean includeInNotification, String defaultValue,
List<AtlasConstraintDef> constraints, Map<String,String> options, String description) {
List<AtlasConstraintDef> constraints, Map<String,String> options, String description, int searchWeight) {
setName(name);
setTypeName(typeName);
setIsOptional(isOptional);
......@@ -317,6 +335,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
setConstraints(constraints);
setOptions(options);
setDescription(description);
setSearchWeight(searchWeight);
}
public AtlasAttributeDef(AtlasAttributeDef other) {
......@@ -334,9 +353,18 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
setConstraints(other.getConstraints());
setOptions(other.getOptions());
setDescription((other.getDescription()));
setSearchWeight(other.getSearchWeight());
}
}
public int getSearchWeight() {
return searchWeight;
}
public void setSearchWeight(int searchWeight) {
this.searchWeight = searchWeight;
}
public String getName() {
return name;
}
......@@ -481,6 +509,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
sb.append(", includeInNotification=").append(includeInNotification);
sb.append(", defaultValue=").append(defaultValue);
sb.append(", options='").append(options).append('\'');
sb.append(", searchWeight='").append(searchWeight).append('\'');
sb.append(", constraints=[");
if (CollectionUtils.isNotEmpty(constraints)) {
int i = 0;
......@@ -515,12 +544,13 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
Objects.equals(defaultValue, that.defaultValue) &&
Objects.equals(description, that.description) &&
Objects.equals(constraints, that.constraints) &&
Objects.equals(options, that.options);
Objects.equals(options, that.options) &&
Objects.equals(searchWeight, that.searchWeight);
}
@Override
public int hashCode() {
return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description);
return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, includeInNotification, defaultValue, constraints, options, description, searchWeight);
}
@Override
......
intg/src/main/java/org/apache/atlas/v1/model/typedef/AttributeDefinition.java
......@@ -32,6 +32,7 @@ import java.util.Objects;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE)
......@@ -40,7 +41,6 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
@XmlAccessorType(XmlAccessType.PROPERTY)
public class AttributeDefinition implements Serializable {
private static final long serialVersionUID = 1L;
private String name;
private String dataTypeName;
private Multiplicity multiplicity;
......@@ -51,22 +51,34 @@ public class AttributeDefinition implements Serializable {
private String defaultValue;
private String description;
private Map<String, String> options;
private int searchWeight = DEFAULT_SEARCHWEIGHT;
public AttributeDefinition() {
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity) {
this(name, dataTypeName, multiplicity, false, false, true, null, null);
this(name, dataTypeName, multiplicity, false, false, true, null, null, DEFAULT_SEARCHWEIGHT);
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
String reverseAttributeName) {
this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName, null);
this(name, dataTypeName, multiplicity, isComposite, reverseAttributeName, DEFAULT_SEARCHWEIGHT);
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
String reverseAttributeName, int searchWeight) {
this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName, null, searchWeight);
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
boolean isUnique, boolean isIndexable, String reverseAttributeName,
Map<String, String> options) {
this(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable,reverseAttributeName, options, DEFAULT_SEARCHWEIGHT);
}
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
boolean isUnique, boolean isIndexable, String reverseAttributeName,
Map<String, String> options, int searchWeight) {
this.name = name;
this.dataTypeName = dataTypeName;
this.multiplicity = multiplicity;
......@@ -75,6 +87,7 @@ public class AttributeDefinition implements Serializable {
this.isIndexable = isIndexable;
this.reverseAttributeName = reverseAttributeName;
this.options = options;
this.searchWeight = searchWeight;
}
......@@ -183,12 +196,21 @@ public class AttributeDefinition implements Serializable {
Objects.equals(defaultValue, that.defaultValue) &&
Objects.equals(description, that.description) &&
Objects.equals(reverseAttributeName, that.reverseAttributeName) &&
Objects.equals(options, that.options);
Objects.equals(options, that.options) &&
Objects.equals(searchWeight, that.searchWeight);
}
@Override
public int hashCode() {
return Objects.hash(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable,
reverseAttributeName, defaultValue, description, options);
reverseAttributeName, defaultValue, description, options, searchWeight);
}
public void setSearchWeight(int searchWeight) {
this.searchWeight = searchWeight;
}
public int getSearchWeight() {
return searchWeight;
}
}
repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
/**
* This class is equivalent to legacy FullTextSearchProcessor--except that it uses a better search techniques using SOLR
* than going through Janus Graph index apis.
*/
public class FreeTextSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(FreeTextSearchProcessor.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("FreeTextSearchProcessor");
public static final String SOLR_QT_PARAMETER = "qt";
public static final String SOLR_REQUEST_HANDLER_NAME = "/freetext";
private static final int MAX_TYPES_STRING_SIZE = 1000;
private final AtlasIndexQuery indexQuery;
public FreeTextSearchProcessor(SearchContext context) {
super(context);
SearchParameters searchParameters = context.getSearchParameters();
StringBuilder queryString = new StringBuilder();
queryString.append(searchParameters.getQuery());
String queryFields = null;
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
if (context.getEntityType() != null) {
String typeString = context.getEntityType().getTypeAndAllSubTypesQryStr();
if (typeString.length() > MAX_TYPES_STRING_SIZE) {
LOG.info("Dropping the use of types string optimization as there are too many types {} for select type {}.", typeString, context.getEntityType().getTypeName());
} else {
LOG.debug("Using the use of types string optimization as there are too many types {} for select type {}.", typeString, context.getEntityType().getTypeName());
final Set<String> types = context.getEntityType().getTypeAndAllSubTypes();
final AtlasGraphManagement managementSystem = context.getGraph().getManagementSystem();
AtlasPropertyKey entityTypeNamePropertyKey = managementSystem.getPropertyKey(AtlasGraphUtilsV2.encodePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY));
String encodedPropertyName = managementSystem.getIndexFieldName(Constants.VERTEX_INDEX, entityTypeNamePropertyKey);
StringBuilder typesStringBuilder = new StringBuilder();
for(String typeName: types) {
typesStringBuilder.append(" ").append(typeName);
}
//append the query with type and substypes listed in it
String typesString = typesStringBuilder.toString();
queryString.append(" AND +").append(encodedPropertyName).append(":[");
queryString.append(typesStringBuilder.toString());
queryString.append("]");
}
}
//just use the query string as is
LOG.debug("Using query string '{}'.", queryString);
indexQuery = context.getGraph().indexQuery(prepareGraphIndexQueryParameters(context, queryString));
}
private GraphIndexQueryParameters prepareGraphIndexQueryParameters(SearchContext context, StringBuilder queryString) {
List<AtlasIndexQueryParameter> parameters = new ArrayList<AtlasIndexQueryParameter>();
parameters.add(context.getGraph().indexQueryParameter(SOLR_QT_PARAMETER, SOLR_REQUEST_HANDLER_NAME));
return new GraphIndexQueryParameters(Constants.VERTEX_INDEX, queryString.toString(), 0, parameters);
}
@Override
public List<AtlasVertex> execute() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> FullTextSearchProcessorUsingFreeText.execute({})", context);
}
List<AtlasVertex> ret = new ArrayList<>();
AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FullTextSearchProcessorUsingFreeText.execute(" + context + ")");
}
try {
final int startIdx = context.getSearchParameters().getOffset();
final int limit = context.getSearchParameters().getLimit();
final boolean activeOnly = context.getSearchParameters().getExcludeDeletedEntities();
// query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
// have been dropped: like vertices of non-entity or non-active-entity
//
// first 'startIdx' number of entries will be ignored
int qryOffset = 0;
int resultIdx = qryOffset;
final List<AtlasVertex> entityVertices = new ArrayList<>();
try {
for (; ret.size() < limit; qryOffset += limit) {
entityVertices.clear();
if (context.terminateSearch()) {
LOG.warn("query terminated: {}", context.getSearchParameters());
break;
}
Iterator<AtlasIndexQuery.Result> idxQueryResult = indexQuery.vertices(qryOffset, limit);
final boolean isLastResultPage;
int resultCount = 0;
while (idxQueryResult.hasNext()) {
AtlasVertex vertex = idxQueryResult.next().getVertex();
resultCount++;
// skip non-entity vertices
if (!AtlasGraphUtilsV2.isEntityVertex(vertex)) {
if (LOG.isDebugEnabled()) {
LOG.debug("FullTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId());
}
continue;
}
if (activeOnly && AtlasGraphUtilsV2.getState(vertex) != AtlasEntity.Status.ACTIVE) {
continue;
}
entityVertices.add(vertex);
}
isLastResultPage = resultCount < limit;
super.filter(entityVertices);
resultIdx = collectResultVertices(ret, startIdx, limit, resultIdx, entityVertices);
if (isLastResultPage) {
break;
}
}
} catch (Throwable t) {
throw t;
}
} finally {
AtlasPerfTracer.log(perf);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== FullTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
}
return ret;
}
}
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
......@@ -18,7 +18,9 @@
package org.apache.atlas.discovery;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
......@@ -39,6 +41,8 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashSet;
......@@ -56,6 +60,7 @@ import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIF
* possible chaining of processor(s)
*/
public class SearchContext {
private static final Logger LOG = LoggerFactory.getLogger(SearchContext.class);
private final SearchParameters searchParameters;
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
......@@ -65,11 +70,21 @@ public class SearchContext {
private final AtlasClassificationType classificationType;
private SearchProcessor searchProcessor;
private boolean terminateSearch = false;
private static boolean isIndexSolrBased = false;
public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
static {
try {
isIndexSolrBased = ApplicationProperties.INDEX_BACKEND_SOLR.equalsIgnoreCase(ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF));
} catch (AtlasException e) {
String msg = String.format("Error encountered in verifying the backend index mode.");
LOG.error(msg, e);
throw new RuntimeException(msg, e);
};
}
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
String classificationName = searchParameters.getClassification();
......@@ -109,13 +124,20 @@ public class SearchContext {
}
if (needFullTextProcessor()) {
addProcessor(new FullTextSearchProcessor(this));
if(!isFreeTextIndexEnabled()) {
LOG.info("Using Full Text index based search.");
addProcessor(new FullTextSearchProcessor(this));
}else {
LOG.info("Using Free Text index based search.");
addProcessor(new FreeTextSearchProcessor(this));
}
}
if (needClassificationProcessor()) {
addProcessor(new ClassificationSearchProcessor(this));
}
if (needEntityProcessor()) {
addProcessor(new EntitySearchProcessor(this));
}
......@@ -262,4 +284,18 @@ public class SearchContext {
private AtlasEntityType getTermEntityType() {
return typeRegistry.getEntityTypeByName(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ENTITY_TYPE);
}
public static boolean isFreeTextIndexEnabled() {
try {
return isIndexSolrBased() && ApplicationProperties.get().getBoolean(ApplicationProperties.FREE_TEXT_INDEX_ENABLED, ApplicationProperties.DEFAULT_FREE_TEXT_INDEX_ENABLED);
} catch (AtlasException e) {
String msg = String.format("Error encountered in fetching the configuration %s.", ApplicationProperties.FREE_TEXT_INDEX_ENABLED);
LOG.error(msg, e);
throw new RuntimeException(msg, e);
}
}
public static boolean isIndexSolrBased() {
return isIndexSolrBased;
}
}
repository/src/main/java/org/apache/atlas/repository/converters/TypeConverterUtil.java
......@@ -307,7 +307,7 @@ public final class TypeConverterUtil {
}
public static AtlasAttributeDef toAtlasAttributeDef(final AttributeDefinition attrDefinition) {
AtlasAttributeDef ret = new AtlasAttributeDef(attrDefinition.getName(), attrDefinition.getDataTypeName());
AtlasAttributeDef ret = new AtlasAttributeDef(attrDefinition.getName(), attrDefinition.getDataTypeName(), attrDefinition.getSearchWeight());
ret.setIsIndexable(attrDefinition.getIsIndexable());
ret.setIsUnique(attrDefinition.getIsUnique());
......@@ -362,7 +362,7 @@ public final class TypeConverterUtil {
AttributeDefinition oldAttrDef = AtlasStructDefStoreV2.toAttributeDefinition(attribute);
ret.add(new AttributeDefinition(oldAttrDef.getName(), oldAttrDef.getDataTypeName(), new Multiplicity(oldAttrDef.getMultiplicity()), oldAttrDef.getIsComposite(), oldAttrDef.getIsUnique(), oldAttrDef.getIsIndexable(), oldAttrDef.getReverseAttributeName(), oldAttrDef.getOptions()));
ret.add(new AttributeDefinition(oldAttrDef.getName(), oldAttrDef.getDataTypeName(), new Multiplicity(oldAttrDef.getMultiplicity()), oldAttrDef.getIsComposite(), oldAttrDef.getIsUnique(), oldAttrDef.getIsIndexable(), oldAttrDef.getReverseAttributeName(), oldAttrDef.getOptions(), oldAttrDef.getSearchWeight()));
}
}
......
repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
......@@ -106,7 +106,7 @@ public class FullTextMapperV2 {
sb.append(classification.getTypeName()).append(FULL_TEXT_DELIMITER);
mapAttributes(classificationType, classification.getAttributes(), entityWithExtInfo, sb, new HashSet<String>(), excludeAttributes);
mapAttributes(classificationType, classification.getAttributes(), entityWithExtInfo, sb, new HashSet<String>(), excludeAttributes, false); //false because of full text context.
}
}
......@@ -138,7 +138,7 @@ public class FullTextMapperV2 {
if (entity != null) {
StringBuilder sb = new StringBuilder();
map(entity, entityExtInfo, sb, new HashSet<String>());
map(entity, entityExtInfo, sb, new HashSet<String>(), false);
ret = sb.toString();
}
......@@ -150,7 +150,22 @@ public class FullTextMapperV2 {
return ret;
}
private void map(AtlasEntity entity, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException {
public String getClassificationTextForEntity(AtlasEntity entity) throws AtlasBaseException {
String ret = null;
if (entity != null) {
StringBuilder sb = new StringBuilder();
map(entity, null, sb, new HashSet<String>(), true);
ret = sb.toString();
}
if (LOG.isDebugEnabled()) {
LOG.info("FullTextMapperV2.getClassificationTextForEntity({}): {}", entity.getGuid(), ret);
}
return ret;
}
private void map(AtlasEntity entity, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids, boolean isClassificationOnly) throws AtlasBaseException {
if (entity == null || processedGuids.contains(entity.getGuid())) {
return;
}
......@@ -159,10 +174,11 @@ public class FullTextMapperV2 {
final Set<String> excludeAttributes = getExcludeAttributesForIndexText(entity.getTypeName());
processedGuids.add(entity.getGuid());
if(!isClassificationOnly) {
sb.append(entity.getTypeName()).append(FULL_TEXT_DELIMITER);
sb.append(entity.getTypeName()).append(FULL_TEXT_DELIMITER);
mapAttributes(entityType, entity.getAttributes(), entityExtInfo, sb, processedGuids, excludeAttributes);
mapAttributes(entityType, entity.getAttributes(), entityExtInfo, sb, processedGuids, excludeAttributes, isClassificationOnly);
}
final List<AtlasClassification> classifications = entity.getClassifications();
if (CollectionUtils.isNotEmpty(classifications)) {
......@@ -173,13 +189,13 @@ public class FullTextMapperV2 {
sb.append(classification.getTypeName()).append(FULL_TEXT_DELIMITER);
mapAttributes(classificationType, classification.getAttributes(), entityExtInfo, sb, processedGuids, excludeClassificationAttributes);
mapAttributes(classificationType, classification.getAttributes(), entityExtInfo, sb, processedGuids, excludeClassificationAttributes, isClassificationOnly);
}
}
}
private void mapAttributes(AtlasStructType structType, Map<String, Object> attributes, AtlasEntityExtInfo entityExtInfo, StringBuilder sb,
Set<String> processedGuids, Set<String> excludeAttributes) throws AtlasBaseException {
Set<String> processedGuids, Set<String> excludeAttributes, boolean isClassificationOnly) throws AtlasBaseException {
if (MapUtils.isEmpty(attributes)) {
return;
}
......@@ -212,32 +228,32 @@ public class FullTextMapperV2 {
sb.append(attribKey).append(FULL_TEXT_DELIMITER);
mapAttribute(attrValue, entityExtInfo, sb, processedGuids);
mapAttribute(attrValue, entityExtInfo, sb, processedGuids, isClassificationOnly);
}
}
private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException {
private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids, boolean isClassificationOnly) throws AtlasBaseException {
if (value instanceof AtlasObjectId) {
if (followReferences && entityExtInfo != null) {
AtlasObjectId objectId = (AtlasObjectId) value;
AtlasEntity entity = entityExtInfo.getEntity(objectId.getGuid());
if (entity != null) {
map(entity, entityExtInfo, sb, processedGuids);
map(entity, entityExtInfo, sb, processedGuids, isClassificationOnly);
}
}
} else if (value instanceof List) {
List valueList = (List) value;
for (Object listElement : valueList) {
mapAttribute(listElement, entityExtInfo, sb, processedGuids);
mapAttribute(listElement, entityExtInfo, sb, processedGuids, isClassificationOnly);
}
} else if (value instanceof Map) {
Map valueMap = (Map) value;
for (Object key : valueMap.keySet()) {
mapAttribute(key, entityExtInfo, sb, processedGuids);
mapAttribute(valueMap.get(key), entityExtInfo, sb, processedGuids);
mapAttribute(key, entityExtInfo, sb, processedGuids, isClassificationOnly);
mapAttribute(valueMap.get(key), entityExtInfo, sb, processedGuids, isClassificationOnly);
}
} else if (value instanceof Enum) {
Enum enumValue = (Enum) value;
......@@ -248,7 +264,7 @@ public class FullTextMapperV2 {
for (Map.Entry<String, Object> entry : atlasStruct.getAttributes().entrySet()) {
sb.append(entry.getKey()).append(FULL_TEXT_DELIMITER);
mapAttribute(entry.getValue(), entityExtInfo, sb, processedGuids);
mapAttribute(entry.getValue(), entityExtInfo, sb, processedGuids, isClassificationOnly);
}
} else {
sb.append(String.valueOf(value)).append(FULL_TEXT_DELIMITER);
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.discovery.SearchIndexer;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
......@@ -36,23 +37,9 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasEnumType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.type.*;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
......@@ -72,6 +59,8 @@ import java.util.List;
import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET;
......@@ -104,12 +93,14 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
// Added for type lookup when indexing the new typedefs
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraphIndexClient atlasGraphIndexClient;
//allows injection of a dummy graph for testing
private IAtlasGraphProvider provider;
private boolean recomputeIndexedKeys = true;
private Set<String> vertexIndexKeys = new HashSet<>();
private final String[] mappedCopyFieldNames;
public enum UniqueKind { NONE, GLOBAL_UNIQUE, PER_TYPE_UNIQUE }
@Inject
......@@ -122,6 +113,18 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
throws IndexException, RepositoryException {
this.provider = provider;
this.typeRegistry = typeRegistry;
mappedCopyFieldNames = new String[11];
for(int i=0; i < mappedCopyFieldNames.length; i++) {
mappedCopyFieldNames[i] = String.format("allt%ds", i);
}
try {
atlasGraphIndexClient = provider.get().getGraphIndexClient();
} catch (Exception e) {
LOG.error("Error encountered in creating solr client.", e);
throw new RepositoryException("Error encountered in creating solr client.", e);
}
if (!HAConfiguration.isHAEnabled(configuration)) {
initialize(provider.get());
}
......@@ -280,6 +283,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
createVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false, 10);
createVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
......@@ -331,6 +335,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
private static boolean isStringAttribute(AtlasStructDef.AtlasAttributeDef attributeDef) {
return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeDef.getTypeName());
}
private void createIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) {
final String propertyName = AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName());
AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality());
......@@ -341,7 +349,22 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
boolean isArrayType = isArrayType(attribTypeName);
boolean isMapType = isMapType(attribTypeName);
final String uniqPropName = isUnique ? AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + attributeDef.getName()) : null;
int searchWeight = attributeDef.getSearchWeight();
if(attributeDef.getSearchWeight() == DEFAULT_SEARCHWEIGHT) {
if (isStringAttribute(attributeDef)) {
//We will use default search weight of 3 for string attributes.
//this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher.
LOG.info("Applying default search weight of {} for attribute{}.", DEFAULT_SEARCHWEIGHT_FOR_STRINGS, propertyName);
searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
}
}
if(!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) {
String msg = String.format("Invalid search weight '%d' was provided for property %s.", searchWeight, propertyName);
LOG.error(msg);
throw new RuntimeException(msg);
}
try {
AtlasType atlasType = typeRegistry.getType(typeName);
......@@ -380,7 +403,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (isRelationshipType(atlasType)) {
createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false);
} else {
createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false);
createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false, searchWeight);
if (uniqPropName != null) {
createVertexIndex(management, uniqPropName, UniqueKind.PER_TYPE_UNIQUE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, true);
......@@ -506,6 +529,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
public void createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass,
AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes) {
createVertexIndex(management, propertyName, uniqueKind, propertyClass, cardinality, createCompositeIndex, createCompositeIndexWithTypeAndSuperTypes, -1);
}
private void createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass,
AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes, int searchWeight) {
if (propertyName != null) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
......@@ -518,11 +545,21 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
management.addMixedIndex(VERTEX_INDEX, propertyKey);
LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
}
//for now, we will support free text based enhancements for SOLR backed index systems only.
//WARNING--it is suggested that we don't change the search weight once is assigned with a number in the range 1 to 10.
//otherwise, we might end up mapping the index to multiple copy fields...unwanted to space usage.
if( searchWeight != -1 && SearchContext.isIndexSolrBased()) {
String encodedPropertyName = management.getIndexFieldName(VERTEX_INDEX, propertyKey);
String mappedCopyFieldName = mappedCopyFieldNames[searchWeight];
atlasGraphIndexClient.createCopyField(VERTEX_INDEX, encodedPropertyName, mappedCopyFieldName);
LOG.info("Created copy field from {} to {} for collection {} for property {}.", encodedPropertyName, mappedCopyFieldName, VERTEX_INDEX, propertyName);
}
if (propertyKey != null) {
if (createCompositeIndex || uniqueKind == UniqueKind.GLOBAL_UNIQUE || uniqueKind == UniqueKind.PER_TYPE_UNIQUE) {
createVertexCompositeIndex(management, propertyClass, propertyKey, uniqueKind == UniqueKind.GLOBAL_UNIQUE);
......@@ -806,4 +843,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
addIndexForType(management, typeDef);
LOG.info("Index creation for type {} complete", typeDef.getName());
}
public static boolean isValidSearchWeight(int searchWeight) {
if(searchWeight != -1 ) {
if(searchWeight < 1 || searchWeight > 10) {
return false;
}
}
return true;
}
}
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
......@@ -40,6 +40,8 @@ import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.patches.AtlasPatchRegistry;
import org.apache.atlas.store.AtlasTypeDefStore;
......@@ -66,11 +68,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
......@@ -425,7 +423,8 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
new UpdateAttributePatchHandler(typeDefStore, typeRegistry),
new RemoveLegacyRefAttributesPatchHandler(typeDefStore, typeRegistry),
new UpdateTypeDefOptionsPatchHandler(typeDefStore, typeRegistry),
new SetServiceTypePatchHandler(typeDefStore, typeRegistry)
new SetServiceTypePatchHandler(typeDefStore, typeRegistry),
new UpdateAttributeMetadataHandler(typeDefStore, typeRegistry)
};
Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
......@@ -506,6 +505,7 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
private List<AtlasAttributeDef> attributeDefs;
private Map<String, String> typeDefOptions;
private String serviceType;
private String attributeName;
public String getId() {
return id;
......@@ -586,6 +586,10 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
public void setServiceType(String serviceType) {
this.serviceType = serviceType;
}
public String getAttributeName() { return attributeName; }
public void setAttributeName(String attributeName) { this.attributeName = attributeName; }
}
/**
......@@ -962,4 +966,92 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
return ret;
}
}
class UpdateAttributeMetadataHandler extends PatchHandler {
public UpdateAttributeMetadataHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry, new String[] { "UPDATE_ATTRIBUTE_METADATA" });
}
@Override
public PatchStatus applyPatch(TypeDefPatch patch) throws AtlasBaseException {
String typeName = patch.getTypeName();
AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName);
PatchStatus ret;
if (typeDef == null) {
throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
}
if (isPatchApplicable(patch, typeDef)) {
String attributeNameFromPatch = patch.getAttributeName();
if (typeDef.getClass().equals(AtlasEntityDef.class)) {
AtlasEntityDef entityDef = new AtlasEntityDef((AtlasEntityDef)typeDef);
updateAttributeMetadata(patch, entityDef.getAttributeDefs());
entityDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateEntityDefByName(typeName, entityDef);
ret = APPLIED;
} else if (typeDef.getClass().equals(AtlasStructDef.class)) {
AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef)typeDef);
updateAttributeMetadata(patch, updatedDef.getAttributeDefs());
updatedDef.setTypeVersion(patch.getUpdateToVersion());
typeDefStore.updateStructDefByName(typeName, updatedDef);
ret = APPLIED;
} else {
throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE, patch.getAction(), typeDef.getClass().getSimpleName());
}
} else {
LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
ret = SKIPPED;
}
return ret;
}
private void updateAttributeMetadata(TypeDefPatch patch, List<AtlasAttributeDef> attributeDefsFromEntity) {
for(AtlasAttributeDef attributeDef: attributeDefsFromEntity) {
if(attributeDef.getName().equalsIgnoreCase(patch.getAttributeName())) {
updateAttribute(attributeDef, patch.getParams());
}
}
}
private void updateAttribute(AtlasAttributeDef atlasAttributeDef, Map<String, Object> params) {
if(!params.isEmpty()) {
for(Map.Entry<String, Object> entry: params.entrySet()) {
try {
if (AtlasAttributeDef.SEARCH_WEIGHT_ATTR_NAME.equalsIgnoreCase(entry.getKey())) {
Number number = (Number) entry.getValue();
int searchWeight = number.intValue();
if(!GraphBackedSearchIndexer.isValidSearchWeight(number.intValue())) {
String msg = String.format("Invalid search weight '%d' was provided for property %s.", searchWeight, atlasAttributeDef.getName());
LOG.error(msg);
throw new RuntimeException(msg);
}
atlasAttributeDef.setSearchWeight(searchWeight);
LOG.info("Updating Model attribute {}'s property{} to {}.", atlasAttributeDef.getName(), entry.getKey(), entry.getValue());
} else {
//sanity exception
//more attributes can be added as needed.
String msg = String.format("Received unknown property{} for attribute {}'s ", entry.getKey(), atlasAttributeDef.getName());
LOG.error(msg);
throw new RuntimeException(msg);
}
} catch (Exception e) {
String msg = String.format("Error encountered in updating Model attribute %s's property '%s' to %s.", atlasAttributeDef.getName(), entry.getKey(), entry.getValue().toString());
LOG.info(msg, e);
throw new RuntimeException(msg, e);
}
}
}
}
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
......@@ -539,6 +539,7 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
attribInfo.put("reverseAttributeName", attribute.getInverseRefAttributeName());
attribInfo.put("defaultValue", attributeDef.getDefaultValue());
attribInfo.put("description", attributeDef.getDescription());
attribInfo.put("searchWeight", attributeDef.getSearchWeight());
if(attributeDef.getOptions() != null) {
attribInfo.put("options", AtlasType.toJson(attributeDef.getOptions()));
......@@ -627,6 +628,12 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
ret.setValuesMaxCount(maxCount.intValue());
}
Number searchWeight = (Number) attribInfo.get("searchWeight");
if( searchWeight != null ) {
ret.setSearchWeight(searchWeight.intValue());
} else {
ret.setSearchWeight(-1);
}
return ret;
}
......@@ -644,7 +651,9 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
ret.setReverseAttributeName(attribute.getInverseRefAttributeName());
ret.setDescription(attrDef.getDescription());
ret.setDefaultValue(attrDef.getDefaultValue());
ret.setSearchWeight(attrDef.getSearchWeight());
return ret;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
......@@ -38,6 +38,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.graph.FullTextMapperV2;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
......@@ -79,28 +80,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DE
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VALIDITY_PERIODS_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY;
import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.HOME_ID_KEY;
import static org.apache.atlas.repository.Constants.IS_PROXY_KEY;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.SUPER_TYPES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
......@@ -136,11 +116,12 @@ public class EntityGraphMapper {
private final AtlasEntityChangeNotifier entityChangeNotifier;
private final AtlasInstanceConverter instanceConverter;
private final EntityGraphRetriever entityRetriever;
private final FullTextMapperV2 fullTextMapperV2;
@Inject
public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph,
AtlasRelationshipStore relationshipStore, AtlasEntityChangeNotifier entityChangeNotifier,
AtlasInstanceConverter instanceConverter) {
AtlasInstanceConverter instanceConverter, FullTextMapperV2 fullTextMapperV2) {
this.deleteDelegate = deleteDelegate;
this.typeRegistry = typeRegistry;
this.graph = atlasGraph;
......@@ -148,6 +129,7 @@ public class EntityGraphMapper {
this.entityChangeNotifier = entityChangeNotifier;
this.instanceConverter = instanceConverter;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.fullTextMapperV2 = fullTextMapperV2;
}
public AtlasVertex createVertex(AtlasEntity entity) {
......@@ -1588,6 +1570,8 @@ public class EntityGraphMapper {
AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
List<AtlasClassification> addedClassifications = StringUtils.equals(entityGuid, guid) ? addClassifications : propagations.get(vertex);
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
if (CollectionUtils.isNotEmpty(addedClassifications)) {
entityChangeNotifier.onClassificationAddedToEntity(entity, addedClassifications);
}
......@@ -1691,10 +1675,12 @@ public class EntityGraphMapper {
updateModificationMetadata(entityVertex);
for (Map.Entry<AtlasVertex, List<AtlasClassification>> entry : removedClassifications.entrySet()) {
String guid = GraphHelper.getGuid(entry.getKey());
AtlasVertex vertex = entry.getKey();
String guid = GraphHelper.getGuid(vertex);
List<AtlasClassification> deletedClassificationNames = entry.getValue();
AtlasEntity entity = instanceConverter.getAndCacheEntity(guid);
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassificationNames);
}
}
......@@ -1854,6 +1840,7 @@ public class EntityGraphMapper {
AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
if (isActive(entity)) {
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
entityChangeNotifier.onClassificationUpdatedToEntity(entity, updatedClassifications);
}
}
......@@ -1866,6 +1853,7 @@ public class EntityGraphMapper {
AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
if (isActive(entity)) {
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
entityChangeNotifier.onClassificationDeletedFromEntity(entity, removedClassifications);
}
}
......
webapp/pom.xml
......@@ -137,6 +137,12 @@
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-janusgraph-hbase2</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.noggit</groupId>
<artifactId>noggit</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment