Skip to content

A
atlas
Commit a7d80449 by skoritala Committed by Madhan Neethiraj
Options

ATLAS-3308: enhanced Quicksearch API to support parameters via POST method

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent ce5b6d7f
Showing with 884 additions and 157 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -47,8 +47,8 @@ public final class Constants {
*/
public static final String ENTITY_TYPE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typeName");
public static final String TYPE_NAME_INTERNAL = INTERNAL_PROPERTY_KEY_PREFIX + "internal";
public static final String ASSET_OWNER_PROPERTY_KEY = "Asset.owner";
public static final String ASSET_ENTITY_TYPE = "Asset";
public static final String OWNER_ATTRIBUTE = "owner";
/**
* Entity type's super types property key.
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graphdb;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import java.util.Map;
import java.util.Set;
public class AggregationContext {
private final String queryString;
private final FilterCriteria filterCriteria;
private final AtlasEntityType searchForEntityType;
private final Set<String> aggregationFieldNames;
private final Set<AtlasAttribute> aggregationAttributes;
private final Map<String, String> indexFieldNameCache;
private final boolean excludeDeletedEntities;
private final boolean includeSubTypes;
/**
* @param queryString the query string whose aggregation metrics need to be retrieved.
* @param searchForEntityType
* @param aggregationFieldNames the set of aggregation fields.
* @param indexFieldNameCache
* @param excludeDeletedEntities a boolean flag to indicate if the deleted entities need to be excluded in search
*/
public AggregationContext(String queryString,
FilterCriteria filterCriteria,
AtlasEntityType searchForEntityType,
Set<String> aggregationFieldNames,
Set<AtlasAttribute> aggregationAttributes,
Map<String, String> indexFieldNameCache,
boolean excludeDeletedEntities,
boolean includeSubTypes) {
this.queryString = queryString;
this.filterCriteria = filterCriteria;
this.searchForEntityType = searchForEntityType;
this.aggregationFieldNames = aggregationFieldNames;
this.aggregationAttributes = aggregationAttributes;
this.indexFieldNameCache = indexFieldNameCache;
this.excludeDeletedEntities = excludeDeletedEntities;
this.includeSubTypes = includeSubTypes;
}
public String getQueryString() {
return queryString;
}
public FilterCriteria getFilterCriteria() {
return filterCriteria;
}
public AtlasEntityType getSearchForEntityType() {
return searchForEntityType;
}
public Set<String> getAggregationFieldNames() {
return aggregationFieldNames;
}
public Set<AtlasAttribute> getAggregationAttributes() {
return aggregationAttributes;
}
public Map<String, String> getIndexFieldNameCache() {
return indexFieldNameCache;
}
public boolean isExcludeDeletedEntities() {
return excludeDeletedEntities;
}
public boolean isIncludeSubTypes() {
return includeSubTypes;
}
}
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
......@@ -340,13 +340,6 @@ public interface AtlasGraph<V, E> {
boolean isMultiProperty(String name);
/**
* return the encoded name used for the attribute identified by property key and index name.
* @param propertyKey the property key of attributed
* @param indexName the name of the index containing the property.
* @return the encoded name of the property.
*/
String getIndexFieldName(AtlasPropertyKey propertyKey, String indexName);
/**
* Create Index query parameter for use with atlas graph.
* @param parameterName the name of the parameter that needs to be passed to index layer.
* @param parameterValue the value of the paratemeter that needs to be passed to the index layer.
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
......@@ -18,6 +18,8 @@
package org.apache.atlas.repository.graphdb;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.type.AtlasEntityType;
import java.util.List;
import java.util.Map;
......@@ -30,11 +32,9 @@ public interface AtlasGraphIndexClient {
/**
* Gets aggregated metrics for the given query string and aggregation field names.
* @param queryString the query string whose aggregation metrics need to be retrieved.
* @param propertyKeyNames the set of aggregation fields.
* @return A map of aggregation field to value-count pairs.
*/
Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(String queryString, Set<String> propertyKeyNames);
Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(AggregationContext aggregationContext);
/**
* Returns top 5 suggestions for the given prefix string.
......@@ -46,9 +46,9 @@ public interface AtlasGraphIndexClient {
/**
* The implementers should apply the search weights for the passed in properties.
* @param collectionName the name of the collection for which the search weight needs to be applied
* @param propertyName2SearchWeightMap the map containing search weights from property name to search weights.
* @param indexFieldName2SearchWeightMap the map containing search weights from index field name to search weights.
*/
void applySearchWeight(String collectionName, Map<String, Integer> propertyName2SearchWeightMap);
void applySearchWeight(String collectionName, Map<String, Integer> indexFieldName2SearchWeightMap);
/**
* The implementors should take the passed in list of suggestion properties for suggestions functionality.
......
graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
......@@ -155,8 +155,9 @@ public interface AtlasGraphManagement {
*
* @param vertexIndex
* @param propertyKey
* @return the index field name used for the given property
*/
void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
String addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
/**
* Gets the index field name for the vertex property.
......
graphdb/janus/pom.xml
......@@ -236,6 +236,12 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
......@@ -75,12 +75,13 @@ import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase.
*/
public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraph.class);
private static Configuration APPLICATION_PROPERTIES = null;
private static final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0];
private static Configuration APPLICATION_PROPERTIES = null;
private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
private final Set<String> multiProperties = new HashSet<>();
private final StandardJanusGraph janusGraph;
private final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0];
public AtlasJanusGraph() {
this(getGraphInstance());
......@@ -198,7 +199,7 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
try {
initApplicationProperties();
return new AtlasJanusGraphIndexClient(this, APPLICATION_PROPERTIES);
return new AtlasJanusGraphIndexClient(APPLICATION_PROPERTIES);
} catch (Exception e) {
LOG.error("Error encountered in creating Graph Index Client.", e);
throw new AtlasException(e);
......@@ -417,6 +418,13 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
String getIndexFieldName(AtlasPropertyKey propertyKey, JanusGraphIndex graphIndex) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey, EMPTY_PARAMETER_ARRAY, graphIndex.getBackingIndex());
}
private String getIndexQueryPrefix() {
final String ret;
......@@ -524,9 +532,4 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
return convertGremlinValue(input);
}
}
public String getIndexFieldName(AtlasPropertyKey propertyKey, String indexName) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey, EMPTY_PARAMETER_ARRAY, indexName);
}
}
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
......@@ -23,10 +23,7 @@ import org.apache.tinkerpop.gremlin.structure.Direction;
import org.janusgraph.core.Cardinality;
import org.janusgraph.core.EdgeLabel;
import org.janusgraph.core.PropertyKey;
import org.janusgraph.core.schema.Mapping;
import org.janusgraph.core.schema.PropertyKeyMaker;
import org.janusgraph.core.schema.JanusGraphIndex;
import org.janusgraph.core.schema.JanusGraphManagement;
import org.janusgraph.core.schema.*;
import org.janusgraph.core.schema.JanusGraphManagement.IndexBuilder;
import org.janusgraph.graphdb.internal.Token;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
......@@ -195,19 +192,24 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
}
@Override
public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
public String addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName);
management.addIndexKey(janusGraphIndex, janusKey);
String encodedName = graph.getIndexFieldName(propertyKey, janusGraphIndex);
management.addIndexKey(vertexIndex, janusKey);
String encodedName = graph.getIndexFieldName(propertyKey, vertexIndex.getBackingIndex());
LOG.info("property '{}' is encoded to '{}'.", propertyKey.getName(), encodedName);
return encodedName;
}
@Override
public String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey) {
JanusGraphIndex index = management.getGraphIndex(indexName);
return graph.getIndexFieldName(propertyKey, index.getBackingIndex());
JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName);
return graph.getIndexFieldName(propertyKey, janusGraphIndex);
}
@Override
......
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClientTest.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.repository.graphdb.janus;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -27,81 +28,99 @@ import java.util.Map;
public class AtlasJanusGraphIndexClientTest {
@Test
public void testGetTop5TermsAsendingInput() {
public void testGetTopTermsAsendingInput() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 12, 15);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
validateOrder(top5Terms, 2,1,0);
assertOrder(top5Terms, 2,1,0);
}
@Test
public void testGetTop5TermsAsendingInput2() {
public void testGetTopTermsAsendingInput2() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 12, 15, 20, 25, 26, 30, 40);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
validateOrder(top5Terms, 7, 6, 5, 4, 3);
assertOrder(top5Terms, 7, 6, 5, 4, 3);
}
@Test
public void testGetTop5TermsDescendingInput() {
public void testGetTopTermsDescendingInput() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 9, 8);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
validateOrder(top5Terms, 0, 1, 2);
assertOrder(top5Terms, 0, 1, 2);
}
@Test
public void testGetTop5TermsDescendingInput2() {
public void testGetTopTermsDescendingInput2() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 9, 8, 7, 6, 5, 4, 3, 2);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
validateOrder(top5Terms, 0, 1, 2, 3, 4);
assertOrder(top5Terms, 0, 1, 2, 3, 4);
}
@Test
public void testGetTop5TermsRandom() {
public void testGetTopTermsRandom() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 19, 28, 27, 16, 1, 30, 3, 36);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//10, 19, 28, 27, 16, 1, 30, 3, 36
//0, 1, 2, 3, 4, 5, 6, 7, 8
validateOrder(top5Terms, 8, 6, 2, 3, 1);
assertOrder(top5Terms, 8, 6, 2, 3, 1);
}
@Test
public void testGetTop5TermsRandom2() {
public void testGetTopTermsRandom2() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 36, 19, 28, 27, 16, 1, 30, 3, 10);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//36, 19, 28, 27, 16, 1, 30, 3, 10
//0, 1, 2, 3, 4, 5, 6, 7, 8
validateOrder(top5Terms, 0, 6, 2, 3, 1);
assertOrder(top5Terms, 0, 6, 2, 3, 1);
}
@Test
public void testGetTop5TermsRandom3() {
public void testGetTopTermsRandom3() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 36, 36, 28, 27, 16, 1, 30, 3, 10);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//36, 36, 28, 27, 16, 1, 30, 3, 10
//0, 1, 2, 3, 4, 5, 6, 7, 8
validateOrder(top5Terms, 0, 1, 6, 2, 3);
assertOrder(top5Terms, 0, 1, 6, 2, 3);
}
@Test
public void testGetTop5TermsRandom4() {
public void testGetTopTermsRandom4() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 10, 28, 27, 16, 1, 30, 36, 36);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//10, 10, 28, 27, 16, 1, 30, 36, 36
//0, 1, 2, 3, 4, 5, 6, 7, 8
validateOrder(top5Terms, 7, 8, 6, 2, 3);
assertOrder(top5Terms, 7, 8, 6, 2, 3);
}
@Test
public void testGetTop5TermsRandom5() {
public void testGetTopTermsRandom5() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 36, 10, 28, 27, 16, 1, 30, 36, 36);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//36, 10, 28, 27, 16, 1, 30, 36, 36
//0, 1, 2, 3, 4, 5, 6, 7, 8
validateOrder(top5Terms, 0, 7, 8, 6, 2);
assertOrder(top5Terms, 0, 7, 8, 6, 2);
}
@Test
public void testGenerateSuggestionString() {
List<String> fields = new ArrayList<>();
fields.add("one");
fields.add("two");
fields.add("three");
String generatedString = AtlasJanusGraphIndexClient.generateSuggestionsString(fields);
Assert.assertEquals(generatedString, "'one', 'two', 'three'");
}
@Test
public void testGenerateSearchWeightString() {
Map<String, Integer> fields = new HashMap<>();
fields.put("one", 10);
fields.put("two", 1);
fields.put("three", 15);
String generatedString = AtlasJanusGraphIndexClient.generateSearchWeightString(fields);
Assert.assertEquals(generatedString, " one^10 two^1 three^15");
}
private void validateOrder(List<String> topTerms, int ... indices) {
private void assertOrder(List<String> topTerms, int ... indices) {
Assert.assertEquals(topTerms.size(), indices.length);
int i = 0;
for(String term: topTerms) {
......
graphdb/janus/src/test/resources/searchParameters3.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"condition":"AND",
"criterion":[
{
"attributeName":"comment",
"operator":"contains",
"attributeValue":"United States"
},
{
"attributeName":"description",
"operator":"contains",
"attributeValue":"nothing"
},
{
"attributeName":"name",
"operator":"contains",
"attributeValue":"t100"
}
]
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t10",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
\ No newline at end of file
graphdb/janus/src/test/resources/searchParametersGT.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"condition":"AND",
"criterion":[
{
"attributeName":"created",
"operator":"gt",
"attributeValue":"100"
}
]
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t10",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
\ No newline at end of file
graphdb/janus/src/test/resources/searchParametersGTE.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"condition":"AND",
"criterion":[
{
"attributeName":"created",
"operator":"gte",
"attributeValue":100
},
{
"attributeName":"started",
"operator":"gte",
"attributeValue": 100
}
]
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t10",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
\ No newline at end of file
graphdb/janus/src/test/resources/searchParametersLT.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"condition":"AND",
"criterion":[
{
"attributeName":"created",
"operator":"lt",
"attributeValue":"100"
}
]
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t10",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
\ No newline at end of file
graphdb/janus/src/test/resources/searchParametersLTE.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"condition":"AND",
"criterion":[
{
"attributeName":"created",
"operator":"lte",
"attributeValue":"100"
},
{
"attributeName":"started",
"operator":"lte",
"attributeValue": 100
}
]
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t10",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
\ No newline at end of file
graphdb/janus/src/test/resources/searchParametersStartsWith.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":
{
"condition":"AND",
"criterion":
[
{
"attributeName":"qualifiedName",
"operator":"startsWith",
"attributeValue":"testdb.t1"
}
]
},
"tagFilters":null,
"limit":25,
"offset":0,
"typeName":"hive_table"
}
\ No newline at end of file
graphdb/janus/src/test/resources/searchparameters0.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"attributeName":"name",
"operator":"eq",
"attributeValue":"t10"
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
\ No newline at end of file
graphdb/janus/src/test/resources/searchparameters1AND.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"condition":"AND",
"criterion":[
{
"attributeName":"name",
"operator":"eq",
"attributeValue":"t10"
}
]
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
\ No newline at end of file
graphdb/janus/src/test/resources/searchparameters1OR.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"condition":"OR",
"criterion":[
{
"attributeName":"name",
"operator":"eq",
"attributeValue":"t10"
}
]
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
\ No newline at end of file
graphdb/janus/src/test/resources/searchparameters2AND.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"condition":"AND",
"criterion":[
{
"attributeName":"name",
"operator":"eq",
"attributeValue":"t10"
},
{
"attributeName":"comment",
"operator":"contains",
"attributeValue":"t10"
}
]
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
graphdb/janus/src/test/resources/searchparameters2OR.json 0 → 100644
{
"excludeDeletedEntities":true,
"includeSubClassifications":true,
"includeSubTypes":true,
"includeClassificationAttributes":true,
"entityFilters":{
"condition":"OR",
"criterion":[
{
"attributeName":"name",
"operator":"eq",
"attributeValue":"t10"
},
{
"attributeName":"comment",
"operator":"contains",
"attributeValue":"t10"
}
]
},
"tagFilters":null,
"attributes":[
"comment"
],
"query":"t",
"limit":25,
"offset":0,
"typeName":"hive_table",
"classification":null,
"termName":null
}
intg/src/main/java/org/apache/atlas/listener/TypeDefChangeListener.java
......@@ -21,4 +21,5 @@ import org.apache.atlas.exception.AtlasBaseException;
public interface TypeDefChangeListener {
void onChange(ChangedTypeDefs changedTypeDefs) throws AtlasBaseException;
void onLoadCompletion() throws AtlasBaseException;
}
intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.model.discovery;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import java.io.Serializable;
import java.util.Set;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
/**
* This is the root class representing the input for quick search puroposes.
*/
public class QuickSearchParameters implements Serializable {
private static final long serialVersionUID = 1L;
private String query;
private String typeName;
private FilterCriteria entityFilters;
private boolean includeSubTypes;
private boolean excludeDeletedEntities;
private int offset;
private int limit;
private Set<String> attributes;
/**
* for framework use.
*/
public QuickSearchParameters() {
}
public QuickSearchParameters(String query,
String typeName,
FilterCriteria entityFilters,
boolean includeSubTypes,
boolean excludeDeletedEntities,
int offset,
int limit,
Set<String> attributes) {
this.query = query;
this.typeName = typeName;
this.entityFilters = entityFilters;
this.includeSubTypes = includeSubTypes;
this.excludeDeletedEntities = excludeDeletedEntities;
this.offset = offset;
this.limit = limit;
this.attributes = attributes;
}
public String getQuery() {
return query;
}
public void setQuery(String query) {
this.query = query;
}
public String getTypeName() {
return typeName;
}
public void setTypeName(String typeName) {
this.typeName = typeName;
}
public FilterCriteria getEntityFilters() {
return entityFilters;
}
public void setEntityFilters(FilterCriteria entityFilters) {
this.entityFilters = entityFilters;
}
public boolean getIncludeSubTypes() {
return includeSubTypes;
}
public void setIncludeSubTypes(boolean includeSubTypes) {
this.includeSubTypes = includeSubTypes;
}
public boolean getExcludeDeletedEntities() {
return excludeDeletedEntities;
}
public void setExcludeDeletedEntities(boolean excludeDeletedEntities) {
this.excludeDeletedEntities = excludeDeletedEntities;
}
public int getOffset() {
return offset;
}
public void setOffset(int offset) {
this.offset = offset;
}
public int getLimit() {
return limit;
}
public void setLimit(int limit) {
this.limit = limit;
}
public Set<String> getAttributes() {
return attributes;
}
public void setAttributes(Set<String> attributes) {
this.attributes = attributes;
}
}
intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
......@@ -106,4 +106,6 @@ public interface AtlasTypeDefStore {
AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException;
void deleteTypeByName(String typeName) throws AtlasBaseException;
void notifyLoadCompletion();
}
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -33,10 +33,8 @@ import org.slf4j.LoggerFactory;
import java.util.*;
import static org.apache.atlas.model.TypeCategory.*;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF;
import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.*;
/**
* class that implements behaviour of a struct-type.
......@@ -710,6 +708,7 @@ public class AtlasStructType extends AtlasType {
private String relationshipEdgeLabel;
private AtlasRelationshipEdgeDirection relationshipEdgeDirection;
private boolean isLegacyAttribute;
private String indexFieldName;
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType, String relationshipName, String relationshipLabel) {
this.definedInType = definedInType;
......@@ -821,6 +820,13 @@ public class AtlasStructType extends AtlasType {
public void setLegacyAttribute(boolean legacyAttribute) { isLegacyAttribute = legacyAttribute; }
public String getIndexFieldName() { return indexFieldName; }
public void setIndexFieldName(String indexFieldName) { this.indexFieldName = indexFieldName; }
public int getSearchWeight() { return attributeDef.getSearchWeight(); }
public static String getEdgeLabel(String property) {
return "__" + property;
}
......
intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
......@@ -33,13 +33,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import javax.inject.Singleton;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
......@@ -57,19 +51,22 @@ public class AtlasTypeRegistry {
protected RegistryData registryData;
private final TypeRegistryUpdateSynchronizer updateSynchronizer;
private final Set<String> missingRelationshipDefs;
private final Map<String, String> commonIndexFieldNameCache;
public AtlasTypeRegistry() {
registryData = new RegistryData();
updateSynchronizer = new TypeRegistryUpdateSynchronizer(this);
missingRelationshipDefs = new HashSet<>();
registryData = new RegistryData();
updateSynchronizer = new TypeRegistryUpdateSynchronizer(this);
missingRelationshipDefs = new HashSet<>();
commonIndexFieldNameCache = new HashMap<>();
}
// used only by AtlasTransientTypeRegistry
protected AtlasTypeRegistry(AtlasTypeRegistry other) {
registryData = new RegistryData();
updateSynchronizer = other.updateSynchronizer;
missingRelationshipDefs = other.missingRelationshipDefs;
registryData = new RegistryData();
updateSynchronizer = other.updateSynchronizer;
missingRelationshipDefs = other.missingRelationshipDefs;
commonIndexFieldNameCache = other.commonIndexFieldNameCache;
}
public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); }
......@@ -240,6 +237,19 @@ public class AtlasTypeRegistry {
}
}
public void addIndexFieldName(String propertyName, String indexFieldName) {
commonIndexFieldNameCache.put(propertyName, indexFieldName);
}
/**
* retrieves the index field name for the common field passed in.
* @param propertyName the name of the common field.
* @return the index name for the common field passed in.
*/
public String getIndexFieldName(String propertyName) {
return commonIndexFieldNameCache.get(propertyName);
}
static class RegistryData {
final TypeCache allTypes;
final TypeDefCache<AtlasEnumDef, AtlasEnumType> enumDefs;
......@@ -1163,4 +1173,5 @@ class TypeDefCache<T1 extends AtlasBaseTypeDef, T2 extends AtlasType> {
typeDefNameMap.clear();
typeNameMap.clear();
}
}
repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
......@@ -19,12 +19,10 @@
package org.apache.atlas.discovery;
import com.sun.xml.bind.v2.model.annotation.Quick;
import org.apache.atlas.SortOrder;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasQuickSearchResult;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.AtlasSuggestionsResult;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.*;
import org.apache.atlas.model.profile.AtlasUserSavedSearch;
import java.util.List;
......@@ -142,7 +140,13 @@ public interface AtlasDiscoveryService {
*/
void deleteSavedSearch(String currentUser, String guid) throws AtlasBaseException;
AtlasQuickSearchResult quickSearchWithParameters(SearchParameters searchParameters) throws AtlasBaseException;
/**
* Search for entities matching the search criteria
* @param searchParameters Search criteria
* @return Matching entities
* @throws AtlasBaseException
*/
AtlasQuickSearchResult quickSearch(QuickSearchParameters searchParameters) throws AtlasBaseException;
/**
* Should return top 5 suggestion strings for the given prefix.
......
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
......@@ -26,14 +26,10 @@ import org.apache.atlas.annotation.GraphTransaction;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.authorize.AtlasSearchResultScrubRequest;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasQuickSearchResult;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.*;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult;
import org.apache.atlas.model.discovery.AtlasSuggestionsResult;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.profile.AtlasUserSavedSearch;
......@@ -43,20 +39,14 @@ import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.repository.userprofile.UserProfileService;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
import org.apache.atlas.util.SearchTracker;
......@@ -80,6 +70,7 @@ import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.SortOrder.DESCENDING;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.*;
@Component
......@@ -102,7 +93,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Inject
EntityDiscoveryService(AtlasTypeRegistry typeRegistry,
AtlasGraph graph, GraphBackedSearchIndexer indexer, SearchTracker searchTracker,
AtlasGraph graph,
GraphBackedSearchIndexer indexer,
SearchTracker searchTracker,
UserProfileService userProfileService) throws AtlasException {
this.graph = graph;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
......@@ -421,8 +414,11 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override
@GraphTransaction
public AtlasQuickSearchResult quickSearchWithParameters(SearchParameters searchParameters) throws AtlasBaseException {
SearchContext searchContext = new SearchContext(searchParameters, typeRegistry, graph, indexer.getVertexIndexKeys());
public AtlasQuickSearchResult quickSearch(QuickSearchParameters quickSearchParameters) throws AtlasBaseException {
SearchContext searchContext = new SearchContext(createSearchParameters(quickSearchParameters),
typeRegistry,
graph,
indexer.getVertexIndexKeys());
if(LOG.isDebugEnabled()) {
LOG.debug("Generating the search results for the query {} .", searchContext.getSearchParameters().getQuery());
......@@ -434,9 +430,12 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
LOG.debug("Generating the aggregated metrics for the query {} .", searchContext.getSearchParameters().getQuery());
}
SearchAggregator searchAggregator = new SearchAggregatorImpl(searchContext);
Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = searchAggregator.getAggregatedMetrics();
AtlasQuickSearchResult ret = new AtlasQuickSearchResult(searchResult, aggregatedMetrics);
// load the facet fields and attributes.
Set<String> aggregationFields = getAggregationFields();
Set<AtlasAttribute> aggregationAttributes = getAggregationAtlasAttributes();
SearchAggregator searchAggregator = new SearchAggregatorImpl(searchContext);
Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = searchAggregator.getAggregatedMetrics(aggregationFields, aggregationAttributes);
AtlasQuickSearchResult ret = new AtlasQuickSearchResult(searchResult, aggregatedMetrics);
return ret;
}
......@@ -946,6 +945,21 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return queryStr;
}
public static SearchParameters createSearchParameters(QuickSearchParameters quickSearchParameters) {
SearchParameters searchParameters = new SearchParameters();
searchParameters.setQuery(quickSearchParameters.getQuery());
searchParameters.setTypeName(quickSearchParameters.getTypeName());
searchParameters.setExcludeDeletedEntities(quickSearchParameters.getExcludeDeletedEntities());
searchParameters.setIncludeSubTypes(quickSearchParameters.getIncludeSubTypes());
searchParameters.setLimit(quickSearchParameters.getLimit());
searchParameters.setOffset(quickSearchParameters.getOffset());
searchParameters.setEntityFilters(quickSearchParameters.getEntityFilters());
searchParameters.setAttributes(quickSearchParameters.getAttributes());
return searchParameters;
}
private String escapeTypeName(String typeName) {
String ret;
......@@ -968,4 +982,36 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private void scrubSearchResults(AtlasSearchResult result) throws AtlasBaseException {
AtlasAuthorizationUtils.scrubSearchResults(new AtlasSearchResultScrubRequest(typeRegistry, result));
}
private Set<String> getAggregationFields() {
Set<String> ret = new HashSet<>(); // for non-modeled attributes.
ret.add(Constants.ENTITY_TYPE_PROPERTY_KEY);
ret.add(Constants.STATE_PROPERTY_KEY);
return ret;
}
private Set<AtlasAttribute> getAggregationAtlasAttributes() {
Set<AtlasAttribute> ret = new HashSet<>(); // for modeled attributes, like Asset.owner
ret.add(getAtlasAttributeForAssetOwner());
return ret;
}
private AtlasAttribute getAtlasAttributeForAssetOwner() {
AtlasEntityType typeAsset = typeRegistry.getEntityTypeByName(ASSET_ENTITY_TYPE);
AtlasAttribute atttOwner = typeAsset != null ? typeAsset.getAttribute(OWNER_ATTRIBUTE) : null;
if(atttOwner == null) {
String msg = String.format("Unable to resolve the attribute %s.%s", ASSET_ENTITY_TYPE, OWNER_ATTRIBUTE);
LOG.error(msg);
throw new RuntimeException(msg);
}
return atttOwner;
}
}
repository/src/main/java/org/apache/atlas/discovery/SearchAggregator.java
......@@ -18,10 +18,22 @@
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.type.AtlasStructType;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* This is an interface to search aggregation mwntrics providers.
*/
public interface SearchAggregator {
Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics();
/**
* returns aggregation metrics for passed in aggregation fields.
* @param aggregationFields the set of aggregation attribute names.
* @param aggregationAttrbutes the set of aggregationAttributes
* @return the result of aggreggations by aggregation fields.
*/
Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(Set<String> aggregationFields,
Set<AtlasStructType.AtlasAttribute> aggregationAttrbutes);
}
repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
......@@ -19,10 +19,15 @@ package org.apache.atlas.discovery;
import org.apache.atlas.AtlasException;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AggregationContext;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -34,31 +39,66 @@ public class SearchAggregatorImpl implements SearchAggregator {
private final SearchContext searchContext;
public SearchAggregatorImpl(SearchContext searchContext) {
this.searchContext = searchContext;
}
public Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics() {
String queryString = searchContext.getSearchParameters().getQuery();
AtlasGraph atlasGraph = searchContext.getGraph();
Set<String> aggregationFields = new HashSet<>();
List<PostProcessor> postProcessors = new ArrayList<>();
aggregationFields.add(Constants.ENTITY_TYPE_PROPERTY_KEY);
aggregationFields.add(Constants.ASSET_OWNER_PROPERTY_KEY);
public Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(Set<String> aggregationFields,
Set<AtlasAttribute> aggregationAttributes) {
SearchParameters searchParameters = searchContext.getSearchParameters();
AtlasGraph graph = searchContext.getGraph();
AtlasTypeRegistry typeRegistry = searchContext.getTypeRegistry();
String queryString = searchParameters.getQuery();
List<PostProcessor> postProcessors = new ArrayList<>();
postProcessors.add(new ServiceTypeAggregator(searchContext.getTypeRegistry()));
try {
Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = atlasGraph.getGraphIndexClient().getAggregatedMetrics(queryString, aggregationFields);
Set<String> aggregationMetricNames = aggregatedMetrics.keySet();
AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
String searchedOnTypeName = searchParameters.getTypeName();
AtlasEntityType searchForEntityType = null;
if (searchedOnTypeName != null) {
searchForEntityType = typeRegistry.getEntityTypeByName(searchedOnTypeName);
}
Map<String, String> indexFieldNameCache = new HashMap<>();
for (String fieldName: aggregationFields) {
String indexFieldName = getIndexFieldNameForCommonFieldName(typeRegistry, fieldName);
indexFieldNameCache.put(fieldName, indexFieldName);
}
for (AtlasAttribute attribute: aggregationAttributes) {
String indexFieldName = attribute.getIndexFieldName();
for(String aggregationMetricName: aggregationMetricNames) {
for(PostProcessor postProcessor: postProcessors) {
if(postProcessor.needsProcessing(aggregationMetricName)) {
if (indexFieldName == null) {
//there is no index field name.
indexFieldName = attribute.getQualifiedName();
}
indexFieldNameCache.put(attribute.getQualifiedName(), indexFieldName);
}
AggregationContext aggregatorContext = new AggregationContext(queryString,
searchParameters.getEntityFilters(),
searchForEntityType,
aggregationFields,
aggregationAttributes,
indexFieldNameCache,
searchParameters.getExcludeDeletedEntities(),
searchParameters.getIncludeSubTypes());
Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = graphIndexClient.getAggregatedMetrics(aggregatorContext);
for (String aggregationMetricName: aggregatedMetrics.keySet()) {
for (PostProcessor postProcessor: postProcessors) {
if (postProcessor.needsProcessing(aggregationMetricName)) {
postProcessor.prepareForMetric(aggregationMetricName);
for(AtlasAggregationEntry aggregationEntry: aggregatedMetrics.get(aggregationMetricName)) {
for (AtlasAggregationEntry aggregationEntry: aggregatedMetrics.get(aggregationMetricName)) {
postProcessor.process(aggregationEntry);
}
......@@ -67,21 +107,10 @@ public class SearchAggregatorImpl implements SearchAggregator {
}
}
for(PostProcessor postProcessor: postProcessors) {
for (PostProcessor postProcessor: postProcessors) {
postProcessor.handleCompletion(aggregatedMetrics);
}
// remove entries with 0 counts
for (List<AtlasAggregationEntry> entries : aggregatedMetrics.values()) {
for (ListIterator<AtlasAggregationEntry> iter = entries.listIterator(); iter.hasNext(); ) {
AtlasAggregationEntry entry = iter.next();
if (entry.getCount() <= 0) {
iter.remove();
}
}
}
return aggregatedMetrics;
} catch (AtlasException e) {
LOG.error("Error encountered in post processing stage of aggrgation metrics collection. Empty metrics will be returned.", e);
......@@ -90,6 +119,20 @@ public class SearchAggregatorImpl implements SearchAggregator {
}
}
private String getIndexFieldNameForCommonFieldName(AtlasTypeRegistry typeRegistry, String fieldName) {
String indexFieldName = typeRegistry.getIndexFieldName(fieldName);
if(indexFieldName != null) {
return indexFieldName;
}
if(LOG.isDebugEnabled()) {
LOG.debug("Could not find index field name from type registry for attribute {}. Will use use the field name as is.", fieldName);
}
return fieldName;
}
static interface PostProcessor {
boolean needsProcessing(String metricName);
void prepareForMetric(String metricName);
......@@ -102,8 +145,8 @@ public class SearchAggregatorImpl implements SearchAggregator {
private static final String SERVICE_TYPE = "ServiceType";
private final AtlasTypeRegistry typeRegistry;
private List<AtlasAggregationEntry> entries;
private Map<String, AtlasAggregationEntry> entityType2MetricsMap;
private final List<AtlasAggregationEntry> entries = new ArrayList<>();;
private final Map<String, AtlasAggregationEntry> entityType2MetricsMap = new HashMap<>();
public ServiceTypeAggregator(AtlasTypeRegistry typeRegistry) {
this.typeRegistry = typeRegistry;
......@@ -118,8 +161,6 @@ public class SearchAggregatorImpl implements SearchAggregator {
public void prepareForMetric(String metricName) {
Map<String, AtlasAggregationEntry> serviceName2MetricsMap = new HashMap<>();
entries = new ArrayList<>();
//prepare the service map to aggregations
for(String serviceName: typeRegistry.getAllServiceTypes()) {
AtlasAggregationEntry serviceMetrics = new AtlasAggregationEntry(serviceName, 0);
......@@ -130,8 +171,6 @@ public class SearchAggregatorImpl implements SearchAggregator {
}
//prepare the map from entity type to aggregations
entityType2MetricsMap = new HashMap<>();
for(AtlasEntityType entityType: typeRegistry.getAllEntityTypes()) {
String serviceName = entityType.getServiceType();
......@@ -157,7 +196,16 @@ public class SearchAggregatorImpl implements SearchAggregator {
@Override
public void handleCompletion(Map<String, List<AtlasAggregationEntry>> aggregatedMetrics) {
aggregatedMetrics.put(SERVICE_TYPE, entries);
//remove all zero count entries.
for (int i = entries.size() - 1; i >= 0; i--) {
if (entries.get(i).getCount() == 0) {
entries.remove(i);
}
}
if (CollectionUtils.isNotEmpty(entries)) {
aggregatedMetrics.put(SERVICE_TYPE, entries);
}
}
}
}
repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
......@@ -19,14 +19,15 @@ package org.apache.atlas.repository.graph;
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.ChangedTypeDefs;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -65,13 +66,12 @@ public class SolrIndexHelper implements IndexChangeListener {
}
try {
AtlasGraph atlasGraph = AtlasGraphProvider.getGraphInstance();
AtlasGraphIndexClient atlasGraphIndexClient = atlasGraph.getGraphIndexClient();
AtlasGraph graph = AtlasGraphProvider.getGraphInstance();
AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
Map<String, Integer> propertyName2SearchWeightMap = gePropertiesWithSearchWeights();
atlasGraphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, propertyName2SearchWeightMap);
atlasGraphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX, getPropertiesForSuggestions(propertyName2SearchWeightMap));
graphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, propertyName2SearchWeightMap);
graphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX, getPropertiesForSuggestions(propertyName2SearchWeightMap));
} catch (AtlasException e) {
LOG.error("Error encountered in handling type system change notification.", e);
throw new RuntimeException("Error encountered in handling type system change notification.", e);
......@@ -79,7 +79,7 @@ public class SolrIndexHelper implements IndexChangeListener {
}
private List<String> getPropertiesForSuggestions(Map<String, Integer> propertyName2SearchWeightMap) {
List<String> propertiesForSuggestions = new ArrayList<>();
List<String> ret = new ArrayList<>();
for(Map.Entry<String, Integer> entry: propertyName2SearchWeightMap.entrySet()) {
if(entry.getValue().intValue() >= MIN_SEARCH_WEIGHT_FOR_SUGGESTIONS) {
......@@ -89,55 +89,64 @@ public class SolrIndexHelper implements IndexChangeListener {
LOG.debug("Adding the property {} for suggestions.", propertyName);
}
propertiesForSuggestions.add(propertyName);
ret.add(propertyName);
}
}
return propertiesForSuggestions;
return ret;
}
private Map<String, Integer> gePropertiesWithSearchWeights() {
Map<String, Integer> propertiesWithSearchWeights = new HashMap<>();
Collection<AtlasEntityDef> allEntityDefs = typeRegistry.getAllEntityDefs();
Map<String, Integer> ret = new HashMap<>();
Collection<AtlasEntityType> entityTypes = typeRegistry.getAllEntityTypes();
//the following two properties are specially added manually.
//as, they don't come in the entity definitions as attributes.
propertiesWithSearchWeights.put(CLASSIFICATION_TEXT_KEY, SEARCHWEIGHT_FOR_CLASSIFICATIONS);
propertiesWithSearchWeights.put(TYPE_NAME_PROPERTY_KEY, SEARCHWEIGHT_FOR_TYPENAME);
ret.put(typeRegistry.getIndexFieldName(CLASSIFICATION_TEXT_KEY), SEARCHWEIGHT_FOR_CLASSIFICATIONS);
ret.put(typeRegistry.getIndexFieldName(TYPE_NAME_PROPERTY_KEY), SEARCHWEIGHT_FOR_TYPENAME);
if (CollectionUtils.isNotEmpty(allEntityDefs)) {
for (AtlasEntityDef entityDef : allEntityDefs) {
processEntity(propertiesWithSearchWeights, entityDef);
if (CollectionUtils.isNotEmpty(entityTypes)) {
for (AtlasEntityType entityType : entityTypes) {
processEntityType(ret, entityType);
}
}
return propertiesWithSearchWeights;
return ret;
}
private void processEntity(Map<String, Integer> propertiesWithSearchWeights, AtlasEntityDef entityDef) {
for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) {
processAttributeDefinition(propertiesWithSearchWeights, entityDef, attributeDef);
private void processEntityType(Map<String, Integer> indexFieldNameWithSearchWeights, AtlasEntityType entityType) {
Map<String, AtlasAttribute> attributes = entityType.getAllAttributes();
if(MapUtils.isNotEmpty(attributes)) {
for (AtlasAttribute attribute : attributes.values()) {
processAttribute(indexFieldNameWithSearchWeights, attribute);
}
} else {
LOG.debug("No attributes are defined for entity {}", entityType.getTypeName());
}
}
private void processAttributeDefinition(Map<String, Integer> propertiesWithSearchWeights, AtlasEntityDef entityDef, AtlasAttributeDef attributeDef) {
if (GraphBackedSearchIndexer.isStringAttribute(attributeDef)) {
final String propertyName = GraphBackedSearchIndexer.getEncodedPropertyName(entityDef.getName(), attributeDef);
int searchWeight = attributeDef.getSearchWeight();
private void processAttribute(Map<String, Integer> indexFieldNameWithSearchWeights, AtlasAttribute attribute) {
if (GraphBackedSearchIndexer.isStringAttribute(attribute)) {
int searchWeight = attribute.getSearchWeight();
if (searchWeight == DEFAULT_SEARCHWEIGHT) {
//We will use default search weight of 3 for string attributes.
//this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher.
searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
} else if (!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) { //validate the value provided in the model.
LOG.warn("Invalid search weight {} for attribute {}.{}. Will use default {}", searchWeight, entityDef.getName(), propertyName, DEFAULT_SEARCHWEIGHT_FOR_STRINGS);
LOG.warn("Invalid search weight {} for attribute {}. Will use default {}",
searchWeight, attribute.getQualifiedName(), DEFAULT_SEARCHWEIGHT_FOR_STRINGS);
searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Applying search weight {} for attribute {}.{}", searchWeight, entityDef.getName(), propertyName);
LOG.debug("Applying search weight {} for attribute {}", searchWeight, attribute.getQualifiedName());
}
propertiesWithSearchWeights.put(propertyName, searchWeight);
indexFieldNameWithSearchWeights.put(attribute.getIndexFieldName(), searchWeight);
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
......@@ -352,7 +352,7 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
try {
typeDefStore.init();
loadBootstrapTypeDefs();
typeDefStore.notifyLoadCompletion();
try {
AtlasAuthorizerFactory.getAtlasAuthorizer();
} catch (Throwable t) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
......@@ -1006,6 +1006,17 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
}
@Override
public void notifyLoadCompletion(){
for (TypeDefChangeListener changeListener : typeDefChangeListeners) {
try {
changeListener.onLoadCompletion();
} catch (Throwable t) {
LOG.error("OnLoadCompletion failed for listener {}", changeListener.getClass().getName(), t);
}
}
}
private void tryUpdateByName(String name, AtlasBaseTypeDef typeDef, AtlasTransientTypeRegistry ttr) throws AtlasBaseException {
try {
ttr.updateTypeByName(name, typeDef);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment