ATLAS-3257: Enhance Classification Basic Search

graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java

@@ -18,8 +18,6 @@
 
 package org.apache.atlas.repository.graphdb;
 
-import java.util.Iterator;
-
 /**
  * Represents an index query parameter for use with AtlasGraph queries.
  */

graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java

@@ -370,6 +370,11 @@ public class Solr6Index implements IndexProvider {
         String analyzer = ParameterType.STRING_ANALYZER.findParameter(information.getParameters(), null);
         if (analyzer != null) {
             //If the key have a tokenizer, we try to get it by reflection
+            // some referred classes might not be able to be found via SystemClassLoader
+            // since they might be associated with other classloader, in this situation
+            // ClassNotFound exception will be thrown. instead of using SystemClassLoader
+            // for all classes, we find its classloader first and then load the class, please
+            // call - instantiateUsingClassLoader()
             try {
                 ((Constructor<Tokenizer>) ClassLoader.getSystemClassLoader().loadClass(analyzer)
                         .getConstructor()).newInstance();
@@ -389,6 +394,17 @@ public class Solr6Index implements IndexProvider {
         }
     }
 
+    private void instantiateUsingClassLoader(String analyzer) throws PermanentBackendException {
+        if (analyzer == null) return;
+        try {
+            Class analyzerClass = Class.forName(analyzer);
+            ClassLoader cl = analyzerClass.getClassLoader();
+            ((Constructor<Tokenizer>) cl.loadClass(analyzer).getConstructor()).newInstance();
+        } catch (final ReflectiveOperationException e) {
+            throw new PermanentBackendException(e.getMessage(),e);
+        }
+    }
+
     @Override
     public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever information,
                        BaseTransaction tx) throws BackendException {

graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java

@@ -33,10 +33,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-
-/**
- *
- */
 public abstract class AbstractGraphDatabaseTest {
 
     protected static final String WEIGHT_PROPERTY = "weight";
@@ -112,13 +108,8 @@ public abstract class AbstractGraphDatabaseTest {
             //ok
             t.printStackTrace();
         }
-
-
     }
 
-
-
-
     protected final <V, E> AtlasGraph<V, E> getGraph() {
         if (graph == null) {
             graph = new AtlasJanusGraph();
@@ -184,5 +175,4 @@ public abstract class AbstractGraphDatabaseTest {
             return exceptionThrown;
         }
     }
-
 }

intg/src/test/java/org/apache/atlas/TestUtilsV2.java

@@ -544,6 +544,8 @@ public final class TestUtilsV2 {
     public static final String COLUMN_TYPE = "column_type";
     public static final String TABLE_NAME = "bar";
     public static final String CLASSIFICATION = "classification";
+    public static final String FETL_CLASSIFICATION = "fetl" + CLASSIFICATION;
+
     public static final String PII = "PII";
     public static final String PHI = "PHI";
     public static final String SUPER_TYPE_NAME = "Base";

repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java

@@ -111,7 +111,7 @@ public class EntitySearchProcessor extends SearchProcessor {
         StringBuilder indexQuery = new StringBuilder();
 
         if (typeSearchByIndex) {
-            constructTypeTestQuery(indexQuery, typeAndSubTypesQryStr);
+            graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(indexQuery, typeAndSubTypesQryStr);
 
             // TypeName check to be done in-memory as well to address ATLAS-2121 (case sensitivity)
             inMemoryPredicate = typeNamePredicate;
@@ -131,9 +131,8 @@ public class EntitySearchProcessor extends SearchProcessor {
         }
 
         if (indexQuery.length() > 0) {
-            if (context.getSearchParameters().getExcludeDeletedEntities()) {
-                constructStateTestQuery(indexQuery);
-            }
+
+            graphIndexQueryBuilder.addActiveStateQueryFilter(indexQuery);
 
             String indexQueryString = STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")");
 

repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
+import static org.apache.atlas.discovery.SearchProcessor.INDEX_SEARCH_PREFIX;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
+
+import org.apache.atlas.repository.Constants;
+import org.apache.commons.lang3.StringUtils;
+
+public class GraphIndexQueryBuilder {
+    SearchContext context;
+
+    GraphIndexQueryBuilder(SearchContext context) {
+        this.context = context;
+    }
+
+    void addClassificationTypeFilter(StringBuilder indexQuery) {
+        if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
+            String classificationName = context.getSearchParameters().getClassification();
+            if (indexQuery.length() != 0) {
+                indexQuery.append(" AND ");
+            }
+
+            indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName)
+                .append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
+                .append('\"').append(':').append(classificationName).append(")");
+        }
+    }
+
+    void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) {
+        if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
+            String classificationTypesQryStr = context.getClassificationTypesQryStr();
+
+            if (indexQuery.length() != 0) {
+                indexQuery.append(" AND ");
+            }
+
+            indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
+                .append("\"").append(":" + classificationTypesQryStr).append(" OR ").append(INDEX_SEARCH_PREFIX)
+                .append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + classificationTypesQryStr).append(")");
+        }
+    }
+
+    void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) {
+        if (indexQuery != null && context.getClassificationType() != null) {
+            if (context.getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() == MATCH_ALL_CLASSIFIED) {
+                if (indexQuery.length() != 0) {
+                    indexQuery.append(" AND ");
+                }
+                indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"")
+                    .append(CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]")
+                    .append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"")
+                    .append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]").append(")");
+
+            } else if (context.getClassificationType() == MATCH_ALL_NOT_CLASSIFIED) {
+                if (indexQuery.length() != 0) {
+                    indexQuery.append(" AND ");
+                }
+                indexQuery.append("(").append("-").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
+                    .append("\"").append(":" + "[* TO *]").append(" AND ").append("-")
+                    .append(INDEX_SEARCH_PREFIX).append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
+                    .append("\"").append(":" + "[* TO *]").append(")");
+            }
+        }
+    }
+
+    void addActiveStateQueryFilter(StringBuilder indexQuery){
+        if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery != null) {
+            if (indexQuery.length() != 0) {
+                indexQuery.append(" AND ");
+            }
+            indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(STATE_PROPERTY_KEY)
+                      .append("\"").append(":" + "ACTIVE").append(")");
+        }
+    }
+
+    void addTypeAndSubTypesQueryFilter(StringBuilder indexQuery, String typeAndAllSubTypesQryStr) {
+        if (indexQuery != null && StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
+            if (indexQuery.length() > 0) {
+                indexQuery.append(" AND ");
+            }
+
+            indexQuery.append("(").append(INDEX_SEARCH_PREFIX + "\"").append(Constants.TYPE_NAME_PROPERTY_KEY)
+                .append("\":").append(typeAndAllSubTypesQryStr).append(")");
+        }
+    }
+}
\ No newline at end of file

repository/src/main/java/org/apache/atlas/discovery/SearchContext.java

@@ -63,6 +63,7 @@ public class SearchContext {
     private final Set<String>             entityAttributes;
     private final AtlasEntityType         entityType;
     private final AtlasClassificationType classificationType;
+    private final String                  classificationName;
     private       SearchProcessor         searchProcessor;
     private       boolean                 terminateSearch = false;
     private final Set<String>             typeAndSubTypes;
@@ -74,10 +75,8 @@ public class SearchContext {
     public final static AtlasClassificationType MATCH_ALL_CLASSIFIED              = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
     public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED          = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
 
-
     public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
-        String classificationName = searchParameters.getClassification();
-
+        this.classificationName = searchParameters.getClassification();
         this.searchParameters   = searchParameters;
         this.typeRegistry       = typeRegistry;
         this.graph              = graph;
@@ -92,7 +91,7 @@ public class SearchContext {
         }
 
         // Validate if the classification exists
-        if (StringUtils.isNotEmpty(classificationName) && classificationType == null) {
+        if ((StringUtils.isNotEmpty(classificationName) && classificationType == null && !classificationName.contains(WILDCARD_CLASSIFICATIONS))) {
             throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName);
         }
 
@@ -239,7 +238,21 @@ public class SearchContext {
     }
 
     boolean needClassificationProcessor() {
-        return classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()));
+        return (classificationType != null || isWildCardSearch());
+    }
+
+    boolean isBuiltInClassificationType() {
+        return getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION
+            || getClassificationType() == MATCH_ALL_CLASSIFIED
+            || getClassificationType() == MATCH_ALL_NOT_CLASSIFIED;
+    }
+
+    boolean isWildCardSearch () {
+        String classification = getSearchParameters().getClassification();
+        if (StringUtils.isNotEmpty(classification) && getClassificationType() == null) {
+            return classification.contains("*");
+        }
+        return false;
     }
 
     boolean needEntityProcessor() {
@@ -263,7 +276,10 @@ public class SearchContext {
 
     private void validateAttributes(final AtlasStructType structType, final String... attributeNames) throws AtlasBaseException {
         for (String attributeName : attributeNames) {
-            if (StringUtils.isNotEmpty(attributeName) && structType.getAttributeType(attributeName) == null) {
+            if (StringUtils.isNotEmpty(attributeName) && (structType == null || structType.getAttributeType(attributeName) == null)) {
+                if (structType == null) {
+                    throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL");
+                }
                 throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, structType.getTypeName());
             }
         }

repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java

@@ -110,13 +110,14 @@ public abstract class SearchProcessor {
         OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_NULL, getNotNullPredicateGenerator());
     }
 
-    protected final SearchContext   context;
-    protected       SearchProcessor nextProcessor;
-    protected       Predicate       inMemoryPredicate;
-
+    protected final SearchContext          context;
+    protected       SearchProcessor        nextProcessor;
+    protected       Predicate              inMemoryPredicate;
+    protected       GraphIndexQueryBuilder graphIndexQueryBuilder;
 
     protected SearchProcessor(SearchContext context) {
         this.context = context;
+        this.graphIndexQueryBuilder = new GraphIndexQueryBuilder(context);
     }
 
     public void addProcessor(SearchProcessor processor) {
@@ -239,13 +240,30 @@ public abstract class SearchProcessor {
         return ret;
     }
 
-    protected void constructTypeTestQuery(StringBuilder indexQuery, String typeAndAllSubTypesQryStr) {
-        if (StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
-            if (indexQuery.length() > 0) {
-                indexQuery.append(AND_STR);
-            }
+    protected void filterWhiteSpaceClassification(List<AtlasVertex> entityVertices) {
+        if (CollectionUtils.isNotEmpty(entityVertices)) {
+
+            boolean hasExactMatch = false;
+            Iterator<AtlasVertex> it = entityVertices.iterator();
+
+            while (it.hasNext()) {
+                AtlasVertex entityVertex = it.next();
+                List<String> classificationNames = AtlasGraphUtilsV2.getClassificationNames(entityVertex);
+                if (CollectionUtils.isNotEmpty(classificationNames) && classificationNames.contains(context.getClassificationType().getTypeName())) {
+                    hasExactMatch = true;
+                }
+
+                if (hasExactMatch) continue;
+
+                classificationNames = AtlasGraphUtilsV2.getPropagatedClassificationNames(entityVertex);
+                if (CollectionUtils.isNotEmpty(classificationNames) && classificationNames.contains(context.getClassificationType().getTypeName())) {
+                    hasExactMatch = true;
+                }
 
-            indexQuery.append(INDEX_SEARCH_PREFIX + "\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr);
+                if (!hasExactMatch) {
+                    it.remove();
+                }
+            }
         }
     }
 
@@ -322,14 +340,6 @@ public abstract class SearchProcessor {
         }
     }
 
-    protected void constructStateTestQuery(StringBuilder indexQuery) {
-        if (indexQuery.length() > 0) {
-            indexQuery.append(AND_STR);
-        }
-
-        indexQuery.append(INDEX_SEARCH_PREFIX + "\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
-    }
-
     private boolean isIndexSearchable(FilterCriteria filterCriteria, AtlasStructType structType) throws AtlasBaseException {
         String      qualifiedName = structType.getQualifiedAttributeName(filterCriteria.getAttributeName());
         Set<String> indexedKeys   = context.getIndexedKeys();
@@ -747,7 +757,7 @@ public abstract class SearchProcessor {
         return false;
     }
 
-    protected List<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, List<AtlasVertex> vertices) {
+    protected Collection<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, Collection<AtlasVertex> vertices) {
         if (idxQueryResult != null) {
             while (idxQueryResult.hasNext()) {
                 AtlasVertex vertex = idxQueryResult.next().getVertex();
@@ -759,7 +769,7 @@ public abstract class SearchProcessor {
         return vertices;
     }
 
-    protected List<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, List<AtlasVertex> vertices) {
+    protected Collection<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, Collection<AtlasVertex> vertices) {
         if (iterator != null) {
             while (iterator.hasNext()) {
                 AtlasVertex vertex = iterator.next();

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java

@@ -50,7 +50,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.Iterator;
@@ -58,9 +58,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
 import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
 import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
 import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
@@ -634,4 +636,21 @@ public class AtlasGraphUtilsV2 {
     public static String getIndexSearchPrefix() {
         return INDEX_SEARCH_PREFIX;
     }
+
+    public static List<String> getClassificationNames(AtlasVertex entityVertex) {
+        return getClassificationNamesHelper(entityVertex, CLASSIFICATION_NAMES_KEY);
+    }
+
+    public static List<String> getPropagatedClassificationNames(AtlasVertex entityVertex) {
+        return getClassificationNamesHelper(entityVertex, PROPAGATED_CLASSIFICATION_NAMES_KEY);
+    }
+
+    private static List<String> getClassificationNamesHelper(AtlasVertex entityVertex, String propertyKey) {
+        List<String> classificationNames = null;
+        String classificationNamesString =  entityVertex.getProperty(propertyKey, String.class);
+        if (StringUtils.isNotEmpty(classificationNamesString)) {
+            classificationNames = Arrays.asList(classificationNamesString.split("\\|"));
+        }
+        return classificationNames;
+    }
 }

webapp/src/test/resources/json/search-parameters/tag-filters.json

@@ -12,22 +12,5 @@
       "classification":"fooTag"
     },
     "expectedCount": 1
-  },
-  {
-    "testDescription": "Search for exact Tag name order by",
-    "searchParameters": {
-      "entityFilters":null,
-      "tagFilters":null,
-      "attributes":null,
-      "query":null,
-      "excludeDeletedEntities":true,
-      "limit":25,
-      "typeName":null,
-      "sortBy": "name",
-      "sortOrder": "DESCENDING",
-      "classification":"fooTag"
-    },
-    "expectedCount": 1
   }
-
 ]
\ No newline at end of file