ATLAS-2229: DSL implementation using ANTLR #2

intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java

@@ -392,8 +392,8 @@ public class SearchParameters implements Serializable {
         GT(new String[]{">", "gt"}),
         LTE(new String[]{"<=", "lte"}),
         GTE(new String[]{">=", "gte"}),
-        EQ(new String[]{"eq", "="}),
-        NEQ(new String[]{"neq", "!="}),
+        EQ(new String[]{"=", "eq"}),
+        NEQ(new String[]{"!=", "neq"}),
         IN(new String[]{"in", "IN"}),
         LIKE(new String[]{"like", "LIKE"}),
         STARTS_WITH(new String[]{"startsWith", "STARTSWITH", "begins_with", "BEGINS_WITH"}),

repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java

@@ -685,7 +685,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
             throw new AtlasBaseException(DISCOVERY_QUERY_FAILED, query);
         }
 
-        QueryProcessor queryProcessor  = new QueryProcessor(typeRegistry);
+        QueryProcessor queryProcessor  = new QueryProcessor(typeRegistry, limit, offset);
         Expression     validExpression = queryProcessor.validate(expression);
         GremlinQuery   gremlinQuery    = new GremlinTranslator(queryProcessor, validExpression).translate();
 
@@ -928,13 +928,24 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
 
     @Override
     public String getDslQueryUsingTypeNameClassification(String query, String typeName, String classification) {
+        final String whereDSLKeyword = "where";
+        final String limitDSLKeyword = "limit";
+        final String whereFormat = whereDSLKeyword + " %s";
+
         String queryStr = query == null ? "" : query;
 
-        if (org.apache.commons.lang3.StringUtils.isNoneEmpty(typeName)) {
+        if (StringUtils.isNotEmpty(typeName)) {
+            if(StringUtils.isNotEmpty(query)) {
+                String s = query.toLowerCase();
+                if(!s.startsWith(whereDSLKeyword) && !s.startsWith(limitDSLKeyword)) {
+                    queryStr = String.format(whereFormat, query);
+                }
+            }
+
             queryStr = escapeTypeName(typeName) + " " + queryStr;
         }
 
-        if (org.apache.commons.lang3.StringUtils.isNoneEmpty(classification)) {
+        if (StringUtils.isNotEmpty(classification)) {
             // isa works with a type name only - like hive_column isa PII; it doesn't work with more complex query
             if (StringUtils.isEmpty(query)) {
                 queryStr += (" isa " + classification);

repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class IdentifierHelper {
+
+    public static String stripQuotes(String quotedIdentifier) {
+        String ret = quotedIdentifier;
+
+        if (isQuoted(quotedIdentifier)) {
+            ret = quotedIdentifier.substring(1, quotedIdentifier.length() - 1);
+        }
+
+        return ret;
+    }
+
+    public static Advice create(QueryProcessor.Context context,
+                                org.apache.atlas.query.Lookup lookup,
+                                String identifier) {
+        Advice ia = new Advice(identifier);
+        ia.update(lookup, context);
+        return ia;
+    }
+
+    private static String extract(Pattern p, String s) {
+        Matcher m = p.matcher(s);
+        return m.find() ? m.group(1) : s;
+    }
+
+    public static String getQualifiedName(org.apache.atlas.query.Lookup lookup,
+                                          QueryProcessor.Context context,
+                                          String name) {
+        return lookup.getQualifiedName(context, name);
+    }
+
+    public static boolean isQuoted(String val) {
+        boolean ret = false;
+
+        if (val != null && val.length() > 1) {
+            char first = val.charAt(0);
+            char last  = val.charAt(val.length() - 1);
+
+            if (first == last && (first == '\'' || first == '"' || first == '`')) {
+                ret = true;
+            }
+        }
+
+        return ret;
+    }
+
+    public static String removeQuotes(String rhs) {
+        return rhs.replace("\"", "").replace("'", "");
+    }
+
+    public static String getQuoted(String s) {
+        return String.format("'%s'", s);
+    }
+
+    public static class Advice {
+        private String raw;
+        private String actual;
+        private String[] parts;
+        private String typeName;
+        private String attributeName;
+        private boolean isPrimitive;
+        private String edgeLabel;
+        private String edgeDirection;
+        private boolean introduceType;
+        private boolean hasSubtypes;
+        private String subTypes;
+        private boolean isTrait;
+        private boolean newContext;
+        private boolean isAttribute;
+        private String qualifiedName;
+        private boolean isDate;
+
+        public Advice(String s) {
+            this.raw = removeQuotes(s);
+            this.actual = IdentifierHelper.stripQuotes(raw);
+        }
+
+        private void update(org.apache.atlas.query.Lookup lookup, QueryProcessor.Context context) {
+            newContext = context.isEmpty();
+            if(!newContext) {
+                if(context.aliasMap.containsKey(this.raw)) {
+                    raw = context.aliasMap.get(this.raw);
+                }
+
+                updateParts();
+                updateTypeInfo(lookup, context);
+                isTrait = lookup.isTraitType(context);
+                updateEdgeInfo(lookup, context);
+                introduceType = !context.hasAlias(parts[0]);
+                updateSubTypes(lookup, context);
+            }
+        }
+
+        private void updateSubTypes(org.apache.atlas.query.Lookup lookup, QueryProcessor.Context context) {
+            if(isTrait) {
+                return;
+            }
+
+            hasSubtypes = lookup.doesTypeHaveSubTypes(context);
+            if(hasSubtypes) {
+                subTypes = lookup.getTypeAndSubTypes(context);
+            }
+        }
+
+        private void updateEdgeInfo(org.apache.atlas.query.Lookup lookup, QueryProcessor.Context context) {
+            if(isPrimitive == false && isTrait == false) {
+                edgeLabel = lookup.getRelationshipEdgeLabel(context, attributeName);
+                edgeDirection = "OUT";
+                typeName = lookup.getTypeFromEdge(context, attributeName);
+            }
+        }
+
+        private void updateTypeInfo(org.apache.atlas.query.Lookup lookup, QueryProcessor.Context context) {
+            if(parts.length == 1) {
+                typeName = context.getActiveTypeName();
+                attributeName = parts[0];
+                isAttribute = lookup.hasAttribute(context, typeName);
+                qualifiedName = lookup.getQualifiedName(context, attributeName);
+                isPrimitive = lookup.isPrimitive(context, attributeName);
+
+                setIsDate(lookup, context);
+            }
+
+            if(parts.length == 2) {
+                if(context.hasAlias(parts[0])) {
+                    typeName = context.getTypeNameFromAlias(parts[0]);
+                    attributeName = parts[1];
+                    isPrimitive = lookup.isPrimitive(context, attributeName);
+                    setIsDate(lookup, context);
+                }
+                else {
+                    isAttribute = lookup.hasAttribute(context, parts[0]);
+                    if(isAttribute) {
+                        attributeName = parts[0];
+                        isPrimitive = lookup.isPrimitive(context, attributeName);
+                        setIsDate(lookup, context);
+                    } else {
+                        typeName = parts[0];
+                        attributeName = parts[1];
+                        isPrimitive = lookup.isPrimitive(context, attributeName);
+                        setIsDate(lookup, context);
+                    }
+                }
+
+                qualifiedName = lookup.getQualifiedName(context, attributeName);
+            }
+        }
+
+        private void setIsDate(Lookup lookup, QueryProcessor.Context context) {
+            if(isPrimitive) {
+                isDate = lookup.isDate(context, attributeName);
+            }
+        }
+
+        private void updateParts() {
+            parts = StringUtils.split(raw, ".");
+        }
+
+        public String getQualifiedName() {
+            return qualifiedName;
+        }
+
+        public boolean isPrimitive() {
+            return isPrimitive;
+        }
+
+        public String getEdgeLabel() {
+            return edgeLabel;
+        }
+
+        public String getTypeName() {
+            return typeName;
+        }
+
+        public boolean getIntroduceType() {
+            return introduceType;
+        }
+
+        public boolean isTrait() {
+            return isTrait;
+        }
+
+        public boolean hasSubtypes() {
+            return hasSubtypes;
+        }
+
+        public String getSubTypes() {
+            return subTypes;
+        }
+
+        public String get() {
+            return actual;
+        }
+
+        public boolean isNewContext() {
+            return newContext;
+        }
+
+        public boolean isDate() {
+            return isDate;
+        }
+    }
+}

repository/src/main/java/org/apache/atlas/query/Lookup.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query;
+
+import org.apache.atlas.type.AtlasType;
+
+public interface Lookup {
+    AtlasType getType(String typeName);
+
+    String getQualifiedName(QueryProcessor.Context context, String name);
+
+    boolean isPrimitive(QueryProcessor.Context context, String attributeName);
+
+    String getRelationshipEdgeLabel(QueryProcessor.Context context, String attributeName);
+
+    boolean hasAttribute(QueryProcessor.Context context, String typeName);
+
+    boolean doesTypeHaveSubTypes(QueryProcessor.Context context);
+
+    String getTypeAndSubTypes(QueryProcessor.Context context);
+
+    boolean isTraitType(QueryProcessor.Context context);
+
+    String getTypeFromEdge(QueryProcessor.Context context, String item);
+
+    boolean isDate(QueryProcessor.Context context, String attributeName);
+}

repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLLexer.java

-// Generated from AtlasDSLLexer.g4 by ANTLR 4.7
+// Generated from repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLLexer.g4 by ANTLR 4.7
 package org.apache.atlas.query.antlr4;
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.CharStream;

repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParser.g4

@@ -20,8 +20,6 @@ parser grammar AtlasDSLParser;
 
 options { tokenVocab=AtlasDSLLexer; }
 
-// Start of rules, bottom-up (rules at the end are built using the core rules)
-
 // Core rules
 identifier: ID ;
 
@@ -98,8 +96,6 @@ selectClause: K_SELECT selectExpr ;
 
 singleQrySrc: fromClause | whereClause | fromExpression | expr ;
 
-loopExpression: K_LOOP K_LPAREN query K_RPAREN NUMBER? (K_AS identifier)? ;
-
 groupByExpression: K_GROUPBY K_LPAREN selectExpr K_RPAREN ;
 
 commaDelimitedQueries: singleQrySrc (K_COMMA singleQrySrc)* ;
@@ -108,10 +104,7 @@ spaceDelimitedQueries: singleQrySrc singleQrySrc* ;
 
 querySrc: commaDelimitedQueries | spaceDelimitedQueries ;
 
-query: querySrc loopExpression?
-                groupByExpression?
+query: querySrc groupByExpression?
                 selectClause?
                 orderByExpr?
                 limitOffset? ;
-
-queryWithPath: query (K_WITHPATH)? ;

repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParserBaseVisitor.java

-// Generated from AtlasDSLParser.g4 by ANTLR 4.7
+// Generated from repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParser.g4 by ANTLR 4.7
 package org.apache.atlas.query.antlr4;
 import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
 
@@ -248,13 +248,6 @@ public class AtlasDSLParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
 	 * <p>The default implementation returns the result of calling
 	 * {@link #visitChildren} on {@code ctx}.</p>
 	 */
-	@Override public T visitLoopExpression(AtlasDSLParser.LoopExpressionContext ctx) { return visitChildren(ctx); }
-	/**
-	 * {@inheritDoc}
-	 *
-	 * <p>The default implementation returns the result of calling
-	 * {@link #visitChildren} on {@code ctx}.</p>
-	 */
 	@Override public T visitGroupByExpression(AtlasDSLParser.GroupByExpressionContext ctx) { return visitChildren(ctx); }
 	/**
 	 * {@inheritDoc}
@@ -284,11 +277,4 @@ public class AtlasDSLParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
 	 * {@link #visitChildren} on {@code ctx}.</p>
 	 */
 	@Override public T visitQuery(AtlasDSLParser.QueryContext ctx) { return visitChildren(ctx); }
-	/**
-	 * {@inheritDoc}
-	 *
-	 * <p>The default implementation returns the result of calling
-	 * {@link #visitChildren} on {@code ctx}.</p>
-	 */
-	@Override public T visitQueryWithPath(AtlasDSLParser.QueryWithPathContext ctx) { return visitChildren(ctx); }
 }
\ No newline at end of file

repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParserVisitor.java

-// Generated from AtlasDSLParser.g4 by ANTLR 4.7
+// Generated from repository/src/main/java/org/apache/atlas/query/antlr4/AtlasDSLParser.g4 by ANTLR 4.7
 package org.apache.atlas.query.antlr4;
 import org.antlr.v4.runtime.tree.ParseTreeVisitor;
 
@@ -209,12 +209,6 @@ public interface AtlasDSLParserVisitor<T> extends ParseTreeVisitor<T> {
 	 */
 	T visitSingleQrySrc(AtlasDSLParser.SingleQrySrcContext ctx);
 	/**
-	 * Visit a parse tree produced by {@link AtlasDSLParser#loopExpression}.
-	 * @param ctx the parse tree
-	 * @return the visitor result
-	 */
-	T visitLoopExpression(AtlasDSLParser.LoopExpressionContext ctx);
-	/**
 	 * Visit a parse tree produced by {@link AtlasDSLParser#groupByExpression}.
 	 * @param ctx the parse tree
 	 * @return the visitor result
@@ -244,10 +238,4 @@ public interface AtlasDSLParserVisitor<T> extends ParseTreeVisitor<T> {
 	 * @return the visitor result
 	 */
 	T visitQuery(AtlasDSLParser.QueryContext ctx);
-	/**
-	 * Visit a parse tree produced by {@link AtlasDSLParser#queryWithPath}.
-	 * @param ctx the parse tree
-	 * @return the visitor result
-	 */
-	T visitQueryWithPath(AtlasDSLParser.QueryWithPathContext ctx);
 }
\ No newline at end of file

webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java

@@ -24,7 +24,10 @@ import com.google.common.base.Preconditions;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasConfiguration;
 import org.apache.atlas.classification.InterfaceAudience;
+import org.apache.atlas.discovery.AtlasDiscoveryService;
+import org.apache.atlas.model.discovery.AtlasSearchResult;
 import org.apache.atlas.query.QueryParams;
+import org.apache.atlas.type.AtlasType;
 import org.apache.atlas.utils.AtlasJson;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.atlas.utils.ParamChecker;
@@ -68,6 +71,7 @@ public class MetadataDiscoveryResource {
     private final  boolean       gremlinSearchEnabled;
     private static Configuration applicationProperties          = null;
     private static final String  ENABLE_GREMLIN_SEARCH_PROPERTY = "atlas.search.gremlin.enable";
+    private final AtlasDiscoveryService atlasDiscoveryService;
 
     /**
      * Created by the Guice ServletModule and injected with the
@@ -76,7 +80,8 @@ public class MetadataDiscoveryResource {
      * @param configuration configuration
      */
     @Inject
-    public MetadataDiscoveryResource(Configuration configuration) {
+    public MetadataDiscoveryResource(AtlasDiscoveryService atlasDiscoveryService, Configuration configuration) {
+        this.atlasDiscoveryService = atlasDiscoveryService;
         applicationProperties  = configuration;
         gremlinSearchEnabled   = applicationProperties != null && applicationProperties.getBoolean(ENABLE_GREMLIN_SEARCH_PROPERTY, false);
     }
@@ -149,7 +154,8 @@ public class MetadataDiscoveryResource {
 
             dslQuery = ParamChecker.notEmpty(dslQuery, "dslQuery cannot be null");
             QueryParams queryParams = validateQueryParams(limit, offset);
-            final String jsonResultStr = ""; // TODO-typeSystem-removal: discoveryService.searchByDSL(dslQuery, queryParams);
+            AtlasSearchResult result = atlasDiscoveryService.searchUsingDslQuery(dslQuery, queryParams.limit(), queryParams.offset());
+            final String jsonResultStr = AtlasType.toJson(result.getEntities());
 
             ObjectNode response = new DSLJSONResponseBuilder().results(jsonResultStr).query(dslQuery).build();
 
@@ -195,62 +201,6 @@ public class MetadataDiscoveryResource {
     }
 
     /**
-     * Search using raw gremlin query format.
-     *
-     * @param gremlinQuery search query in raw gremlin format.
-     * @return JSON representing the type and results.
-     */
-    @GET
-    @Path("search/gremlin")
-    @Consumes(Servlets.JSON_MEDIA_TYPE)
-    @Produces(Servlets.JSON_MEDIA_TYPE)
-    @InterfaceAudience.Private
-    public Response searchUsingGremlinQuery(@QueryParam("query") String gremlinQuery) {
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("==> MetadataDiscoveryResource.searchUsingGremlinQuery({})", gremlinQuery);
-        }
-
-        AtlasPerfTracer perf = null;
-        try {
-            if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
-                perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MetadataDiscoveryResource.searchUsingGremlinQuery(" + gremlinQuery + ")");
-            }
-
-            if (!gremlinSearchEnabled) {
-                throw new Exception("Gremlin search is not enabled.");
-            }
-
-            gremlinQuery = ParamChecker.notEmpty(gremlinQuery, "gremlinQuery cannot be null or empty");
-            final List<Map<String, String>> results = new ArrayList<>(); // TODO-typeSystem-removal: discoveryService.searchByGremlin(gremlinQuery);
-
-            ObjectNode response = AtlasJson.createV1ObjectNode();
-            response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
-            response.put(AtlasClient.QUERY, gremlinQuery);
-            response.put(AtlasClient.QUERY_TYPE, QUERY_TYPE_GREMLIN);
-
-            response.putPOJO(AtlasClient.RESULTS, results);
-            response.put(AtlasClient.COUNT, results.size());
-
-            return Response.ok(response).build();
-        } catch (IllegalArgumentException e) {
-            LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
-            throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
-        } catch (WebApplicationException e) {
-            LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
-            throw e;
-        } catch (Throwable e) {
-            LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
-            throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
-        } finally {
-            AtlasPerfTracer.log(perf);
-
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("<== MetadataDiscoveryResource.searchUsingGremlinQuery({})", gremlinQuery);
-            }
-        }
-    }
-
-    /**
      * Search using full text search.
      *
      * @param query search query.
@@ -277,7 +227,8 @@ public class MetadataDiscoveryResource {
 
             query = ParamChecker.notEmpty(query, "query cannot be null or empty");
             QueryParams queryParams = validateQueryParams(limit, offset);
-            final String jsonResultStr = ""; // TODO-typeSystem-removal: discoveryService.searchByFullText(query, queryParams);
+            AtlasSearchResult result = atlasDiscoveryService.searchUsingFullTextQuery(query, false, queryParams.limit(), queryParams.offset());
+            final String jsonResultStr = AtlasType.toJson(result.getEntities());
             ArrayNode rowsJsonArr = AtlasJson.parseToV1ArrayNode(jsonResultStr);
 
             ObjectNode response = new FullTextJSonResponseBuilder().results(rowsJsonArr).query(query).build();