ATLAS-2174: Query length validations and path normalization

client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java

@@ -356,7 +356,7 @@ public class AtlasClient extends AtlasBaseClient {
         JSONObject response = callAPIWithRetries(api, null, new ResourceCreator() {
             @Override
             public WebResource createResource() {
-                WebResource resource = getResource(api.getPath());
+                WebResource resource = getResource(api.getNormalizedPath());
                 resource = resource.queryParam(TYPE, category.name());
                 return resource;
             }
@@ -924,12 +924,12 @@ public class AtlasClient extends AtlasBaseClient {
 
     @VisibleForTesting
     public WebResource getResource(API api, String... params) {
-        return getResource(api.getPath(), params);
+        return getResource(api.getNormalizedPath(), params);
     }
 
     @VisibleForTesting
     public WebResource getResource(API_V1 apiV1, String... params) {
-        return getResource(apiV1.getPath(), params);
+        return getResource(apiV1.getNormalizedPath(), params);
     }
 
     @VisibleForTesting

client/client-v1/src/test/java/org/apache/atlas/AtlasClientTest.java

@@ -108,6 +108,7 @@ public class AtlasClientTest {
 
     private WebResource.Builder setupBuilder(AtlasClient.API_V1 api, WebResource webResource) {
         when(webResource.path(api.getPath())).thenReturn(service);
+        when(webResource.path(api.getNormalizedPath())).thenReturn(service);
         return getBuilder(service);
     }
 

client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java

@@ -46,6 +46,7 @@ import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
 import java.io.IOException;
 import java.net.ConnectException;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 
@@ -151,7 +152,7 @@ public abstract class AtlasBaseClient {
     }
 
     public boolean isServerReady() throws AtlasServiceException {
-        WebResource resource   = getResource(API_VERSION.getPath());
+        WebResource resource   = getResource(API_VERSION.getNormalizedPath());
         try {
             callAPIWithResource(API_VERSION, resource, null, JSONObject.class);
             return true;
@@ -175,7 +176,7 @@ public abstract class AtlasBaseClient {
      */
     public String getAdminStatus() throws AtlasServiceException {
         String      result    = AtlasBaseClient.UNKNOWN_STATUS;
-        WebResource resource  = getResource(service, API_STATUS.getPath());
+        WebResource resource  = getResource(service, API_STATUS.getNormalizedPath());
         JSONObject  response  = callAPIWithResource(API_STATUS, resource, null, JSONObject.class);
         try {
             result = response.getString("Status");
@@ -315,7 +316,7 @@ public abstract class AtlasBaseClient {
         int i = 0;
         do {
             if (LOG.isDebugEnabled()) {
-                LOG.debug("Calling API [ {} : {} ] {}", api.getMethod(), api.getPath(), requestObject != null ? "<== " + requestObject : "");
+                LOG.debug("Calling API [ {} : {} ] {}", api.getMethod(), api.getNormalizedPath(), requestObject != null ? "<== " + requestObject : "");
             }
 
             WebResource.Builder requestBuilder = resource.getRequestBuilder();
@@ -375,7 +376,7 @@ public abstract class AtlasBaseClient {
     }
 
     protected WebResource getResource(API api, MultivaluedMap<String, String> queryParams, String... pathParams) {
-        WebResource resource = service.path(api.getPath());
+        WebResource resource = service.path(api.getNormalizedPath());
         resource = appendPathParams(resource, pathParams);
         resource = appendQueryParams(queryParams, resource);
         return resource;
@@ -447,7 +448,7 @@ public abstract class AtlasBaseClient {
                 if (i == (getNumberOfRetries() - 1)) {
                     throw che;
                 }
-                LOG.warn("Handled exception in calling api {}", api.getPath(), che);
+                LOG.warn("Handled exception in calling api {}", api.getNormalizedPath(), che);
                 LOG.warn("Exception's cause: {}", che.getCause().getClass());
                 handleClientHandlerException(che);
             }
@@ -515,13 +516,13 @@ public abstract class AtlasBaseClient {
 
     // Modify URL to include the path params
     private WebResource getResource(WebResource service, API api, String... pathParams) {
-        WebResource resource = service.path(api.getPath());
+        WebResource resource = service.path(api.getNormalizedPath());
         resource = appendPathParams(resource, pathParams);
         return resource;
     }
 
     private WebResource getResource(API api, String queryParamKey, List<String> queryParamValues) {
-        WebResource resource = service.path(api.getPath());
+        WebResource resource = service.path(api.getNormalizedPath());
         for (String queryParamValue : queryParamValues) {
             if (StringUtils.isNotBlank(queryParamKey) && StringUtils.isNotBlank(queryParamValue)) {
                 resource = resource.queryParam(queryParamKey, queryParamValue);
@@ -541,7 +542,7 @@ public abstract class AtlasBaseClient {
 
     // Modify URL to include the query params
     private WebResource getResource(WebResource service, API api, MultivaluedMap<String, String> queryParams) {
-        WebResource resource = service.path(api.getPath());
+        WebResource resource = service.path(api.getNormalizedPath());
         resource = appendQueryParams(queryParams, resource);
         return resource;
     }
@@ -578,6 +579,10 @@ public abstract class AtlasBaseClient {
             return path;
         }
 
+        public String getNormalizedPath() {
+            return Paths.get(path).normalize().toString();
+        }
+
         public Response.Status getExpectedStatus() {
             return status;
         }

client/common/src/main/java/org/apache/atlas/AtlasServiceException.java

@@ -28,7 +28,7 @@ public class AtlasServiceException extends Exception {
     private ClientResponse.Status status;
 
     public AtlasServiceException(AtlasBaseClient.API api, Exception e) {
-        super("Metadata service API " + api.getMethod() + " : " + api.getPath() + " failed", e);
+        super("Metadata service API " + api.getMethod() + " : " + api.getNormalizedPath() + " failed", e);
     }
 
     public AtlasServiceException(AtlasBaseClient.API api, WebApplicationException e) throws JSONException {

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -101,6 +101,9 @@ public final class Constants {
     public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length";
     public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH  = "atlas.graph.index.search.tags.max-query-str-length";
 
+    public static final String MAX_FULLTEXT_QUERY_STR_LENGTH  = "atlas.graph.fulltext-max-query-str-length";
+    public static final String MAX_DSL_QUERY_STR_LENGTH  = "atlas.graph.dsl-max-query-str-length";
+
     private Constants() {
     }
 

intg/src/main/java/org/apache/atlas/AtlasErrorCode.java

@@ -102,6 +102,7 @@ public enum AtlasErrorCode {
     INVALID_ENTITY_FOR_CLASSIFICATION (400, "ATLAS-400-00-055", "Entity (guid=‘{0}‘,typename=‘{1}‘) cannot be classified by Classification ‘{2}‘, because ‘{1}‘ is not in the ClassificationDef's restrictions."),
     SAVED_SEARCH_CHANGE_USER(400, "ATLAS-400-00-056", "saved-search {0} can not be moved from user {1} to {2}"),
     INVALID_QUERY_PARAM_LENGTH(400, "ATLAS-400-00-057" , "Length of query param {0} exceeds the limit"),
+    INVALID_QUERY_LENGTH(400, "ATLAS-400-00-057" , "Invalid query length, update {0} to change the limit" ),
 
     // All Not found enums go here
     TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"),

webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java

@@ -355,7 +355,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
 
                                 if (numRetries == 0) { // audit only on the first attempt
                                     AtlasBaseClient.API api = AtlasClient.API_V1.CREATE_ENTITY;
-                                    audit(messageUser, api.getMethod(), api.getPath());
+                                    audit(messageUser, api.getMethod(), api.getNormalizedPath());
                                 }
 
                                 entities = instanceConverter.toAtlasEntities(createRequest.getEntities());
@@ -369,7 +369,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
                                 if (numRetries == 0) { // audit only on the first attempt
                                     AtlasBaseClient.API api = UPDATE_ENTITY_BY_ATTRIBUTE;
                                     audit(messageUser, api.getMethod(),
-                                          String.format(api.getPath(), partialUpdateRequest.getTypeName()));
+                                          String.format(api.getNormalizedPath(), partialUpdateRequest.getTypeName()));
                                 }
 
                                 Referenceable referenceable = partialUpdateRequest.getEntity();
@@ -394,7 +394,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
                                 if (numRetries == 0) { // audit only on the first attempt
                                     AtlasBaseClient.API api = DELETE_ENTITY_BY_ATTRIBUTE;
                                     audit(messageUser, api.getMethod(),
-                                          String.format(api.getPath(), deleteRequest.getTypeName()));
+                                          String.format(api.getNormalizedPath(), deleteRequest.getTypeName()));
                                 }
 
                                 try {
@@ -413,7 +413,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
 
                                 if (numRetries == 0) { // audit only on the first attempt
                                     AtlasBaseClient.API api = UPDATE_ENTITY;
-                                    audit(messageUser, api.getMethod(), api.getPath());
+                                    audit(messageUser, api.getMethod(), api.getNormalizedPath());
                                 }
 
                                 entities = instanceConverter.toAtlasEntities(updateRequest.getEntities());

webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java

@@ -215,7 +215,7 @@ public class EntityResource {
             UriBuilder ub = uriInfo.getAbsolutePathBuilder();
             locationURI = CollectionUtils.isEmpty(guids) ? null : ub.path(guids.get(0)).build();
         } else {
-            String uriPath = AtlasClient.API_V1.GET_ENTITY.getPath();
+            String uriPath = AtlasClient.API_V1.GET_ENTITY.getNormalizedPath();
             locationURI = guids.isEmpty() ? null : UriBuilder
                 .fromPath(AtlasConstants.DEFAULT_ATLAS_REST_ADDRESS)
                 .path(uriPath).path(guids.get(0)).build();

webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java

@@ -17,7 +17,6 @@
  */
 package org.apache.atlas.web.rest;
 
-import org.apache.atlas.AtlasConfiguration;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.SortOrder;
 import org.apache.atlas.discovery.AtlasDiscoveryService;
@@ -25,9 +24,11 @@ import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.discovery.AtlasSearchResult;
 import org.apache.atlas.model.discovery.SearchParameters;
 import org.apache.atlas.model.profile.AtlasUserSavedSearch;
+import org.apache.atlas.repository.Constants;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.atlas.web.util.Servlets;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.configuration.Configuration;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.springframework.stereotype.Service;
@@ -59,12 +60,16 @@ public class DiscoveryREST {
 
     @Context
     private       HttpServletRequest httpServletRequest;
+    private final int                maxFullTextQueryLength;
+    private final int                maxDslQueryLength;
 
     private final AtlasDiscoveryService atlasDiscoveryService;
 
     @Inject
-    public DiscoveryREST(AtlasDiscoveryService discoveryService) {
-        this.atlasDiscoveryService = discoveryService;
+    public DiscoveryREST(AtlasDiscoveryService atlasDiscoveryService, Configuration configuration) {
+        this.atlasDiscoveryService = atlasDiscoveryService;
+        maxFullTextQueryLength = configuration.getInt(Constants.MAX_FULLTEXT_QUERY_STR_LENGTH, 4096);
+        maxDslQueryLength = configuration.getInt(Constants.MAX_DSL_QUERY_STR_LENGTH, 4096);
     }
 
     /**
@@ -90,10 +95,13 @@ public class DiscoveryREST {
                                             @QueryParam("classification") String classification,
                                             @QueryParam("limit")          int    limit,
                                             @QueryParam("offset")         int    offset) throws AtlasBaseException {
-        Servlets.validateQueryParamLength("query", query);
         Servlets.validateQueryParamLength("typeName", typeName);
         Servlets.validateQueryParamLength("classification", classification);
 
+        if (StringUtils.isNotEmpty(query) && query.length() > maxDslQueryLength) {
+            throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_DSL_QUERY_STR_LENGTH);
+        }
+
         AtlasPerfTracer perf = null;
 
         try {
@@ -132,7 +140,10 @@ public class DiscoveryREST {
                                                  @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities,
                                                  @QueryParam("limit")                  int     limit,
                                                  @QueryParam("offset")                 int     offset) throws AtlasBaseException {
-        Servlets.validateQueryParamLength("query", query);
+        // Validate FullText query for max allowed length
+        if(StringUtils.isNotEmpty(query) && query.length() > maxFullTextQueryLength){
+            throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_FULLTEXT_QUERY_STR_LENGTH );
+        }
 
         AtlasPerfTracer perf = null;
 
@@ -172,9 +183,11 @@ public class DiscoveryREST {
                                               @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities,
                                               @QueryParam("limit")                  int     limit,
                                               @QueryParam("offset")                 int     offset) throws AtlasBaseException {
-        Servlets.validateQueryParamLength("query", query);
         Servlets.validateQueryParamLength("typeName", typeName);
         Servlets.validateQueryParamLength("classification", classification);
+        if (StringUtils.isNotEmpty(query) && query.length() > maxFullTextQueryLength) {
+            throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_FULLTEXT_QUERY_STR_LENGTH);
+        }
 
         AtlasPerfTracer perf = null;
 
@@ -556,7 +569,10 @@ public class DiscoveryREST {
         if (parameters != null) {
             Servlets.validateQueryParamLength("typeName", parameters.getTypeName());
             Servlets.validateQueryParamLength("classification", parameters.getClassification());
-            Servlets.validateQueryParamLength("query", parameters.getQuery());
+            if (StringUtils.isNotEmpty(parameters.getQuery()) && parameters.getQuery().length() > maxFullTextQueryLength) {
+                throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_FULLTEXT_QUERY_STR_LENGTH);
+            }
+
         }
     }
 }