ATLAS-1391 Add exclusion mechanism for Atlas audit

release-log.txt

@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1391 Add exclusion mechanism for Atlas audit
 ATLAS-1407 improve LOG statement performance (apoorvnaik via mneethiraj)
 ATLAS-1350 update authorization to handle v2 REST endpoints (saqeeb.s via mneethiraj)
 ATLAS-1311 Integration tests for V2 Entity APIs (apoorvnaik via mneethiraj)

repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java

@@ -17,6 +17,9 @@
  */
 package org.apache.atlas.util;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.repository.audit.EntityAuditRepository;
@@ -35,14 +38,16 @@ import org.slf4j.LoggerFactory;
  *
  */
 public class AtlasRepositoryConfiguration {
-    
+
     private static Logger LOG = LoggerFactory.getLogger(AtlasRepositoryConfiguration.class);
-  
+
     public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
+    public static final String AUDIT_EXCLUDED_OPERATIONS = "atlas.audit.excludes";
+    private static List<String> skippedOperations = null;
+    public static final String SEPARATOR = ":";
 
     @SuppressWarnings("unchecked")
     public static Class<? extends TypeCache> getTypeCache() {
-
         // Get the type cache implementation class from Atlas configuration.
         try {
             Configuration config = ApplicationProperties.get();
@@ -92,6 +97,48 @@ public class AtlasRepositoryConfiguration {
             throw new RuntimeException(e);
         }
     }
-   
+
+    /**
+     * Get the list of operations which are configured to be skipped from auditing
+     * Valid format is HttpMethod:URL eg: GET:Version
+     * @return list of string
+     */
+    public static List<String> getAuditExcludedOperations(Configuration config) {
+        if (config == null) {
+            try {
+                config = ApplicationProperties.get();
+            } catch (AtlasException e) {
+                LOG.error(" Error reading operations for auditing ", e);
+            }
+        }
+        if (skippedOperations == null) {
+            skippedOperations = new ArrayList<String>();
+                String[] skipAuditForOperations = config
+                        .getStringArray(AUDIT_EXCLUDED_OPERATIONS);
+                if (skipAuditForOperations != null
+                        && skipAuditForOperations.length > 0) {
+                    for (String skippedOperation : skipAuditForOperations) {
+                        String[] excludedOperations = skippedOperation.trim().toLowerCase().split(SEPARATOR);
+                        if (excludedOperations!= null && excludedOperations.length == 2) {
+                            skippedOperations.add(skippedOperation.toLowerCase());
+                        } else {
+                            LOG.error("Invalid format for skipped operation {}. Valid format is HttpMethod:URL eg: GET:Version", skippedOperation);
+                        }
+                    }
+                }
+        }
+        return skippedOperations;
+    }
+
+    public static boolean isExcludedFromAudit(Configuration config, String httpMethod, String httpUrl) {
+        if (getAuditExcludedOperations(config).size() > 0) {
+            return getAuditExcludedOperations(config).contains(httpMethod.toLowerCase() + SEPARATOR + httpUrl.toLowerCase());
+        } else {
+            return false;
+        }
+    }
+   public static void resetExcludedOperations() { //for test purpose only
+        skippedOperations = null;
+    }
 
 }

webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java

@@ -22,6 +22,8 @@ import com.google.inject.Singleton;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.RequestContext;
 import org.apache.atlas.metrics.Metrics;
+import org.apache.atlas.util.AtlasRepositoryConfiguration;
+import org.apache.commons.configuration.Configuration;
 import org.apache.atlas.web.util.DateTimeHelper;
 import org.apache.atlas.web.util.Servlets;
 import org.slf4j.Logger;
@@ -35,8 +37,10 @@ import javax.servlet.ServletRequest;
 import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+
 import java.io.IOException;
 import java.util.Date;
+import java.util.List;
 import java.util.UUID;
 
 /**
@@ -45,7 +49,6 @@ import java.util.UUID;
  */
 @Singleton
 public class AuditFilter implements Filter {
-
     private static final Logger AUDIT_LOG = LoggerFactory.getLogger("AUDIT");
     private static final Logger LOG = LoggerFactory.getLogger(AuditFilter.class);
     private static final Logger METRICS_LOG = LoggerFactory.getLogger("METRICS");
@@ -91,7 +94,15 @@ public class AuditFilter implements Filter {
         final String whatURL = Servlets.getRequestURL(httpRequest);
         final String whatAddrs = httpRequest.getLocalAddr();
 
-        audit(who, fromAddress, whatRequest, fromHost, whatURL, whatAddrs, whenISO9601);
+        final String whatUrlPath = httpRequest.getRequestURL().toString();//url path without query string
+
+        if (!isOperationExcludedFromAudit(whatRequest, whatUrlPath.toLowerCase(), null)) {
+            audit(who, fromAddress, whatRequest, fromHost, whatURL, whatAddrs, whenISO9601);
+        } else {
+            if(LOG.isDebugEnabled()) {
+                LOG.debug(" Skipping Audit for {} ", whatURL);
+            }
+        }
     }
 
     private String getUserFromRequest(HttpServletRequest httpRequest) {
@@ -114,6 +125,10 @@ public class AuditFilter implements Filter {
         }
      }
 
+    boolean isOperationExcludedFromAudit(String requestHttpMethod, String requestOperation, Configuration config) {
+       return AtlasRepositoryConfiguration.isExcludedFromAudit(config, requestHttpMethod, requestOperation);
+    }
+
     @Override
     public void destroy() {
         // do nothing

webapp/src/test/java/org/apache/atlas/web/filters/AuditFilterTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.web.filters;
+
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.atlas.util.AtlasRepositoryConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+/**
+ * This is the test class to test Audit filter functionality
+ *
+ */
+public class AuditFilterTest { 
+
+    public static final String ACTIVE_SERVER_ADDRESS = "http://localhost:21000/";
+    @Mock
+    private HttpServletRequest servletRequest;
+
+    @Mock
+    private HttpServletResponse servletResponse;
+
+    @Mock
+    private FilterChain filterChain;
+
+    @Mock
+    private Configuration configuration;
+
+    @BeforeMethod
+    public void setUp() {
+        MockitoAnnotations.initMocks(this);
+    }
+
+    @Test
+    public void testVerifyExcludedOperations() {
+        AtlasRepositoryConfiguration.resetExcludedOperations();
+        when(configuration.getStringArray(AtlasRepositoryConfiguration.AUDIT_EXCLUDED_OPERATIONS)).thenReturn(new String[]{"GET:Version", "GET:Ping"});
+        AuditFilter auditFilter = new AuditFilter();
+        assertTrue(auditFilter.isOperationExcludedFromAudit("GET", "Version", configuration));
+        assertTrue(auditFilter.isOperationExcludedFromAudit("get", "Version", configuration));
+        assertTrue(auditFilter.isOperationExcludedFromAudit("GET", "Ping", configuration));
+        assertFalse(auditFilter.isOperationExcludedFromAudit("GET", "Types", configuration));
+    }
+
+    @Test
+    public void testVerifyNotExcludedOperations() {
+        AtlasRepositoryConfiguration.resetExcludedOperations();
+        when(configuration.getStringArray(AtlasRepositoryConfiguration.AUDIT_EXCLUDED_OPERATIONS)).thenReturn(new String[]{"Version", "Ping"});
+        AuditFilter auditFilter = new AuditFilter();
+        assertFalse(auditFilter.isOperationExcludedFromAudit("GET", "Version", configuration));
+        assertFalse(auditFilter.isOperationExcludedFromAudit("GET", "Ping", configuration));
+        assertFalse(auditFilter.isOperationExcludedFromAudit("GET", "Types", configuration));
+    }
+
+    @Test
+    public void testAudit() throws IOException, ServletException {
+        AtlasRepositoryConfiguration.resetExcludedOperations();
+        when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("api/atlas/types"));
+        when(servletRequest.getMethod()).thenReturn("GET");
+        AuditFilter auditFilter = new AuditFilter();
+        auditFilter.doFilter(servletRequest, servletResponse, filterChain);
+        verify(filterChain).doFilter(servletRequest, servletResponse);
+
+        assertFalse(auditFilter.isOperationExcludedFromAudit("GET", "Version", configuration));
+        assertFalse(auditFilter.isOperationExcludedFromAudit("GET", "Ping", configuration));
+        assertFalse(auditFilter.isOperationExcludedFromAudit("GET", "Types", configuration));
+    }
+
+    @Test
+    public void testAuditWithExcludedOperation() throws IOException, ServletException {
+        AtlasRepositoryConfiguration.resetExcludedOperations();
+        when(configuration.getStringArray(AtlasRepositoryConfiguration.AUDIT_EXCLUDED_OPERATIONS)).thenReturn(new String[]{"GET:Version", "GET:Ping"});
+        when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("api/atlas/version"));
+        when(servletRequest.getMethod()).thenReturn("GET");
+        AuditFilter auditFilter = new AuditFilter();
+        auditFilter.doFilter(servletRequest, servletResponse, filterChain);
+        verify(filterChain).doFilter(servletRequest, servletResponse);
+    }
+
+    @Test
+    public void testAuditWithExcludedOperationInIncorrectFormat() throws IOException, ServletException {
+        AtlasRepositoryConfiguration.resetExcludedOperations();
+        when(configuration.getStringArray(AtlasRepositoryConfiguration.AUDIT_EXCLUDED_OPERATIONS)).thenReturn(new String[]{"Version", "Ping"});
+        when(servletRequest.getRequestURL()).thenReturn(new StringBuffer("api/atlas/version"));
+        when(servletRequest.getMethod()).thenReturn("GET");
+        AuditFilter auditFilter = new AuditFilter();
+        auditFilter.doFilter(servletRequest, servletResponse, filterChain);
+        verify(filterChain).doFilter(servletRequest, servletResponse);
+    }
+
+}