ATLAS-539 Store for entity audit events (shwethags)

common/src/main/java/org/apache/atlas/service/Service.java

@@ -26,5 +26,5 @@ public interface Service {
 
     void start() throws AtlasException;
 
-    void stop();
+    void stop() throws AtlasException;
 }

common/src/main/java/org/apache/atlas/service/Services.java

@@ -52,7 +52,11 @@ public class Services {
     public void stop() {
         for (Service service : services) {
             LOG.debug("Stopping service {}", service.getClass().getName());
+            try {
                 service.stop();
+            } catch (Throwable e) {
+                LOG.warn("Error stopping service {}", service.getClass().getName(), e);
+            }
         }
     }
 }

pom.xml

@@ -880,6 +880,30 @@
             </dependency>
 
             <dependency>
+                <groupId>org.apache.hbase</groupId>
+                <artifactId>hbase-server</artifactId>
+                <version>${hbase.version}</version>
+                <classifier>tests</classifier>
+                <scope>test</scope>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.hbase</groupId>
+                <artifactId>hbase-server</artifactId>
+                <version>${hbase.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.mortbay.jetty</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>tomcat</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
+            <dependency>
                 <groupId>com.thinkaurelius.titan</groupId>
                 <artifactId>titan-es</artifactId>
                 <version>${titan.version}</version>
@@ -1447,14 +1471,14 @@
                     <systemProperties>
                         <user.dir>${project.basedir}</user.dir>
                         <atlas.data>${project.build.directory}/data</atlas.data>
+                        <log4j.configuration>atlas-log4j.xml</log4j.configuration>
                     </systemProperties>
                     <skipTests>${skipTests}</skipTests>
                     <forkMode>always</forkMode>
                     <redirectTestOutputToFile>true</redirectTestOutputToFile>
                     <argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
                         -Dhadoop.tmp.dir="${project.build.directory}/tmp-hadoop-${user.name}"
-                        -Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=atlas-log4j.xml
-                        -Djava.net.preferIPv4Stack=true
+                        -Xmx1024m -XX:MaxPermSize=512m -Djava.net.preferIPv4Stack=true
                     </argLine>
                     <skip>${skipUTs}</skip>
                     <excludes>
@@ -1473,16 +1497,17 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-failsafe-plugin</artifactId>
-                <version>2.18.1</version>
+                <version>2.19.1</version>
                 <configuration>
                     <systemPropertyVariables>
                         <projectBaseDir>${projectBaseDir}</projectBaseDir>
                         <atlas.data>${project.build.directory}/data</atlas.data>
+                        <log4j.configuration>atlas-log4j.xml</log4j.configuration>
                     </systemPropertyVariables>
                     <redirectTestOutputToFile>true</redirectTestOutputToFile>
                     <argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
                         -Dhadoop.tmp.dir="${project.build.directory}/tmp-hadoop-${user.name}"
-                        -Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=atlas-log4j.xml
+                        -Xmx1024m -XX:MaxPermSize=512m
                     </argLine>
                     <skip>${skipITs}</skip>
                     <parallel>none</parallel>

release-log.txt

@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-539 Store for entity audit events (shwethags)
 ATLAS-523 Support alter view (sumasai via shwethags)
 ATLAS-555 Tag creation from UI fails due to missing description attribute (guptaneeru via shwethags)
 ATLAS-522 Support Alter table commands (sumasai via shwethags)

repository/pom.xml

@@ -138,6 +138,18 @@
             <groupId>org.mockito</groupId>
             <artifactId>mockito-all</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-server</artifactId>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-server</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditRepository.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.audit;
+
+import org.apache.atlas.AtlasException;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.List;
+
+/**
+ * Interface for repository for storing entity audit events
+ */
+public interface EntityAuditRepository {
+    /**
+     * Structure of entity audit event
+     */
+    class EntityAuditEvent {
+        String entityId;
+        Long timestamp;
+        String user;
+        String action;
+        String details;
+
+        public EntityAuditEvent() {
+        }
+
+        public EntityAuditEvent(String entityId, long ts, String user, String action, String details) {
+            this.entityId = entityId;
+            this.timestamp = ts;
+            this.user = user;
+            this.action = action;
+            this.details = details;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (this == other) {
+                return true;
+            }
+
+            if (!(other instanceof EntityAuditEvent)) {
+                return false;
+            }
+
+            EntityAuditEvent otherEvent = (EntityAuditEvent) other;
+            return StringUtils.equals(entityId, otherEvent.entityId) &&
+                    (timestamp.longValue() == otherEvent.timestamp.longValue()) &&
+                    StringUtils.equals(user, otherEvent.user) && StringUtils.equals(action, otherEvent.action) &&
+                    StringUtils.equals(details, otherEvent.details);
+        }
+
+        @Override
+        public int hashCode() {
+            return toString().hashCode();
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("EntityId=").append(entityId).append(";Timestamp=").append(timestamp).append(";User=")
+                   .append(user).append(";Action=").append(action).append(";Details=").append(details);
+            return builder.toString();
+        }
+    }
+
+    /**
+     * Add events to the event repository
+     * @param events events to be added
+     * @throws AtlasException
+     */
+    void putEvents(EntityAuditEvent... events) throws AtlasException;
+
+    /**
+     * List events for the given entity id in decreasing order of timestamp, from the given timestamp. Returns n results
+     * @param entityId entity id
+     * @param ts starting timestamp for events
+     * @param n number of events to be returned
+     * @return list of events
+     * @throws AtlasException
+     */
+    List<EntityAuditRepository.EntityAuditEvent> listEvents(String entityId, Long ts, short n) throws AtlasException;
+}

repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.audit;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+public class HBaseBasedAuditRepositoryTest {
+    private HBaseTestingUtility testUtility;
+    private HBaseBasedAuditRepository eventRepository;
+    private LocalHBaseCluster hbaseCluster;
+    private TableName tableName;
+
+    @BeforeClass
+    public void setup() throws Exception {
+        testUtility = HBaseTestingUtility.createLocalHTU();
+        testUtility.startMiniZKCluster();
+        testUtility.getConfiguration().set("zookeeper.session.timeout.ms", "1000");
+        hbaseCluster = new LocalHBaseCluster(testUtility.getConfiguration());
+        hbaseCluster.startup();
+
+        eventRepository = new HBaseBasedAuditRepository() {
+            @Override
+            public org.apache.hadoop.conf.Configuration getHBaseConfiguration(Configuration atlasConf)
+                    throws AtlasException {
+                return testUtility.getConfiguration();
+            }
+        };
+        eventRepository.start();
+
+        Configuration properties = ApplicationProperties.get();
+        String tableNameStr = properties.getString(HBaseBasedAuditRepository.CONFIG_TABLE_NAME,
+                HBaseBasedAuditRepository.DEFAULT_TABLE_NAME);
+        tableName = TableName.valueOf(tableNameStr);
+    }
+
+    @AfterClass
+    public void teardown() throws Exception {
+        eventRepository.stop();
+        testUtility.getConnection().close();
+        hbaseCluster.shutdown();
+        testUtility.shutdownMiniZKCluster();
+    }
+
+    private String rand() {
+        return RandomStringUtils.randomAlphanumeric(10);
+    }
+
+    @Test
+    public void testTableCreated() throws Exception {
+        Admin admin = testUtility.getConnection().getAdmin();
+        assertTrue(admin.tableExists(tableName));
+    }
+
+    @Test
+    public void testAddEvents() throws Exception {
+        EntityAuditRepository.EntityAuditEvent event =
+                new EntityAuditRepository.EntityAuditEvent(rand(), System.currentTimeMillis(), "u1", "a1", "d1");
+
+        eventRepository.putEvents(event);
+
+        List<EntityAuditRepository.EntityAuditEvent> events =
+                eventRepository.listEvents(event.entityId, System.currentTimeMillis(), (short) 10);
+        assertEquals(events.size(), 1);
+        assertEquals(events.get(0), event);
+    }
+
+    @Test
+    public void testListPagination() throws Exception {
+        String id1 = "id1" + rand();
+        String id2 = "id2" + rand();
+        String id3 = "id3" + rand();
+        long ts = System.nanoTime();
+        List<EntityAuditRepository.EntityAuditEvent> expectedEvents = new ArrayList<>(3);
+        for (int i = 0; i < 3; i++) {
+            //Add events for both ids
+            EntityAuditRepository.EntityAuditEvent event =
+                    new EntityAuditRepository.EntityAuditEvent(id2, ts - i, "user" + i, "action" + i, "details" + i);
+            eventRepository.putEvents(event);
+            expectedEvents.add(event);
+            eventRepository.putEvents(new EntityAuditRepository.EntityAuditEvent(id1, ts - i, "user" + i,
+                    "action" + i, "details" + i));
+            eventRepository.putEvents(new EntityAuditRepository.EntityAuditEvent(id3, ts - i, "user" + i,
+                    "action" + i, "details" + i));
+        }
+
+        //Use ts for which there is no event - ts + 2
+        List<EntityAuditRepository.EntityAuditEvent> events = eventRepository.listEvents(id2, ts + 2, (short) 2);
+        assertEquals(events.size(), 2);
+        assertEquals(events.get(0), expectedEvents.get(0));
+        assertEquals(events.get(1), expectedEvents.get(1));
+
+        //Use last event's timestamp for next list(). Should give only 1 event and shouldn't include events from other id
+        events = eventRepository.listEvents(id2, events.get(1).timestamp - 1, (short) 3);
+        assertEquals(events.size(), 1);
+        assertEquals(events.get(0), expectedEvents.get(2));
+    }
+}

typesystem/src/main/resources/atlas-application.properties

@@ -78,3 +78,5 @@ atlas.enableTLS=false
 atlas.server.https.port=31443
 
 #########  Security Properties  #########
+
+hbase.security.authentication=simple

webapp/pom.xml

@@ -353,7 +353,11 @@
                     <systemProperties>
                         <systemProperty>
                             <name>log4j.configuration</name>
-                            <value>atlas-log4j.xml</value>
+                            <value>file://${project.build.directory}/../../distro/src/conf/atlas-log4j.xml</value>
+                        </systemProperty>
+                        <systemProperty>
+                            <name>atlas.log.file</name>
+                            <value>application.log</value>
                         </systemProperty>
                         <systemProperty>
                             <name>atlas.log.dir</name>

webapp/src/main/resources/atlas-log4j.xml

-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-    <appender name="console" class="org.apache.log4j.ConsoleAppender">
-        <param name="Target" value="System.out"/>
-        <layout class="org.apache.log4j.PatternLayout">
-            <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
-        </layout>
-    </appender>
-
-    <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
-        <param name="File" value="${atlas.log.dir}/application.log"/>
-        <param name="Append" value="true"/>
-        <param name="Threshold" value="debug"/>
-        <layout class="org.apache.log4j.PatternLayout">
-            <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
-        </layout>
-    </appender>
-
-    <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
-        <param name="File" value="${atlas.log.dir}/audit.log"/>
-        <param name="Append" value="true"/>
-        <param name="Threshold" value="debug"/>
-        <layout class="org.apache.log4j.PatternLayout">
-            <param name="ConversionPattern" value="%d %x %m%n"/>
-        </layout>
-    </appender>
-
-    <logger name="org.apache.atlas" additivity="false">
-        <level value="debug"/>
-        <appender-ref ref="FILE"/>
-    </logger>
-
-    <logger name="com.thinkaurelius.titan" additivity="false">
-        <level value="info"/>
-        <appender-ref ref="FILE"/>
-    </logger>
-
-    <logger name="AUDIT">
-        <level value="info"/>
-        <appender-ref ref="AUDIT"/>
-    </logger>
-
-    <root>
-        <priority value="warn"/>
-        <appender-ref ref="FILE"/>
-    </root>
-
-</log4j:configuration>

webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerTest.java

@@ -48,7 +48,7 @@ public class SecureEmbeddedServerTest extends SecureEmbeddedServerTestBase {
             };
             secureEmbeddedServer.server.start();
 
-            URL url = new URL("https://localhost:21443/");
+            URL url = new URL("https://localhost:21443/api/atlas/admin/version");
             HttpURLConnection connection = (HttpURLConnection) url.openConnection();
             connection.setRequestMethod("GET");
             connection.connect();