Skip to content

A
atlas
Commit ad0d764d by Shwetha GS
Options

ATLAS-539 Store for entity audit events (shwethags)

parent ad7604fc
Showing with 527 additions and 76 deletions
common/src/main/java/org/apache/atlas/service/Service.java
......@@ -26,5 +26,5 @@ public interface Service {
void start() throws AtlasException;
void stop();
void stop() throws AtlasException;
}
common/src/main/java/org/apache/atlas/service/Services.java
......@@ -52,7 +52,11 @@ public class Services {
public void stop() {
for (Service service : services) {
LOG.debug("Stopping service {}", service.getClass().getName());
service.stop();
try {
service.stop();
} catch (Throwable e) {
LOG.warn("Error stopping service {}", service.getClass().getName(), e);
}
}
}
}
pom.xml
......@@ -880,6 +880,30 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-es</artifactId>
<version>${titan.version}</version>
......@@ -1447,14 +1471,14 @@
<systemProperties>
<user.dir>${project.basedir}</user.dir>
<atlas.data>${project.build.directory}/data</atlas.data>
<log4j.configuration>atlas-log4j.xml</log4j.configuration>
</systemProperties>
<skipTests>${skipTests}</skipTests>
<forkMode>always</forkMode>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
-Dhadoop.tmp.dir="${project.build.directory}/tmp-hadoop-${user.name}"
-Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=atlas-log4j.xml
-Djava.net.preferIPv4Stack=true
-Xmx1024m -XX:MaxPermSize=512m -Djava.net.preferIPv4Stack=true
</argLine>
<skip>${skipUTs}</skip>
<excludes>
......@@ -1473,16 +1497,17 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.18.1</version>
<version>2.19.1</version>
<configuration>
<systemPropertyVariables>
<projectBaseDir>${projectBaseDir}</projectBaseDir>
<atlas.data>${project.build.directory}/data</atlas.data>
<log4j.configuration>atlas-log4j.xml</log4j.configuration>
</systemPropertyVariables>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
-Dhadoop.tmp.dir="${project.build.directory}/tmp-hadoop-${user.name}"
-Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=atlas-log4j.xml
-Xmx1024m -XX:MaxPermSize=512m
</argLine>
<skip>${skipITs}</skip>
<parallel>none</parallel>
......
release-log.txt
......@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-539 Store for entity audit events (shwethags)
ATLAS-523 Support alter view (sumasai via shwethags)
ATLAS-555 Tag creation from UI fails due to missing description attribute (guptaneeru via shwethags)
ATLAS-522 Support Alter table commands (sumasai via shwethags)
......
repository/pom.xml
......@@ -138,6 +138,18 @@
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
</dependency>
</dependencies>
<build>
......
repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditRepository.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.audit;
import org.apache.atlas.AtlasException;
import org.apache.commons.lang.StringUtils;
import java.util.List;
/**
* Interface for repository for storing entity audit events
*/
public interface EntityAuditRepository {
/**
* Structure of entity audit event
*/
class EntityAuditEvent {
String entityId;
Long timestamp;
String user;
String action;
String details;
public EntityAuditEvent() {
}
public EntityAuditEvent(String entityId, long ts, String user, String action, String details) {
this.entityId = entityId;
this.timestamp = ts;
this.user = user;
this.action = action;
this.details = details;
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof EntityAuditEvent)) {
return false;
}
EntityAuditEvent otherEvent = (EntityAuditEvent) other;
return StringUtils.equals(entityId, otherEvent.entityId) &&
(timestamp.longValue() == otherEvent.timestamp.longValue()) &&
StringUtils.equals(user, otherEvent.user) && StringUtils.equals(action, otherEvent.action) &&
StringUtils.equals(details, otherEvent.details);
}
@Override
public int hashCode() {
return toString().hashCode();
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("EntityId=").append(entityId).append(";Timestamp=").append(timestamp).append(";User=")
.append(user).append(";Action=").append(action).append(";Details=").append(details);
return builder.toString();
}
}
/**
* Add events to the event repository
* @param events events to be added
* @throws AtlasException
*/
void putEvents(EntityAuditEvent... events) throws AtlasException;
/**
* List events for the given entity id in decreasing order of timestamp, from the given timestamp. Returns n results
* @param entityId entity id
* @param ts starting timestamp for events
* @param n number of events to be returned
* @return list of events
* @throws AtlasException
*/
List<EntityAuditRepository.EntityAuditEvent> listEvents(String entityId, Long ts, short n) throws AtlasException;
}
repository/src/test/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepositoryTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.audit;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.LocalHBaseCluster;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.List;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
public class HBaseBasedAuditRepositoryTest {
private HBaseTestingUtility testUtility;
private HBaseBasedAuditRepository eventRepository;
private LocalHBaseCluster hbaseCluster;
private TableName tableName;
@BeforeClass
public void setup() throws Exception {
testUtility = HBaseTestingUtility.createLocalHTU();
testUtility.startMiniZKCluster();
testUtility.getConfiguration().set("zookeeper.session.timeout.ms", "1000");
hbaseCluster = new LocalHBaseCluster(testUtility.getConfiguration());
hbaseCluster.startup();
eventRepository = new HBaseBasedAuditRepository() {
@Override
public org.apache.hadoop.conf.Configuration getHBaseConfiguration(Configuration atlasConf)
throws AtlasException {
return testUtility.getConfiguration();
}
};
eventRepository.start();
Configuration properties = ApplicationProperties.get();
String tableNameStr = properties.getString(HBaseBasedAuditRepository.CONFIG_TABLE_NAME,
HBaseBasedAuditRepository.DEFAULT_TABLE_NAME);
tableName = TableName.valueOf(tableNameStr);
}
@AfterClass
public void teardown() throws Exception {
eventRepository.stop();
testUtility.getConnection().close();
hbaseCluster.shutdown();
testUtility.shutdownMiniZKCluster();
}
private String rand() {
return RandomStringUtils.randomAlphanumeric(10);
}
@Test
public void testTableCreated() throws Exception {
Admin admin = testUtility.getConnection().getAdmin();
assertTrue(admin.tableExists(tableName));
}
@Test
public void testAddEvents() throws Exception {
EntityAuditRepository.EntityAuditEvent event =
new EntityAuditRepository.EntityAuditEvent(rand(), System.currentTimeMillis(), "u1", "a1", "d1");
eventRepository.putEvents(event);
List<EntityAuditRepository.EntityAuditEvent> events =
eventRepository.listEvents(event.entityId, System.currentTimeMillis(), (short) 10);
assertEquals(events.size(), 1);
assertEquals(events.get(0), event);
}
@Test
public void testListPagination() throws Exception {
String id1 = "id1" + rand();
String id2 = "id2" + rand();
String id3 = "id3" + rand();
long ts = System.nanoTime();
List<EntityAuditRepository.EntityAuditEvent> expectedEvents = new ArrayList<>(3);
for (int i = 0; i < 3; i++) {
//Add events for both ids
EntityAuditRepository.EntityAuditEvent event =
new EntityAuditRepository.EntityAuditEvent(id2, ts - i, "user" + i, "action" + i, "details" + i);
eventRepository.putEvents(event);
expectedEvents.add(event);
eventRepository.putEvents(new EntityAuditRepository.EntityAuditEvent(id1, ts - i, "user" + i,
"action" + i, "details" + i));
eventRepository.putEvents(new EntityAuditRepository.EntityAuditEvent(id3, ts - i, "user" + i,
"action" + i, "details" + i));
}
//Use ts for which there is no event - ts + 2
List<EntityAuditRepository.EntityAuditEvent> events = eventRepository.listEvents(id2, ts + 2, (short) 2);
assertEquals(events.size(), 2);
assertEquals(events.get(0), expectedEvents.get(0));
assertEquals(events.get(1), expectedEvents.get(1));
//Use last event's timestamp for next list(). Should give only 1 event and shouldn't include events from other id
events = eventRepository.listEvents(id2, events.get(1).timestamp - 1, (short) 3);
assertEquals(events.size(), 1);
assertEquals(events.get(0), expectedEvents.get(2));
}
}
typesystem/src/main/resources/atlas-application.properties
......@@ -78,3 +78,5 @@ atlas.enableTLS=false
atlas.server.https.port=31443
######### Security Properties #########
hbase.security.authentication=simple
webapp/pom.xml
......@@ -353,7 +353,11 @@
<systemProperties>
<systemProperty>
<name>log4j.configuration</name>
<value>atlas-log4j.xml</value>
<value>file://${project.build.directory}/../../distro/src/conf/atlas-log4j.xml</value>
</systemProperty>
<systemProperty>
<name>atlas.log.file</name>
<value>application.log</value>
</systemProperty>
<systemProperty>
<name>atlas.log.dir</name>
......
webapp/src/main/resources/atlas-log4j.xml deleted 100755 → 0
<?xml version="1.0" encoding="UTF-8" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="console" class="org.apache.log4j.ConsoleAppender">
<param name="Target" value="System.out"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
</layout>
</appender>
<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${atlas.log.dir}/application.log"/>
<param name="Append" value="true"/>
<param name="Threshold" value="debug"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
</layout>
</appender>
<appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="${atlas.log.dir}/audit.log"/>
<param name="Append" value="true"/>
<param name="Threshold" value="debug"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %x %m%n"/>
</layout>
</appender>
<logger name="org.apache.atlas" additivity="false">
<level value="debug"/>
<appender-ref ref="FILE"/>
</logger>
<logger name="com.thinkaurelius.titan" additivity="false">
<level value="info"/>
<appender-ref ref="FILE"/>
</logger>
<logger name="AUDIT">
<level value="info"/>
<appender-ref ref="AUDIT"/>
</logger>
<root>
<priority value="warn"/>
<appender-ref ref="FILE"/>
</root>
</log4j:configuration>
webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerTest.java
......@@ -48,7 +48,7 @@ public class SecureEmbeddedServerTest extends SecureEmbeddedServerTestBase {
};
secureEmbeddedServer.server.start();
URL url = new URL("https://localhost:21443/");
URL url = new URL("https://localhost:21443/api/atlas/admin/version");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment