ATLAS-871 Make audit repository implementation configurable (jnhagelb via shwethags)

distro/pom.xml

@@ -52,6 +52,7 @@ atlas.graph.index.search.solr.zookeeper-url=
         </titan.index.properties>
         <hbase.embedded>false</hbase.embedded>
         <solr.embedded>false</solr.embedded>
+	<entity.repository.properties>#atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository</entity.repository.properties>
     </properties>
 
     <profiles>
@@ -130,6 +131,7 @@ atlas.graph.index.search.elasticsearch.client-only=false
 atlas.graph.index.search.elasticsearch.local-mode=true
 atlas.graph.index.search.elasticsearch.create.sleep=2000
                 </titan.index.properties>
+		<entity.repository.properties>atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.NoopEntityAuditRepository</entity.repository.properties>
             </properties>
         </profile>
 

distro/src/bin/atlas_start.py

@@ -72,32 +72,36 @@ def main():
     web_app_dir = mc.webAppDir(atlas_home)
     mc.expandWebApp(atlas_home)
 
-    #add hbase-site.xml to classpath
-    hbase_conf_dir = mc.hbaseConfDir(atlas_home)
-
     p = os.pathsep
     atlas_classpath = confdir + p \
                        + os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \
                        + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "atlas-titan-${project.version}.jar" ) + p \
                        + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "*" )  + p \
                        + os.path.join(atlas_home, "libext", "*")
-    if os.path.exists(hbase_conf_dir):
-        atlas_classpath = atlas_classpath + p \
+
+    is_hbase = mc.is_hbase(confdir)
+
+    if is_hbase:
+        #add hbase-site.xml to classpath
+        hbase_conf_dir = mc.hbaseConfDir(atlas_home)
+
+        if os.path.exists(hbase_conf_dir):
+            atlas_classpath = atlas_classpath + p \
                             + hbase_conf_dir
-    else: 
-       if mc.is_hbase(confdir):
-           raise Exception("Could not find hbase-site.xml in %s. Please set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir)
+        else:
+            if mc.is_hbase(confdir):
+                raise Exception("Could not find hbase-site.xml in %s. Please set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir)
 
     if mc.isCygwin():
         atlas_classpath = mc.convertCygwinPath(atlas_classpath, True)
 
     atlas_pid_file = mc.pidFile(atlas_home)
-            
+
     if os.path.isfile(atlas_pid_file):
        #Check if process listed in atlas.pid file is still running
        pf = file(atlas_pid_file, 'r')
        pid = pf.read().strip()
-       pf.close() 
+       pf.close()
 
        if mc.exist_pid((int)(pid)):
            if is_setup:
@@ -106,7 +110,7 @@ def main():
        else:
            mc.server_pid_not_running(pid)
 
-    if mc.is_hbase_local(confdir):
+    if is_hbase and mc.is_hbase_local(confdir):
         print "configured for local hbase."
         mc.configure_hbase(atlas_home)
         mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir)

distro/src/conf/atlas-application.properties

@@ -22,6 +22,27 @@ atlas.graph.storage.backend=${titan.storage.backend}
 
 ${titan.storage.properties}
 
+# Delete handler
+#
+# This allows the default behavior of doing "soft" deletes to be changed.
+#
+# Allowed Values:
+# org.apache.atlas.repository.graph.SoftDeleteHandler - all deletes are "soft" deletes
+# org.apache.atlas.repository.graph.HardDeleteHandler - all deletes are "hard" deletes
+#
+#atlas.DeleteHandler.impl=org.apache.atlas.repository.graph.SoftDeleteHandler
+
+# Entity audit repository
+#
+# This allows the default behavior of logging entity changes to hbase to be changed.
+#
+# Allowed Values:
+# org.apache.atlas.repository.audit.HBaseBasedAuditRepository - log entity changes to hbase
+# org.apache.atlas.repository.audit.NoopEntityAuditRepository - disable the audit repository
+#
+${entity.repository.properties}
+
+
 # Graph Search Index
 atlas.graph.index.search.backend=${titan.index.backend}
 

pom.xml

@@ -411,6 +411,7 @@
 
         <titan.storage.backend>berkeleyje</titan.storage.backend>
         <titan.index.backend>elasticsearch</titan.index.backend>
+	    <entity.repository.impl>org.apache.atlas.repository.audit.InMemoryEntityAuditRepository</entity.repository.impl>
     </properties>
 
     <profiles>
@@ -425,6 +426,7 @@
                 <titan.index.backend>solr5</titan.index.backend>
                 <solr.zk.address>localhost:9983</solr.zk.address>
                 <titan.storage.hostname>localhost</titan.storage.hostname>
+		        <entity.repository.impl>org.apache.atlas.repository.audit.HBaseBasedAuditRepository</entity.repository.impl>
             </properties>
         </profile>
 

release-log.txt

@@ -22,6 +22,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-871 Make audit repository implementation configurable (jnhagelb via shwethags)
 ATLAS-885 optimize HBaseStoreManager to avoid expensive HTable instantiation every 5 seconds (madhan.neethiraj via yhemanth)
 ATLAS-878 UI: Not showing details of SD, DB and COLUMNS (saqeeb.s via shwethags)
 ATLAS-853 User's name to be mentioned in the top user drop down (saqeeb.s via shwethags)

repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java

@@ -87,7 +87,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
 
         bindAuditRepository(binder());
 
-        bind(DeleteHandler.class).to(getDeleteHandler()).asEagerSingleton();
+        bind(DeleteHandler.class).to(getDeleteHandlerImpl()).asEagerSingleton();
 
         //Add EntityAuditListener as EntityChangeListener
         Multibinder<EntityChangeListener> entityChangeListenerBinder =
@@ -100,17 +100,35 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
     }
 
     protected void bindAuditRepository(Binder binder) {
-        //Map EntityAuditRepository interface to hbase based implementation
-        binder.bind(EntityAuditRepository.class).to(HBaseBasedAuditRepository.class).asEagerSingleton();
 
-        //Add HBaseBasedAuditRepository to service so that connection is closed at shutdown
-        Multibinder<Service> serviceBinder = Multibinder.newSetBinder(binder, Service.class);
-        serviceBinder.addBinding().to(HBaseBasedAuditRepository.class);
+        Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl();
+
+        //Map EntityAuditRepository interface to configured implementation
+        binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton();
+
+        if(Service.class.isAssignableFrom(auditRepoImpl)) {
+            Class<? extends Service> auditRepoService = (Class<? extends Service>)auditRepoImpl;
+            //if it's a service, make sure that it gets properly closed at shutdown
+            Multibinder<Service> serviceBinder = Multibinder.newSetBinder(binder, Service.class);
+            serviceBinder.addBinding().to(auditRepoService);
+        }
+    }
+
+
+    private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl";
+
+    private Class<? extends EntityAuditRepository> getAuditRepositoryImpl() {
+        try {
+            return ApplicationProperties.getClass(AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY,
+                    HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
+        } catch (AtlasException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
 
-    private Class<? extends DeleteHandler> getDeleteHandler() {
+    private Class<? extends DeleteHandler> getDeleteHandlerImpl() {
         try {
             return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY,
                     SoftDeleteHandler.class.getName(), DeleteHandler.class);

repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java

@@ -21,6 +21,8 @@ package org.apache.atlas.repository.audit;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.EntityAuditEvent;
 
+import com.google.inject.Singleton;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -30,6 +32,7 @@ import java.util.TreeMap;
 /**
  * Entity audit repository where audit events are stored in-memory. Used only for integration tests
  */
+@Singleton
 public class InMemoryEntityAuditRepository implements EntityAuditRepository {
     private TreeMap<String, EntityAuditEvent> auditEvents = new TreeMap<>();
 

repository/src/main/java/org/apache/atlas/repository/audit/NoopEntityAuditRepository.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.audit;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.EntityAuditEvent;
+
+import com.google.inject.Singleton;
+
+/**
+ * Implementation that completely disables the audit repository.
+ */
+@Singleton
+public class NoopEntityAuditRepository implements EntityAuditRepository {
+
+    @Override
+    public void putEvents(EntityAuditEvent... events) throws AtlasException {
+        //do nothing
+    }
+
+    @Override
+    public synchronized void putEvents(List<EntityAuditEvent> events) throws AtlasException {
+        //do nothing
+    }
+
+    @Override
+    public List<EntityAuditEvent> listEvents(String entityId, String startKey, short maxResults)
+            throws AtlasException {
+        return Collections.emptyList();
+    }
+}

typesystem/src/main/resources/atlas-application.properties

@@ -23,6 +23,9 @@ atlas.rest.address=http://localhost:31000
 # Graph Storage
 atlas.graph.storage.backend=${titan.storage.backend}
 
+# Entity repository implementation
+atlas.EntityAuditRepository.impl=${entity.repository.impl}
+
 # Graph Search Index Backend
 atlas.graph.index.search.backend=${titan.index.backend}