ATLAS-1463: option to exclude specific entity attributes in audit records

repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditRepository.java

@@ -50,4 +50,18 @@ public interface EntityAuditRepository {
      * @throws AtlasException
      */
     List<EntityAuditEvent> listEvents(String entityId, String startKey, short n) throws AtlasException;
+
+    /**
+     * Returns maximum allowed repository size per EntityAuditEvent
+     * @throws AtlasException
+     */
+    long repositoryMaxSize() throws AtlasException;
+
+    /**
+     * list of attributes to be excluded when storing in audit repo.
+     * @param entityType type of entity
+     * @return list of attribute names to be excluded
+     * @throws AtlasException
+     */
+    List<String> getAuditExcludeAttributes(String entityType) throws AtlasException;
 }

repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java

@@ -52,8 +52,10 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
 /**
  * HBase based repository for entity audit events
@@ -74,9 +76,6 @@ public class HBaseBasedAuditRepository implements Service, EntityAuditRepository
     public static final String CONFIG_PREFIX = "atlas.audit";
     public static final String CONFIG_TABLE_NAME = CONFIG_PREFIX + ".hbase.tablename";
     public static final String DEFAULT_TABLE_NAME = "ATLAS_ENTITY_AUDIT_EVENTS";
-
-    private static final String FIELD_SEPARATOR = ":";
-
     public static final String CONFIG_PERSIST_ENTITY_DEFINITION = CONFIG_PREFIX + ".persistEntityDefinition";
 
     public static final byte[] COLUMN_FAMILY = Bytes.toBytes("dt");
@@ -85,8 +84,16 @@ public class HBaseBasedAuditRepository implements Service, EntityAuditRepository
     public static final byte[] COLUMN_USER = Bytes.toBytes("u");
     public static final byte[] COLUMN_DEFINITION = Bytes.toBytes("f");
 
+    private static final String  AUDIT_REPOSITORY_MAX_SIZE_PROPERTY = "atlas.hbase.client.keyvalue.maxsize";
+    private static final String  AUDIT_EXCLUDE_ATTRIBUTE_PROPERTY   = "atlas.audit.hbase.entity";
+    private static final String  FIELD_SEPARATOR = ":";
+    private static final long    ATLAS_HBASE_KEYVALUE_DEFAULT_SIZE = 1024 * 1024;
+    private static Configuration APPLICATION_PROPERTIES = null;
+
     private static boolean       persistEntityDefinition;
 
+    private Map<String, List<String>> auditExcludedAttributesCache = new HashMap<>();
+
     static {
         try {
             persistEntityDefinition = ApplicationProperties.get().getBoolean(CONFIG_PERSIST_ENTITY_DEFINITION, false);
@@ -219,6 +226,52 @@ public class HBaseBasedAuditRepository implements Service, EntityAuditRepository
         }
     }
 
+    @Override
+    public long repositoryMaxSize() throws AtlasException {
+        long ret;
+        initApplicationProperties();
+
+        if (APPLICATION_PROPERTIES == null) {
+            ret = ATLAS_HBASE_KEYVALUE_DEFAULT_SIZE;
+        } else {
+            ret = APPLICATION_PROPERTIES.getLong(AUDIT_REPOSITORY_MAX_SIZE_PROPERTY, ATLAS_HBASE_KEYVALUE_DEFAULT_SIZE);
+        }
+
+        return ret;
+    }
+
+    @Override
+    public List<String> getAuditExcludeAttributes(String entityType) throws AtlasException {
+        List<String> ret = null;
+
+        initApplicationProperties();
+
+        if (auditExcludedAttributesCache.containsKey(entityType)) {
+            ret = auditExcludedAttributesCache.get(entityType);
+        } else if (APPLICATION_PROPERTIES != null) {
+            String[] excludeAttributes = APPLICATION_PROPERTIES.getStringArray(AUDIT_EXCLUDE_ATTRIBUTE_PROPERTY + "." +
+                    entityType + "." +  "attributes.exclude");
+
+            if (excludeAttributes != null) {
+                ret = Arrays.asList(excludeAttributes);
+            }
+
+            auditExcludedAttributesCache.put(entityType, ret);
+        }
+
+        return ret;
+    }
+
+    private void initApplicationProperties() {
+        if (APPLICATION_PROPERTIES == null) {
+            try {
+                APPLICATION_PROPERTIES = ApplicationProperties.get();
+            } catch (AtlasException ex) {
+                // ignore
+            }
+        }
+    }
+
     private String getResultString(Result result, byte[] columnName) {
         byte[] rawValue = result.getValue(COLUMN_FAMILY, columnName);
         if ( rawValue != null) {

repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java

@@ -66,4 +66,14 @@ public class InMemoryEntityAuditRepository implements EntityAuditRepository {
         }
         return events;
     }
+
+    @Override
+    public long repositoryMaxSize() throws AtlasException {
+        return -1;
+    }
+
+    @Override
+    public List<String> getAuditExcludeAttributes(String entityType) throws AtlasException {
+        return null;
+    }
 }

repository/src/main/java/org/apache/atlas/repository/audit/NoopEntityAuditRepository.java

@@ -47,4 +47,14 @@ public class NoopEntityAuditRepository implements EntityAuditRepository {
             throws AtlasException {
         return Collections.emptyList();
     }
+
+    @Override
+    public long repositoryMaxSize() throws AtlasException {
+        return -1;
+    }
+
+    @Override
+    public List<String> getAuditExcludeAttributes(String entityType) throws AtlasException {
+        return null;
+    }
 }

typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java

@@ -254,6 +254,10 @@ public class StructInstance implements ITypedStruct {
             bigDecimals[pos] = null;
         } else if (i.dataType() == DataTypes.DATE_TYPE) {
             dates[pos] = null;
+        } else if (i.dataType() == DataTypes.INT_TYPE) {
+            ints[pos] = 0;
+        } else if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
+            bools[pos] = false;
         } else if (i.dataType() == DataTypes.STRING_TYPE) {
             strings[pos] = null;
         } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {