Commit 0323ce38 by nixonrodrigues Committed by Madhan Neethiraj

ATLAS-3033: Skip hive temp table while getting Table object details from hiveContext.

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org> (cherry picked from commit b5472ce313dcf24b0bffbe5201f0c95eec025848)
parent aaf0ba23
...@@ -151,6 +151,18 @@ public class AtlasHiveHookContext { ...@@ -151,6 +151,18 @@ public class AtlasHiveHookContext {
return hook.getPreprocessActionForHiveTable(qualifiedName); return hook.getPreprocessActionForHiveTable(qualifiedName);
} }
public List getIgnoreDummyDatabaseName() {
return hook.getIgnoreDummyDatabaseName();
}
public List getIgnoreDummyTableName() {
return hook.getIgnoreDummyTableName();
}
public String getIgnoreValuesTmpTableNamePrefix() {
return hook.getIgnoreValuesTmpTableNamePrefix();
}
public String getQualifiedName(Database db) { public String getQualifiedName(Database db) {
return (db.getName() + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); return (db.getName() + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName();
} }
......
...@@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils; ...@@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HookContext;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
...@@ -83,6 +84,9 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ...@@ -83,6 +84,9 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
private static final List<Pattern> hiveTablesToIgnore = new ArrayList<>(); private static final List<Pattern> hiveTablesToIgnore = new ArrayList<>();
private static final List<Pattern> hiveTablesToPrune = new ArrayList<>(); private static final List<Pattern> hiveTablesToPrune = new ArrayList<>();
private static final Map<String, PreprocessAction> hiveTablesCache; private static final Map<String, PreprocessAction> hiveTablesCache;
private static final List ignoreDummyDatabaseName;
private static final List ignoreDummyTableName;
private static final String ignoreValuesTmpTableNamePrefix;
private static HiveHookObjectNamesCache knownObjects = null; private static HiveHookObjectNamesCache knownObjects = null;
private static String hostName; private static String hostName;
...@@ -138,6 +142,16 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ...@@ -138,6 +142,16 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
knownObjects = nameCacheEnabled ? new HiveHookObjectNamesCache(nameCacheDatabaseMaxCount, nameCacheTableMaxCount, nameCacheRebuildIntervalSeconds) : null; knownObjects = nameCacheEnabled ? new HiveHookObjectNamesCache(nameCacheDatabaseMaxCount, nameCacheTableMaxCount, nameCacheRebuildIntervalSeconds) : null;
List<String> defaultDummyDatabase = new ArrayList<>();
List<String> defaultDummyTable = new ArrayList<>();
defaultDummyDatabase.add(SemanticAnalyzer.DUMMY_DATABASE);
defaultDummyTable.add(SemanticAnalyzer.DUMMY_TABLE);
ignoreDummyDatabaseName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.database.name", defaultDummyDatabase);
ignoreDummyTableName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.table.name", defaultDummyTable);
ignoreValuesTmpTableNamePrefix = atlasProperties.getString("atlas.hook.hive.ignore.values.tmp.table.name.prefix", "Values__Tmp__Table__");
try { try {
hostName = InetAddress.getLocalHost().getHostName(); hostName = InetAddress.getLocalHost().getHostName();
} catch (UnknownHostException e) { } catch (UnknownHostException e) {
...@@ -256,6 +270,18 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ...@@ -256,6 +270,18 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
return skipHiveColumnLineageHive20633InputsThreshold; return skipHiveColumnLineageHive20633InputsThreshold;
} }
public List getIgnoreDummyDatabaseName() {
return ignoreDummyDatabaseName;
}
public List getIgnoreDummyTableName() {
return ignoreDummyTableName;
}
public String getIgnoreValuesTmpTableNamePrefix() {
return ignoreValuesTmpTableNamePrefix;
}
public PreprocessAction getPreprocessActionForHiveTable(String qualifiedName) { public PreprocessAction getPreprocessActionForHiveTable(String qualifiedName) {
PreprocessAction ret = PreprocessAction.NONE; PreprocessAction ret = PreprocessAction.NONE;
......
...@@ -246,18 +246,30 @@ public abstract class BaseHiveEvent { ...@@ -246,18 +246,30 @@ public abstract class BaseHiveEvent {
switch (entity.getType()) { switch (entity.getType()) {
case DATABASE: { case DATABASE: {
if (!context.getIgnoreDummyDatabaseName().contains(entity.getDatabase().getName())) {
Database db = getHive().getDatabase(entity.getDatabase().getName()); Database db = getHive().getDatabase(entity.getDatabase().getName());
ret = toDbEntity(db); ret = toDbEntity(db);
} }
}
break; break;
case TABLE: case TABLE:
case PARTITION: { case PARTITION: {
Table table = getHive().getTable(entity.getTable().getDbName(), entity.getTable().getTableName()); String dbName = entity.getTable().getDbName();
String tableName = entity.getTable().getTableName();
boolean skipTable = StringUtils.isNotEmpty(context.getIgnoreValuesTmpTableNamePrefix()) && tableName.toLowerCase().startsWith(context.getIgnoreValuesTmpTableNamePrefix());
if (!skipTable) {
skipTable = context.getIgnoreDummyTableName().contains(tableName) && context.getIgnoreDummyDatabaseName().contains(dbName);
}
if (!skipTable) {
Table table = getHive().getTable(dbName, tableName);
ret = toTableEntity(table, entityExtInfo); ret = toTableEntity(table, entityExtInfo);
} }
}
break; break;
case DFS_DIR: { case DFS_DIR: {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment