Skip to content

A
atlas
Commit 4e6339d8 by Le Ma Committed by Sarath Subramanian
Options

ATLAS-3197: Disable creation of hive process, hive process execution and ddl…

ATLAS-3197: Disable creation of hive process, hive process execution and ddl query entities for HMS hook
parent f75d7f40
master
No related merge requests found
Showing with 82 additions and 17 deletions
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java
......@@ -148,11 +148,13 @@ public class AlterTableRename extends BaseHiveEvent {
ret.add(new EntityPartialUpdateRequestV2(getUserName(), oldTableId, renamedTableEntity));
// partial update relationship attribute ddl
if (!context.isMetastoreHook()) {
AtlasEntity ddlEntity = createHiveDDLEntity(renamedTableEntity.getEntity(), true);
if (ddlEntity != null) {
ret.add(new HookNotification.EntityCreateRequestV2(getUserName(), new AtlasEntitiesWithExtInfo(ddlEntity)));
}
}
context.removeFromKnownTable(oldTableQualifiedName);
}
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
......@@ -108,6 +108,14 @@ public class CreateHiveProcess extends BaseHiveEvent {
if (entity != null) {
outputs.add(entity);
}
if (entity != null && !context.isMetastoreHook()) {
AtlasEntity ddlEntity = createHiveDDLEntity(entity);
if (ddlEntity != null) {
ret.addEntity(ddlEntity);
}
}
}
}
......@@ -121,7 +129,7 @@ public class CreateHiveProcess extends BaseHiveEvent {
}
}
if (!skipProcess) {
if (!skipProcess && !context.isMetastoreHook()) {
AtlasEntity process = getHiveProcessEntity(inputs, outputs);
ret.addEntity(process);
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
......@@ -117,7 +117,7 @@ public class CreateTable extends BaseHiveEvent {
if (table != null) {
AtlasEntity tblEntity = toTableEntity(table, ret);
if (tblEntity != null) {
if (tblEntity != null && !context.isMetastoreHook()) {
if (isHBaseStore(table)) {
// This create lineage to HBase table in case of Hive on HBase
AtlasEntity hbaseTableEntity = toReferencedHBaseTable(table, ret);
......@@ -134,7 +134,6 @@ public class CreateTable extends BaseHiveEvent {
AtlasEntity processExecution = getHiveProcessExecutionEntity(processEntity);
ret.addEntity(processExecution);
}
} else {
if (EXTERNAL_TABLE.equals(table.getTableType())) {
......@@ -148,7 +147,6 @@ public class CreateTable extends BaseHiveEvent {
ret.addEntity(processExecution);
}
}
}
AtlasEntity tableDDLEntity = createHiveDDLEntity(tblEntity);
......@@ -157,6 +155,7 @@ public class CreateTable extends BaseHiveEvent {
}
}
}
}
private static boolean isAlterTable(HiveOperation oper) {
return (oper == ALTERTABLE_PROPERTIES || oper == ALTERTABLE_RENAME || oper == ALTERTABLE_RENAMECOL);
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
......@@ -128,7 +128,7 @@ public class HiveHookIT extends HiveITBase {
String tableId = assertTableIsRegistered(dbName, tableName);
String colId = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, dbName, tableName), colName)); //there is only one instance of column registered
AtlasEntity colEntity = atlasClientV2.getEntityByGuid(colId).getEntity();
AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(colId).getEntity();
AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(tableId).getEntity();
Assert.assertEquals(colEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), String.format("%s.%s.%s@%s", dbName.toLowerCase(), tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME));
Assert.assertNotNull(colEntity.getAttribute(ATTRIBUTE_TABLE));
......@@ -202,7 +202,12 @@ public class HiveHookIT extends HiveITBase {
runCommand(query);
assertTableIsRegistered(DEFAULT_DB, tableName, null, true);
String tblId = assertTableIsRegistered(DEFAULT_DB, tableName, null, true);
AtlasEntity tblEnity = atlasClientV2.getEntityByGuid(tblId).getEntity();
List ddlList = (List) tblEnity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
assertNotNull(ddlList);
assertEquals(ddlList.size(), 1);
String processId = assertEntityIsRegistered(HiveDataTypes.HIVE_PROCESS.getName(), ATTRIBUTE_QUALIFIED_NAME, getTableProcessQualifiedName(DEFAULT_DB, tableName), null);
......@@ -381,15 +386,20 @@ public class HiveHookIT extends HiveITBase {
BaseHiveEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process.getGuid(), processEntity1.getGuid());
String drpquery = String.format("drop table %s ", ctasTableName);
String dropQuery = String.format("drop table %s ", ctasTableName);
runCommandWithDelay(drpquery, 100);
runCommandWithDelay(dropQuery, 5000);
assertTableIsNotRegistered(DEFAULT_DB, ctasTableName);
runCommand(query);
assertTableIsRegistered(DEFAULT_DB, ctasTableName);
String tblId = assertTableIsRegistered(DEFAULT_DB, ctasTableName);
AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(tblId).getEntity();
List ddlList = (List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
assertNotNull(ddlList);
assertEquals(ddlList.size(), 1);
outputs = getOutputs(ctasTableName, Entity.Type.TABLE);
......@@ -423,6 +433,13 @@ public class HiveHookIT extends HiveITBase {
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
assertTableIsRegistered(DEFAULT_DB, viewName);
String viewId = assertTableIsRegistered(DEFAULT_DB, viewName);
AtlasEntity viewEntity = atlasClientV2.getEntityByGuid(viewId).getEntity();
List ddlQueries = (List) viewEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
Assert.assertNotNull(ddlQueries);
Assert.assertEquals(ddlQueries.size(), 1);
}
@Test
......@@ -476,8 +493,15 @@ public class HiveHookIT extends HiveITBase {
Assert.assertEquals(processEntity1.getGuid(), processEntity2.getGuid());
String table2Id = assertTableIsRegistered(DEFAULT_DB, table2Name);
String viewId2 = assertTableIsRegistered(DEFAULT_DB, viewName);
Assert.assertEquals(viewId2, viewId);
AtlasEntity viewEntity = atlasClientV2.getEntityByGuid(viewId2).getEntity();
List ddlQueries = (List) viewEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
Assert.assertEquals(assertTableIsRegistered(DEFAULT_DB, viewName), viewId);
Assert.assertNotNull(ddlQueries);
Assert.assertEquals(ddlQueries.size(), 2);
datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName);
......@@ -745,7 +769,13 @@ public class HiveHookIT extends HiveITBase {
Assert.assertEquals(process.getGuid(), hiveProcess.getGuid());
Assert.assertEquals(numberOfProcessExecutions(hiveProcess), 1);
assertTableIsRegistered(DEFAULT_DB, tableName);
String tblId = assertTableIsRegistered(DEFAULT_DB, tableName);
AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(tblId).getEntity();
List ddlQueries = (List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
Assert.assertNotNull(ddlQueries);
Assert.assertEquals(ddlQueries.size(), 1);
}
@Test
......@@ -797,7 +827,13 @@ public class HiveHookIT extends HiveITBase {
runCommandWithDelay(query, 1000);
assertTableIsRegistered(DEFAULT_DB, tableName);
String tblId = assertTableIsRegistered(DEFAULT_DB, tableName);
AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(tblId).getEntity();
List ddlQueries = (List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
Assert.assertNotNull(ddlQueries);
Assert.assertEquals(ddlQueries.size(), 1);
Set<WriteEntity> p3Outputs = new LinkedHashSet<WriteEntity>() {{
addAll(getOutputs(pFile2, Entity.Type.DFS_DIR));
......@@ -1430,7 +1466,7 @@ public class HiveHookIT extends HiveITBase {
List ddlQueries3 = (List) tblEntity3.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
Assert.assertNotNull(ddlQueries3);
Assert.assertEquals(ddlQueries3.size(), 4);
Assert.assertEquals(ddlQueries3.size(), 5);
//Change col position again
oldColName = "name4";
......@@ -1466,7 +1502,7 @@ public class HiveHookIT extends HiveITBase {
List ddlQueries4 = (List) tblEntity4.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
Assert.assertNotNull(ddlQueries4);
Assert.assertEquals(ddlQueries4.size(), 5);
Assert.assertEquals(ddlQueries4.size(), 6);
}
/**
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java
......@@ -26,9 +26,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.hive.hook.events.BaseHiveEvent.ATTRIBUTE_DDL_QUERIES;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.testng.AssertJUnit.*;
......@@ -43,9 +44,9 @@ public class HiveMetastoreHookIT extends HiveITBase {
runCommand(query);
String dbId = assertDatabaseIsRegistered(dbName);
AtlasEntity dbEntity = getAtlasEntity(dbId);
assertNotNull(dbEntity);
assertEquals(((List) dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
}
@Test (priority = 2)
......@@ -66,6 +67,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
dbEntity = getAtlasEntity(dbId);
Map parameters = (Map) dbEntity.getAttribute("parameters");
assertEquals(((List) dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
assertNotNull(parameters);
assertEquals(2, parameters.size());
......@@ -75,6 +77,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
dbEntity = getAtlasEntity(dbId);
assertEquals(((List) dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
assertEquals(dbEntity.getAttribute("owner"), "hive");
assertEquals(dbEntity.getAttribute("ownerType"), "USER");
......@@ -86,6 +90,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
dbEntity = getAtlasEntity(dbId);
assertEquals(((List) dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
String location = (String) dbEntity.getAttribute("location");
assertEquals(location, hdfsPath);
}
......@@ -149,6 +155,9 @@ public class HiveMetastoreHookIT extends HiveITBase {
String tableName = tableName();
runCommand("CREATE TABLE " + dbName + "." + tableName + " (name string, age int, dob date)");
String tblId = assertTableIsRegistered(dbName, tableName);
AtlasEntity tblEntity = getAtlasEntity(tblId);
assertEquals(((List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
}
......@@ -167,8 +176,13 @@ public class HiveMetastoreHookIT extends HiveITBase {
assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
String viewName = tableName();
runCommand("CREATE VIEW " + dbName + "." + viewName + " AS SELECT * FROM " + dbName + "." + tableName);
tblId = assertTableIsRegistered(dbName, viewName);
AtlasEntity tblEntity = getAtlasEntity(tblId);
assertEquals(((List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
}
......@@ -180,11 +194,13 @@ public class HiveMetastoreHookIT extends HiveITBase {
runCommand(query);
String dbId = assertDatabaseIsRegistered(dbName);
assertEquals(getAtlasEntity(dbId).getStatus(), ACTIVE);
assertEquals(((List) getAtlasEntity(dbId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
String tableName = tableName();
runCommand("CREATE TABLE " + dbName + "." + tableName + " (name string, age int, dob date)");
String tblId = assertTableIsRegistered(dbName, tableName);
assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
assertEquals(((List) getAtlasEntity(tblId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
// SET TBLPROPERTIES
query = "ALTER TABLE " + dbName + "." + tableName + " SET TBLPROPERTIES (\"prop1\"=\"val1\", \"prop2\"=\"val2\", \"prop3\"=\"val3\")";
......@@ -216,6 +232,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
assertEquals(serdeAttrs.get("serializationLib"), "org.apache.hadoop.hive.ql.io.orc.OrcSerde");
assertEquals(((Map) serdeAttrs.get("parameters")).get("prop1"), "val1");
assertEquals(((Map) serdeAttrs.get("parameters")).get("prop2"), "val2");
assertEquals(((List) tableEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
}
@Test (priority = 8)
......@@ -240,6 +257,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
AtlasEntityWithExtInfo tableEntityWithExtInfo = getAtlasEntityWithExtInfo(tblId);
AtlasEntity tableEntity = tableEntityWithExtInfo.getEntity();
assertEquals(((List) tableEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
// validate table rename in table entity
assertEquals(newTableName, tableEntity.getAttribute("name"));
assertTrue(((String) tableEntity.getAttribute("qualifiedName")).contains(newTableName));
......@@ -284,6 +303,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
AtlasEntity col2Entity = getAtlasEntity(col2Id);
assertEquals(col2Entity.getAttribute("name"), "col22");
assertEquals(col2Entity.getAttribute("type"), "string");
assertEquals(((List) getAtlasEntity(tblId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
}
@Test (priority = 10)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment