ATLAS-3197: Disable creation of hive process, hive process execution and ddl…

addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java

@@ -148,11 +148,13 @@ public class AlterTableRename extends BaseHiveEvent {
         ret.add(new EntityPartialUpdateRequestV2(getUserName(), oldTableId, renamedTableEntity));
 
         // partial update relationship attribute ddl
+        if (!context.isMetastoreHook()) {
             AtlasEntity ddlEntity = createHiveDDLEntity(renamedTableEntity.getEntity(), true);
 
             if (ddlEntity != null) {
                 ret.add(new HookNotification.EntityCreateRequestV2(getUserName(), new AtlasEntitiesWithExtInfo(ddlEntity)));
             }
+        }
 
         context.removeFromKnownTable(oldTableQualifiedName);
     }

addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java

@@ -108,6 +108,14 @@ public class CreateHiveProcess extends BaseHiveEvent {
                     if (entity != null) {
                         outputs.add(entity);
                     }
+
+                    if (entity != null && !context.isMetastoreHook()) {
+                        AtlasEntity ddlEntity = createHiveDDLEntity(entity);
+
+                        if (ddlEntity != null) {
+                            ret.addEntity(ddlEntity);
+                        }
+                    }
                 }
             }
 
@@ -121,7 +129,7 @@ public class CreateHiveProcess extends BaseHiveEvent {
                 }
             }
 
-            if (!skipProcess) {
+            if (!skipProcess && !context.isMetastoreHook()) {
                 AtlasEntity process = getHiveProcessEntity(inputs, outputs);
 
                 ret.addEntity(process);

addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java

@@ -117,7 +117,7 @@ public class CreateTable extends BaseHiveEvent {
         if (table != null) {
             AtlasEntity tblEntity = toTableEntity(table, ret);
 
-            if (tblEntity != null) {
+            if (tblEntity != null && !context.isMetastoreHook()) {
                 if (isHBaseStore(table)) {
                     // This create lineage to HBase table in case of Hive on HBase
                     AtlasEntity hbaseTableEntity = toReferencedHBaseTable(table, ret);
@@ -134,7 +134,6 @@ public class CreateTable extends BaseHiveEvent {
 
                         AtlasEntity processExecution = getHiveProcessExecutionEntity(processEntity);
                         ret.addEntity(processExecution);
-
                     }
                 } else {
                     if (EXTERNAL_TABLE.equals(table.getTableType())) {
@@ -148,7 +147,6 @@ public class CreateTable extends BaseHiveEvent {
                         ret.addEntity(processExecution);
                     }
                 }
-            }
 
                 AtlasEntity tableDDLEntity = createHiveDDLEntity(tblEntity);
 
@@ -157,6 +155,7 @@ public class CreateTable extends BaseHiveEvent {
                 }
             }
         }
+    }
 
     private static boolean isAlterTable(HiveOperation oper) {
         return (oper == ALTERTABLE_PROPERTIES || oper == ALTERTABLE_RENAME || oper == ALTERTABLE_RENAMECOL);

addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java

@@ -128,7 +128,7 @@ public class HiveHookIT extends HiveITBase {
         String      tableId   = assertTableIsRegistered(dbName, tableName);
         String      colId     = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, dbName, tableName), colName)); //there is only one instance of column registered
         AtlasEntity colEntity = atlasClientV2.getEntityByGuid(colId).getEntity();
-        AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(colId).getEntity();
+        AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(tableId).getEntity();
 
         Assert.assertEquals(colEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), String.format("%s.%s.%s@%s", dbName.toLowerCase(), tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME));
         Assert.assertNotNull(colEntity.getAttribute(ATTRIBUTE_TABLE));
@@ -202,7 +202,12 @@ public class HiveHookIT extends HiveITBase {
 
         runCommand(query);
 
-        assertTableIsRegistered(DEFAULT_DB, tableName, null, true);
+        String tblId         = assertTableIsRegistered(DEFAULT_DB, tableName, null, true);
+        AtlasEntity tblEnity = atlasClientV2.getEntityByGuid(tblId).getEntity();
+        List ddlList         = (List) tblEnity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        assertNotNull(ddlList);
+        assertEquals(ddlList.size(), 1);
 
         String processId = assertEntityIsRegistered(HiveDataTypes.HIVE_PROCESS.getName(), ATTRIBUTE_QUALIFIED_NAME, getTableProcessQualifiedName(DEFAULT_DB, tableName), null);
 
@@ -381,15 +386,20 @@ public class HiveHookIT extends HiveITBase {
                 BaseHiveEvent.ATTRIBUTE_PROCESS));
         Assert.assertEquals(process.getGuid(), processEntity1.getGuid());
 
-        String           drpquery         = String.format("drop table %s ", ctasTableName);
+        String           dropQuery         = String.format("drop table %s ", ctasTableName);
 
-        runCommandWithDelay(drpquery, 100);
+        runCommandWithDelay(dropQuery, 5000);
 
         assertTableIsNotRegistered(DEFAULT_DB, ctasTableName);
 
         runCommand(query);
 
-        assertTableIsRegistered(DEFAULT_DB, ctasTableName);
+        String tblId          = assertTableIsRegistered(DEFAULT_DB, ctasTableName);
+        AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(tblId).getEntity();
+        List ddlList          = (List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        assertNotNull(ddlList);
+        assertEquals(ddlList.size(), 1);
 
         outputs =  getOutputs(ctasTableName, Entity.Type.TABLE);
 
@@ -423,6 +433,13 @@ public class HiveHookIT extends HiveITBase {
         Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
         Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
         assertTableIsRegistered(DEFAULT_DB, viewName);
+
+        String viewId          = assertTableIsRegistered(DEFAULT_DB, viewName);
+        AtlasEntity viewEntity = atlasClientV2.getEntityByGuid(viewId).getEntity();
+        List ddlQueries        = (List) viewEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        Assert.assertNotNull(ddlQueries);
+        Assert.assertEquals(ddlQueries.size(), 1);
     }
 
     @Test
@@ -476,8 +493,15 @@ public class HiveHookIT extends HiveITBase {
         Assert.assertEquals(processEntity1.getGuid(), processEntity2.getGuid());
 
         String table2Id = assertTableIsRegistered(DEFAULT_DB, table2Name);
+        String viewId2  = assertTableIsRegistered(DEFAULT_DB, viewName);
+
+        Assert.assertEquals(viewId2, viewId);
+
+        AtlasEntity viewEntity = atlasClientV2.getEntityByGuid(viewId2).getEntity();
+        List ddlQueries        = (List) viewEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
 
-        Assert.assertEquals(assertTableIsRegistered(DEFAULT_DB, viewName), viewId);
+        Assert.assertNotNull(ddlQueries);
+        Assert.assertEquals(ddlQueries.size(), 2);
 
         datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName);
 
@@ -745,7 +769,13 @@ public class HiveHookIT extends HiveITBase {
         Assert.assertEquals(process.getGuid(), hiveProcess.getGuid());
         Assert.assertEquals(numberOfProcessExecutions(hiveProcess), 1);
 
-        assertTableIsRegistered(DEFAULT_DB, tableName);
+        String tblId          = assertTableIsRegistered(DEFAULT_DB, tableName);
+
+        AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(tblId).getEntity();
+        List ddlQueries       = (List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        Assert.assertNotNull(ddlQueries);
+        Assert.assertEquals(ddlQueries.size(), 1);
     }
 
     @Test
@@ -797,7 +827,13 @@ public class HiveHookIT extends HiveITBase {
 
         runCommandWithDelay(query, 1000);
 
-        assertTableIsRegistered(DEFAULT_DB, tableName);
+        String tblId          = assertTableIsRegistered(DEFAULT_DB, tableName);
+
+        AtlasEntity tblEntity = atlasClientV2.getEntityByGuid(tblId).getEntity();
+        List ddlQueries       = (List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        Assert.assertNotNull(ddlQueries);
+        Assert.assertEquals(ddlQueries.size(), 1);
 
         Set<WriteEntity> p3Outputs = new LinkedHashSet<WriteEntity>() {{
             addAll(getOutputs(pFile2, Entity.Type.DFS_DIR));
@@ -1430,7 +1466,7 @@ public class HiveHookIT extends HiveITBase {
         List        ddlQueries3 = (List) tblEntity3.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
 
         Assert.assertNotNull(ddlQueries3);
-        Assert.assertEquals(ddlQueries3.size(), 4);
+        Assert.assertEquals(ddlQueries3.size(), 5);
 
         //Change col position again
         oldColName = "name4";
@@ -1466,7 +1502,7 @@ public class HiveHookIT extends HiveITBase {
         List        ddlQueries4 = (List) tblEntity4.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
 
         Assert.assertNotNull(ddlQueries4);
-        Assert.assertEquals(ddlQueries4.size(), 5);
+        Assert.assertEquals(ddlQueries4.size(), 6);
     }
 
     /**

addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java

@@ -26,9 +26,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.annotations.Test;
 
-import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 
+import static org.apache.atlas.hive.hook.events.BaseHiveEvent.ATTRIBUTE_DDL_QUERIES;
 import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
 import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
 import static org.testng.AssertJUnit.*;
@@ -43,9 +44,9 @@ public class HiveMetastoreHookIT extends HiveITBase {
 
         runCommand(query);
         String dbId          = assertDatabaseIsRegistered(dbName);
-
         AtlasEntity dbEntity = getAtlasEntity(dbId);
-        assertNotNull(dbEntity);
+
+        assertEquals(((List) dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
     }
 
     @Test (priority = 2)
@@ -66,6 +67,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
         dbEntity = getAtlasEntity(dbId);
         Map parameters = (Map) dbEntity.getAttribute("parameters");
 
+        assertEquals(((List) dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
         assertNotNull(parameters);
         assertEquals(2, parameters.size());
 
@@ -75,6 +77,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
 
         dbEntity = getAtlasEntity(dbId);
 
+
+        assertEquals(((List) dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
         assertEquals(dbEntity.getAttribute("owner"), "hive");
         assertEquals(dbEntity.getAttribute("ownerType"), "USER");
 
@@ -86,6 +90,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
 
         dbEntity = getAtlasEntity(dbId);
 
+        assertEquals(((List) dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
+
         String location = (String) dbEntity.getAttribute("location");
         assertEquals(location, hdfsPath);
     }
@@ -149,6 +155,9 @@ public class HiveMetastoreHookIT extends HiveITBase {
         String tableName      = tableName();
         runCommand("CREATE TABLE " + dbName + "." + tableName + " (name string, age int, dob date)");
         String tblId          = assertTableIsRegistered(dbName, tableName);
+        AtlasEntity tblEntity = getAtlasEntity(tblId);
+
+        assertEquals(((List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
         assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
     }
 
@@ -167,8 +176,13 @@ public class HiveMetastoreHookIT extends HiveITBase {
         assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
 
         String viewName = tableName();
+
         runCommand("CREATE VIEW " + dbName + "." + viewName + " AS SELECT * FROM " + dbName + "." + tableName);
+
         tblId                 = assertTableIsRegistered(dbName, viewName);
+        AtlasEntity tblEntity = getAtlasEntity(tblId);
+
+        assertEquals(((List) tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
         assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
     }
 
@@ -180,11 +194,13 @@ public class HiveMetastoreHookIT extends HiveITBase {
         runCommand(query);
         String dbId = assertDatabaseIsRegistered(dbName);
         assertEquals(getAtlasEntity(dbId).getStatus(), ACTIVE);
+        assertEquals(((List) getAtlasEntity(dbId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
 
         String tableName = tableName();
         runCommand("CREATE TABLE " + dbName + "." + tableName + " (name string, age int, dob date)");
         String tblId = assertTableIsRegistered(dbName, tableName);
         assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
+        assertEquals(((List) getAtlasEntity(tblId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
 
         // SET TBLPROPERTIES
         query = "ALTER TABLE " + dbName + "." + tableName + " SET TBLPROPERTIES (\"prop1\"=\"val1\", \"prop2\"=\"val2\", \"prop3\"=\"val3\")";
@@ -216,6 +232,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
         assertEquals(serdeAttrs.get("serializationLib"), "org.apache.hadoop.hive.ql.io.orc.OrcSerde");
         assertEquals(((Map) serdeAttrs.get("parameters")).get("prop1"), "val1");
         assertEquals(((Map) serdeAttrs.get("parameters")).get("prop2"), "val2");
+        assertEquals(((List) tableEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
     }
 
     @Test (priority = 8)
@@ -240,6 +257,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
         AtlasEntityWithExtInfo tableEntityWithExtInfo = getAtlasEntityWithExtInfo(tblId);
         AtlasEntity            tableEntity            = tableEntityWithExtInfo.getEntity();
 
+        assertEquals(((List) tableEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
+
         // validate table rename in table entity
         assertEquals(newTableName, tableEntity.getAttribute("name"));
         assertTrue(((String) tableEntity.getAttribute("qualifiedName")).contains(newTableName));
@@ -284,6 +303,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
         AtlasEntity col2Entity = getAtlasEntity(col2Id);
         assertEquals(col2Entity.getAttribute("name"), "col22");
         assertEquals(col2Entity.getAttribute("type"), "string");
+        assertEquals(((List) getAtlasEntity(tblId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
     }
 
     @Test (priority = 10)