ATLAS-528 Support drop table,view (sumasai)

addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java

@@ -335,8 +335,8 @@ public class HiveMetaStoreBridge {
 
         tableReference.set(HiveDataModelGenerator.COLUMNS, getColumns(hiveTable.getCols(), tableQualifiedName));
 
-        // add reference to the StorageDescriptorx
-        Referenceable sdReferenceable = fillStorageDescStruct(hiveTable.getSd(), tableQualifiedName, tableQualifiedName);
+        // add reference to the StorageDescriptor
+        Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName));
         tableReference.set("sd", sdReferenceable);
 
         // add reference to the Partition Keys
@@ -359,6 +359,10 @@ public class HiveMetaStoreBridge {
         return tableReference;
     }
 
+    private String getStorageDescQFName(String entityQualifiedName) {
+        return entityQualifiedName + "_storage";
+    }
+
     private Referenceable registerTable(Referenceable dbReference, Table table) throws Exception {
         String dbName = table.getDbName();
         String tableName = table.getTableName();
@@ -410,7 +414,7 @@ public class HiveMetaStoreBridge {
         return new Referenceable(sd.getId().id, sd.getTypeName(), null);
     }
 
-    public Referenceable fillStorageDescStruct(StorageDescriptor storageDesc, String tableQualifiedName,
+    public Referenceable fillStorageDesc(StorageDescriptor storageDesc, String tableQualifiedName,
         String sdQualifiedName) throws Exception {
         LOG.debug("Filling storage descriptor information for " + storageDesc);
 
@@ -466,7 +470,7 @@ public class HiveMetaStoreBridge {
         ref.set("path", pathUri);
 //        Path path = new Path(pathUri);
 //        ref.set("name", path.getName());
-//        TODO - Fix after ATLAS-542 to shorter Name
+        //TODO - Fix after ATLAS-542 to shorter Name
         ref.set("name", pathUri);
         ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri);
         return ref;

addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java

@@ -308,6 +308,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
         case EXPORT:
         case IMPORT:
         case QUERY:
+        case TRUNCATETABLE:
             registerProcess(dgiBridge, event);
             break;
 
@@ -326,6 +327,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
         case ALTERTABLE_ADDCOLS:
         case ALTERTABLE_REPLACECOLS:
         case ALTERTABLE_RENAMECOL:
+        case ALTERTABLE_PARTCOLTYPE:
             handleEventOutputs(dgiBridge, event, Type.TABLE);
             break;
         case ALTERTABLE_LOCATION:
@@ -334,17 +336,37 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
                 //Track altered lineage in case of external tables
                 handleExternalTables(dgiBridge, event, tablesUpdated.get(0).getLeft(), tablesUpdated.get(0).getRight());
             }
+            break;
         case ALTERDATABASE:
         case ALTERDATABASE_OWNER:
             handleEventOutputs(dgiBridge, event, Type.DATABASE);
             break;
 
+        case DROPTABLE:
+        case DROPVIEW:
+            deleteTable(dgiBridge, event);
+            break;
+
         default:
         }
 
         notifyEntities(messages);
     }
 
+    private void deleteTable(HiveMetaStoreBridge dgiBridge, HiveEventContext event) {
+        for (Entity output : event.outputs) {
+            if (Type.TABLE.equals(output.getType())) {
+                final String tblQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(dgiBridge.getClusterName(), output.getTable().getDbName(), output.getTable().getTableName());
+                LOG.info("Deleting table {} ", tblQualifiedName);
+                messages.add(
+                    new HookNotification.EntityDeleteRequest(event.getUser(),
+                        HiveDataTypes.HIVE_TABLE.getName(),
+                        HiveDataModelGenerator.NAME,
+                        tblQualifiedName));
+            }
+        }
+    }
+
     private void renameTable(HiveMetaStoreBridge dgiBridge, HiveEventContext event) throws Exception {
         //crappy, no easy of getting new name
         assert event.getInputs() != null && event.getInputs().size() == 1;

addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java

@@ -497,7 +497,6 @@ public class HiveHookIT {
         runCommand(query);
         Referenceable processReference = validateProcess(query, 1, 1);
         validateHDFSPaths(processReference, filename, OUTPUTS);
-
         validateInputTables(processReference, tableId);
 
         //Import
@@ -510,7 +509,6 @@ public class HiveHookIT {
         validateHDFSPaths(processReference, filename, INPUTS);
 
         validateOutputTables(processReference, tableId);
-
     }
 
     @Test
@@ -541,8 +539,6 @@ public class HiveHookIT {
         validateHDFSPaths(processReference, filename, INPUTS);
 
         validateOutputTables(processReference, tableId);
-
-
     }
 
     @Test
@@ -684,6 +680,41 @@ public class HiveHookIT {
         Assert.assertEquals(columns.get(0).get(HiveDataModelGenerator.NAME), "id");
     }
 
+    @Test()
+    public void testTruncateTable() throws Exception {
+        String tableName = createTable(false);
+        String query = String.format("truncate table %s", tableName);
+        runCommand(query);
+
+        String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
+        validateProcess(query, 0, 1);
+
+        //Check lineage
+        String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName);
+        JSONObject response = dgiCLient.getInputGraph(datasetName);
+        JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
+        //Below should be assertTrue - Fix https://issues.apache.org/jira/browse/ATLAS-653
+        Assert.assertFalse(vertices.has(tableId));
+    }
+
+    @Test
+    public void testAlterTablePartitionColumnType() throws Exception {
+        String tableName = createTable(true, true, false);
+        final String newType = "int";
+        String query = String.format("ALTER TABLE %s PARTITION COLUMN (dt %s)", tableName, newType);
+        runCommand(query);
+
+        final String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
+        final String dtColId = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "dt"));
+        Referenceable table = dgiCLient.getEntity(tableId);
+        Referenceable column = dgiCLient.getEntity(dtColId);
+        Assert.assertEquals(column.get("type"), newType);
+
+        final List<Referenceable> partitionKeys = (List<Referenceable>) table.get("partitionKeys");
+        Assert.assertEquals(partitionKeys.size(), 1);
+        Assert.assertEquals(partitionKeys.get(0).getId()._getId(), dtColId);
+    }
+
     @Test
     public void testAlterViewRename() throws Exception {
         String tableName = createTable();
@@ -834,7 +865,56 @@ public class HiveHookIT {
 
         //Add another property
         runSerdePropsQuery(tableName, expectedProps);
+    }
+
+    @Test
+    public void testDropTable() throws Exception {
+        //Test Deletion of tables and its corrresponding columns
+        String tableName = createTable(true, true, false);
 
+        assertTableIsRegistered(DEFAULT_DB, tableName);
+        assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "id"));
+        assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "name"));
+
+        final String query = String.format("drop table %s ", tableName);
+
+        runCommand(query);
+        assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "id"));
+        assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "name"));
+        assertTableIsNotRegistered(DEFAULT_DB, tableName);
+    }
+
+    @Test
+    public void testDropNonExistingTable() throws Exception {
+        //Test Deletion of a non existing table
+        final String tableName = "nonexistingtable";
+        assertTableIsNotRegistered(DEFAULT_DB, tableName);
+        final String query = String.format("drop table if exists %s", tableName);
+        runCommand(query);
+
+        //Should have no effect
+        assertTableIsNotRegistered(DEFAULT_DB, tableName);
+        assertProcessIsNotRegistered(query);
+    }
+
+    @Test
+    public void testDropView() throws Exception {
+        //Test Deletion of tables and its corrresponding columns
+        String tableName = createTable(true, true, false);
+        String viewName = tableName();
+        String query = "create view " + viewName + " as select * from " + tableName;
+        runCommand(query);
+
+        assertTableIsRegistered(DEFAULT_DB, viewName);
+        assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "id"));
+        assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "name"));
+
+        query = String.format("drop view %s ", viewName);
+
+        runCommand(query);
+        assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "id"));
+        assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "name"));
+        assertTableIsNotRegistered(DEFAULT_DB, viewName);
     }
 
     private void runSerdePropsQuery(String tableName, Map<String, String> expectedProps) throws Exception {

release-log.txt

@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-528 Support drop table,view (sumasai)
 ATLAS-603 Document High Availability of Atlas (yhemanth via sumasai)
 ATLAS-498 Support Embedded HBase (tbeerbower via sumasai)
 ATLAS-527 Support lineage for load table, import, export (sumasai via shwethags)