ATLAS-524 Support alter database (sumasai via shwethags)

addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java

@@ -166,7 +166,7 @@ public class HiveMetaStoreBridge {
         dbRef.set(DESCRIPTION_ATTR, hiveDB.getDescription());
         dbRef.set("locationUri", hiveDB.getLocationUri());
         dbRef.set(HiveDataModelGenerator.PARAMETERS, hiveDB.getParameters());
-        dbRef.set("ownerName", hiveDB.getOwnerName());
+        dbRef.set(HiveDataModelGenerator.OWNER, hiveDB.getOwnerName());
         if (hiveDB.getOwnerType() != null) {
             dbRef.set("ownerType", hiveDB.getOwnerType().getValue());
         }

addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java

@@ -241,12 +241,28 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
             alterTable(dgiBridge, event);
             break;
 
+        case ALTERDATABASE:
+        case ALTERDATABASE_OWNER:
+            alterDatabase(dgiBridge, event);
+            break;
+
         default:
         }
 
         notifyEntities(messages);
     }
 
+    private void alterDatabase(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
+        assert event.outputs != null && event.outputs.size() > 0;
+
+        for (WriteEntity writeEntity : event.outputs) {
+            if (writeEntity.getType() == Type.DATABASE) {
+                //Create/update table entity
+                createOrUpdateEntities(dgiBridge, writeEntity);
+            }
+        }
+    }
+
     private void alterTable(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
         assert event.inputs != null && event.inputs.size() == 1;
         assert event.outputs != null && event.outputs.size() > 0;

addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java

@@ -42,11 +42,6 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -81,6 +76,7 @@ public class HiveDataModelGenerator {
     public static final String STORAGE_DESC = "sd";
     public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
     public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
+    public static final String OWNER = "owner";
 
     public HiveDataModelGenerator() {
         classTypeDefinitions = new HashMap<>();
@@ -242,7 +238,7 @@ public class HiveDataModelGenerator {
                 new AttributeDefinition("locationUri", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
                         null),
                 new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
-                new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
+                new AttributeDefinition(OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
                         null),
                 new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), Multiplicity.OPTIONAL,
                         false, null),};
@@ -307,7 +303,7 @@ public class HiveDataModelGenerator {
                 new AttributeDefinition(TABLE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
                         null),
                 new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(), Multiplicity.REQUIRED, false, null),
-                new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
+                new AttributeDefinition(OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
                 new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false,
                         null),
                 new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false,
@@ -342,7 +338,7 @@ public class HiveDataModelGenerator {
                         null),
                 new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(), Multiplicity.REQUIRED, false,
                         null),
-                new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
+                new AttributeDefinition(OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
                         null),};
         HierarchicalTypeDefinition<ClassType> definition =
                 new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_ROLE.getName(), null, null,

addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java

@@ -36,6 +36,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.hooks.Entity;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.codehaus.jettison.json.JSONArray;
@@ -236,7 +237,6 @@ public class HiveHookIT {
         String table2Id = assertTableIsRegistered(DEFAULT_DB, table2Name);
         Assert.assertEquals(assertTableIsRegistered(DEFAULT_DB, viewName), viewId);
 
-        //Check lineage which includes both table1 and table2
         datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName);
         response = dgiCLient.getInputGraph(datasetName);
         vertices = response.getJSONObject("values").getJSONObject("vertices");
@@ -469,7 +469,7 @@ public class HiveHookIT {
         Struct serdeInfo = (Struct) sdRef.get("serdeInfo");
         Assert.assertEquals(serdeInfo.get("serializationLib"), "org.apache.hadoop.hive.ql.io.orc.OrcSerde");
         Assert.assertNotNull(serdeInfo.get(HiveDataModelGenerator.PARAMETERS));
-        Assert.assertEquals(((Map<String, String>) serdeInfo.get(HiveDataModelGenerator.PARAMETERS)).get("serialization.format"), "1");
+        Assert.assertEquals(((Map<String, String>)serdeInfo.get(HiveDataModelGenerator.PARAMETERS)).get("serialization.format"), "1");
 
 
         /**
@@ -499,7 +499,6 @@ public class HiveHookIT {
         final String fmtQuery = "alter table %s CLUSTERED BY (%s) SORTED BY (%s) INTO %s BUCKETS";
         String query = String.format(fmtQuery, tableName, stripListBrackets(bucketCols.toString()), stripListBrackets(sortCols.toString()), numBuckets);
         runCommand(query);
-
         verifyBucketSortingProperties(tableName, numBuckets, bucketCols, sortCols);
     }
 
@@ -568,14 +567,35 @@ public class HiveHookIT {
     }
 
     @Test
+    public void testAlterDBOwner() throws Exception {
+        String dbName = createDatabase();
+        final String owner = "testOwner";
+        String dbId = assertDatabaseIsRegistered(dbName);
+        final String fmtQuery = "alter database %s set OWNER %s %s";
+        String query = String.format(fmtQuery, dbName, "USER", owner);
+
+        runCommand(query);
+
+        assertDatabaseIsRegistered(dbName);
+        Referenceable entity = dgiCLient.getEntity(dbId);
+        Assert.assertEquals(entity.get(HiveDataModelGenerator.OWNER), owner);
+    }
+
+    @Test
+    public void testAlterDBProperties() throws Exception {
+        String dbName = createDatabase();
+        final String fmtQuery = "alter database %s %s DBPROPERTIES (%s)";
+        testAlterProperties(Entity.Type.DATABASE, dbName, fmtQuery);
+    }
+
+    @Test
     public void testAlterTableProperties() throws Exception {
         String tableName = createTable();
         final String fmtQuery = "alter table %s %s TBLPROPERTIES (%s)";
-        testAlterProperties(tableName, fmtQuery);
+        testAlterProperties(Entity.Type.TABLE, tableName, fmtQuery);
     }
 
-    private void testAlterProperties(String tableName, String fmtQuery) throws Exception {
-
+    private void testAlterProperties(Entity.Type entityType, String entityName, String fmtQuery) throws Exception {
         final String SET_OP = "set";
         final String UNSET_OP = "unset";
 
@@ -584,24 +604,25 @@ public class HiveHookIT {
             put("comment", "test comment");
         }};
 
-        String query = String.format(fmtQuery, tableName, SET_OP, getSerializedProps(expectedProps));
+        String query = String.format(fmtQuery, entityName, SET_OP, getSerializedProps(expectedProps));
         runCommand(query);
-
-        verifyTableProperties(tableName, expectedProps, false);
+        verifyEntityProperties(entityType, entityName, expectedProps, false);
 
         expectedProps.put("testPropKey2", "testPropValue2");
         //Add another property
-        query = String.format(fmtQuery, tableName, SET_OP, getSerializedProps(expectedProps));
+        query = String.format(fmtQuery, entityName, SET_OP, getSerializedProps(expectedProps));
         runCommand(query);
+        verifyEntityProperties(entityType, entityName, expectedProps, false);
 
-        verifyTableProperties(tableName, expectedProps, false);
+        if (entityType != Entity.Type.DATABASE) {
+            //Database unset properties doesnt work strangely - alter database %s unset DBPROPERTIES doesnt work
+            //Unset all the props
+            StringBuilder sb = new StringBuilder("'");
+            query = String.format(fmtQuery, entityName, UNSET_OP, Joiner.on("','").skipNulls().appendTo(sb, expectedProps.keySet()).append('\''));
+            runCommand(query);
 
-        //Unset all the props
-        StringBuilder sb = new StringBuilder("'");
-        query = String.format(fmtQuery, tableName, UNSET_OP, Joiner.on("','").skipNulls().appendTo(sb, expectedProps.keySet()).append('\''));
-        runCommand(query);
-
-        verifyTableProperties(tableName, expectedProps, true);
+            verifyEntityProperties(entityType, entityName, expectedProps, true);
+        }
     }
 
     @Test
@@ -612,13 +633,38 @@ public class HiveHookIT {
         runCommand(query);
 
         final String fmtQuery = "alter view %s %s TBLPROPERTIES (%s)";
-        testAlterProperties(viewName, fmtQuery);
+        testAlterProperties(Entity.Type.TABLE, viewName, fmtQuery);
+    }
+
+    private void verifyEntityProperties(Entity.Type type, String entityName, Map<String, String> expectedProps, boolean checkIfNotExists) throws Exception {
+
+        String entityId = null;
+
+        switch(type) {
+        case TABLE:
+            entityId = assertTableIsRegistered(DEFAULT_DB, entityName);
+            break;
+        case DATABASE:
+            entityId = assertDatabaseIsRegistered(entityName);
+            break;
+        }
+
+        Referenceable ref = dgiCLient.getEntity(entityId);
+        verifyProperties(ref, expectedProps, checkIfNotExists);
     }
 
-    private void verifyTableProperties(String tableName, Map<String, String> expectedProps, boolean checkIfNotExists) throws Exception {
+    private void verifyTableSdProperties(String tableName, String serdeLib, Map<String, String> expectedProps) throws Exception {
         String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
         Referenceable tableRef = dgiCLient.getEntity(tableId);
-        Map<String, String> parameters = (Map<String, String>) tableRef.get(HiveDataModelGenerator.PARAMETERS);
+        Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
+        Struct serdeInfo = (Struct) sdRef.get("serdeInfo");
+        Assert.assertEquals(serdeInfo.get("serializationLib"), serdeLib);
+        verifyProperties(serdeInfo, expectedProps, false);
+    }
+
+    private void verifyProperties(Struct referenceable, Map<String, String> expectedProps, boolean checkIfNotExists) {
+        Map<String, String> parameters = (Map<String, String>) referenceable.get(HiveDataModelGenerator.PARAMETERS);
+
         if (checkIfNotExists == false) {
             //Check if properties exist
             Assert.assertNotNull(parameters);
@@ -635,20 +681,6 @@ public class HiveHookIT {
         }
     }
 
-    private void verifyTableSdProperties(String tableName, String serdeLib, Map<String, String> expectedProps) throws Exception {
-        String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
-        Referenceable tableRef = dgiCLient.getEntity(tableId);
-        Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
-        Struct serdeInfo = (Struct) sdRef.get("serdeInfo");
-        Assert.assertEquals(serdeInfo.get("serializationLib"), serdeLib);
-        Map<String, String> parameters = (Map<String, String>) serdeInfo.get(HiveDataModelGenerator.PARAMETERS);
-        Assert.assertNotNull(parameters);
-        //Comment should exist since SET TBLPOPERTIES only adds properties. Doe not remove existing ones
-        for (String propKey : expectedProps.keySet()) {
-            Assert.assertEquals(parameters.get(propKey), expectedProps.get(propKey));
-        }
-    }
-
     private String assertProcessIsRegistered(String queryStr) throws Exception {
         //        String dslQuery = String.format("%s where queryText = \"%s\"", HiveDataTypes.HIVE_PROCESS.getName(),
         //                normalize(queryStr));

docs/src/site/twiki/Bridge-Hive.twiki

@@ -79,4 +79,4 @@ Refer [[Configuration][Configuration]] for notification related configurations
 
 ---++ Limitations
    * Since database name, table name and column names are case insensitive in hive, the corresponding names in entities are lowercase. So, any search APIs should use lowercase while querying on the entity names
-   * Only the following hive operations are captured by hive hook currently - create database, create table, create view, CTAS, load, import, export, query, alter table rename and alter view rename
+   * Only the following hive operations are captured by hive hook currently - create database, create table, create view, CTAS, load, import, export, query, alter database, alter table(except alter table replace columns and alter table change column position), alter view (except replacing and changing column position)

release-log.txt

@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-524 Support alter database (sumasai via shwethags)
 ATLAS-539 Store for entity audit events (shwethags)
 ATLAS-523 Support alter view (sumasai via shwethags)
 ATLAS-555 Tag creation from UI fails due to missing description attribute (guptaneeru via shwethags)