BUG-38679 change hive lineage api's to require the syntax cluster.db.tablename

addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java

@@ -37,7 +37,6 @@ package org.apache.hadoop.metadata.hive.hook;
 
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.QueryPlan;
@@ -248,10 +247,9 @@ public class HiveHook implements ExecuteWithHookContext {
         Referenceable dbReferenceable = dgiBridge.registerDatabase(oldTable.getDbName());
         Referenceable tableReferenceable =
                 dgiBridge.registerTable(dbReferenceable, oldTable.getDbName(), oldTable.getTableName());
-        LOG.info("Updating entity name {}.{} to {}",
-                oldTable.getDbName(), oldTable.getTableName(), newTable.getTableName());
-        dgiBridge.getMetadataServiceClient().updateEntity(tableReferenceable.getId()._getId(), "name",
-                newTable.getTableName().toLowerCase());
+        LOG.info("Updating entity name {}.{} to {}", oldTable.getDbName(), oldTable.getTableName(),
+                newTable.getTableName());
+        dgiBridge.updateTable(tableReferenceable, newTable);
     }
 
     private void handleCreateTable(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {

addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataTypes.java

@@ -27,11 +27,9 @@ public enum HiveDataTypes {
     HIVE_OBJECT_TYPE,
     HIVE_PRINCIPAL_TYPE,
     HIVE_RESOURCE_TYPE,
-    HIVE_FUNCTION_TYPE,
 
     // Structs
     HIVE_SERDE,
-    HIVE_SKEWEDINFO,
     HIVE_ORDER,
     HIVE_RESOURCEURI,
 
@@ -42,7 +40,6 @@ public enum HiveDataTypes {
     HIVE_COLUMN,
     HIVE_PARTITION,
     HIVE_INDEX,
-    HIVE_FUNCTION,
     HIVE_ROLE,
     HIVE_TYPE,
     HIVE_PROCESS,

addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java

@@ -121,8 +121,8 @@ public class HiveHookIT {
 
     private String createTable(boolean partition) throws Exception {
         String tableName = tableName();
-        runCommand("create table " + tableName + "(id int, name string) comment 'table comment' " + (partition ? " partitioned by(dt string)"
-                : ""));
+        runCommand("create table " + tableName + "(id int, name string) comment 'table comment' "
+                + (partition ? " partitioned by(dt string)" : ""));
         return tableName;
     }
 
@@ -304,7 +304,7 @@ public class HiveHookIT {
 
     private String assertTableIsRegistered(String dbName, String tableName, boolean registered) throws Exception {
         LOG.debug("Searching for table {}.{}", dbName, tableName);
-        String query = String.format("%s as t where name = '%s', dbName where name = '%s' and clusterName = '%s'"
+        String query = String.format("%s as t where tableName = '%s', db where name = '%s' and clusterName = '%s'"
                 + " select t", HiveDataTypes.HIVE_TABLE.getName(), tableName.toLowerCase(), dbName.toLowerCase(),
                 CLUSTER_NAME);
         return assertEntityIsRegistered(query, registered);
@@ -321,14 +321,13 @@ public class HiveHookIT {
         String typeName = HiveDataTypes.HIVE_PARTITION.getName();
         String dbType = HiveDataTypes.HIVE_DB.getName();
         String tableType = HiveDataTypes.HIVE_TABLE.getName();
-        String datasetType = MetadataServiceClient.DATA_SET_SUPER_TYPE;
 
         LOG.debug("Searching for partition of {}.{} with values {}", dbName, tableName, value);
         //todo replace with DSL
         String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', ['%s']).as('p')."
-                        + "out('__%s.tableName').has('%s.name', '%s').out('__%s.dbName').has('%s.name', '%s')"
+                        + "out('__%s.table').has('%s.tableName', '%s').out('__%s.db').has('%s.name', '%s')"
                         + ".has('%s.clusterName', '%s').back('p').toList()", typeName, typeName, value, typeName,
-                datasetType, tableName.toLowerCase(), tableType, dbType, dbName.toLowerCase(), dbType, CLUSTER_NAME);
+                tableType, tableName.toLowerCase(), tableType, dbType, dbName.toLowerCase(), dbType, CLUSTER_NAME);
         JSONObject response = dgiCLient.searchByGremlin(gremlinQuery);
         JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
         Assert.assertEquals(results.length(), 1);
@@ -349,4 +348,21 @@ public class HiveHookIT {
             return null;
         }
     }
+
+    @Test(enabled = false)
+    public void testLineage() throws Exception {
+        String table1 = createTable(false);
+
+        String db2 = createDatabase();
+        String table2 = tableName();
+
+        String db3 = createDatabase();
+        String table3 = tableName();
+
+        String query = String.format("create table %s.%s as select * from %s", db2, table2, table1);
+        runCommand(query);
+
+        query = String.format("create table %s.%s as select * from %s.%s", db3, table3, db2, table2);
+        runCommand(query);
+    }
 }

client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java

@@ -64,6 +64,7 @@ public class MetadataServiceClient {
     public static final String URI_ENTITIES = "entities";
     public static final String URI_TRAITS = "traits";
     public static final String URI_SEARCH = "discovery/search";
+    public static final String URI_LINEAGE = "lineage/hive";
 
     public static final String QUERY = "query";
     public static final String QUERY_TYPE = "queryType";
@@ -128,6 +129,8 @@ public class MetadataServiceClient {
         SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET),
         SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET);
 
+        //Lineage operations
+
         private final String method;
         private final String path;
 
@@ -255,6 +258,7 @@ public class MetadataServiceClient {
      * @throws MetadataServiceException
      */
     public JSONArray searchByDSL(String query) throws MetadataServiceException {
+        LOG.debug("DSL query: {}", query);
         WebResource resource = getResource(API.SEARCH_DSL);
         resource = resource.queryParam(QUERY, query);
         JSONObject result = callAPIWithResource(API.SEARCH_DSL, resource);
@@ -272,6 +276,7 @@ public class MetadataServiceClient {
      * @throws MetadataServiceException
      */
     public JSONObject searchByGremlin(String gremlinQuery) throws MetadataServiceException {
+        LOG.debug("Gremlin query: " + gremlinQuery);
         WebResource resource = getResource(API.SEARCH_GREMLIN);
         resource = resource.queryParam(QUERY, gremlinQuery);
         return callAPIWithResource(API.SEARCH_GREMLIN, resource);