auto registering hive model in hive hook

addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java

@@ -351,13 +351,15 @@ public class HiveMetaStoreBridge {
         return fieldsList;
     }
 
-    private void registerHiveDataModel() throws Exception {
+    public synchronized void registerHiveDataModel() throws Exception {
         HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
-        try {
-            getMetadataServiceClient().createType(dataModelGenerator.getModelAsJson());
-        } catch (Exception e) {
-            //Ignore if type is already registered
-            //TODO make createType idempotent
+        MetadataServiceClient dgiClient = getMetadataServiceClient();
+
+        //Register hive data model if its not already registered
+        if (dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()) == null ) {
+            dgiClient.createType(dataModelGenerator.getModelAsJson());
+        } else {
+            LOG.debug("Hive data model is already registered!");
         }
     }
 

addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java

@@ -38,8 +38,6 @@ package org.apache.hadoop.metadata.hive.hook;
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.antlr.runtime.tree.Tree;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -65,6 +63,8 @@ import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
 import org.apache.hadoop.metadata.typesystem.Referenceable;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -86,7 +86,8 @@ import java.util.concurrent.TimeUnit;
  */
 public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHook {
 
-    private static final Log LOG = LogFactory.getLog(HiveHook.class.getName());
+    private static final Logger LOG = LoggerFactory.getLogger(HiveHook.class);
+
     // wait time determines how long we wait before we exit the jvm on
     // shutdown. Pending requests after that will not be sent.
     private static final int WAIT_TIME = 3;
@@ -99,6 +100,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
     private static final int minThreadsDefault = 5;
     private static final int maxThreadsDefault = 5;
     private static final long keepAliveTimeDefault = 10;
+    private static boolean typesRegistered = false;
 
     static {
         // anything shared should be initialized here and destroyed in the
@@ -174,14 +176,17 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
     }
 
     private void fireAndForget(HookContext hookContext, HiveConf conf) throws Exception {
-        LOG.info("Entered DGI hook for query hook " + hookContext.getHookType());
-        if (hookContext.getHookType() != HookContext.HookType.POST_EXEC_HOOK) {
-            LOG.debug("No-op for query hook " + hookContext.getHookType());
-        }
+        assert hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK : "Non-POST_EXEC_HOOK not supported!";
+
+        HiveOperation operation = HiveOperation.valueOf(hookContext.getOperationName());
+        LOG.info("Entered DGI hook for hook type {} operation {}", hookContext.getHookType(), operation);
 
         HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(conf);
 
-        HiveOperation operation = HiveOperation.valueOf(hookContext.getOperationName());
+        if (!typesRegistered) {
+            dgiBridge.registerHiveDataModel();
+            typesRegistered = true;
+        }
 
         switch (operation) {
             case CREATEDATABASE:
@@ -199,7 +204,8 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
                     if (entity.getType() == Entity.Type.TABLE) {
 
                         Table table = entity.getTable();
-                        //TODO table.getDbName().toLowerCase() is required as hive stores in lowercase, but table.getDbName() is not lowercase
+                        //TODO table.getDbName().toLowerCase() is required as hive stores in lowercase,
+                        // but table.getDbName() is not lowercase
                         Referenceable dbReferenceable = getDatabaseReference(dgiBridge, table.getDbName().toLowerCase());
                         dgiBridge.registerTable(dbReferenceable, table.getDbName(), table.getTableName());
                     }
@@ -215,8 +221,6 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
     }
 
     private void registerCTAS(HiveMetaStoreBridge dgiBridge, HookContext hookContext, HiveConf conf) throws Exception {
-        LOG.debug("Registering CTAS");
-
         Set<ReadEntity> inputs = hookContext.getInputs();
         Set<WriteEntity> outputs = hookContext.getOutputs();
 
@@ -238,6 +242,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
             queryStartTime = plan.getQueryStartTime();
         }
 
+        LOG.debug("Registering CTAS query: {}", queryStr);
         Referenceable processReferenceable = new Referenceable(HiveDataTypes.HIVE_PROCESS.getName());
         processReferenceable.set("processName", operation.getOperationName());
         processReferenceable.set("startTime", queryStartTime);

addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java

@@ -40,16 +40,6 @@ public class HiveHookIT {
 
     @BeforeClass
     public void setUp() throws Exception {
-        //Register hive types
-        HiveDataModelGenerator hiveModel = new HiveDataModelGenerator();
-        String typesAsJson = hiveModel.getModelAsJson();
-        MetadataServiceClient dgiClient = new MetadataServiceClient(DGI_URL);
-        try {
-            dgiClient.createType(typesAsJson);
-        } catch (Exception e) {
-            //ignore if types are already defined
-        }
-
         //Set-up hive session
         HiveConf conf = getHiveConf();
         driver = new Driver(conf);
@@ -100,7 +90,6 @@ public class HiveHookIT {
 
         //Create table where database doesn't exist, will create database instance as well
         assertDatabaseIsRegistered("default");
-
     }
 
     @Test

client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java

@@ -108,6 +108,21 @@ public class MetadataServiceClient {
         }
     }
 
+    public String getType(String typeName) throws MetadataServiceException {
+        WebResource resource = getResource(API.GET_TYPE, typeName);
+        try {
+            JSONObject response = callAPIWithResource(API.GET_TYPE, resource);
+            return response.getString("definition");
+        } catch (MetadataServiceException e) {
+            if (e.getStatus() == ClientResponse.Status.NOT_FOUND) {
+                return null;
+            }
+            throw e;
+        } catch (JSONException e) {
+            throw new MetadataServiceException(e);
+        }
+    }
+
     /**
      * Register the given type(meta model)
      * @param typeAsJson type definition a jaon
@@ -216,8 +231,7 @@ public class MetadataServiceClient {
         return resource;
     }
 
-    private JSONObject callAPIWithResource(API api,
-                                           WebResource resource) throws MetadataServiceException {
+    private JSONObject callAPIWithResource(API api, WebResource resource) throws MetadataServiceException {
         return callAPIWithResource(api, resource, null);
     }
 

client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java

@@ -21,6 +21,8 @@ package org.apache.hadoop.metadata;
 import com.sun.jersey.api.client.ClientResponse;
 
 public class MetadataServiceException extends Exception {
+    private ClientResponse.Status status;
+
     public MetadataServiceException(MetadataServiceClient.API api, Exception e) {
         super("Metadata service API " + api + " failed", e);
     }
@@ -28,9 +30,14 @@ public class MetadataServiceException extends Exception {
     public MetadataServiceException(MetadataServiceClient.API api, ClientResponse.Status status) {
         super("Metadata service API " + api + " failed with status " + status.getStatusCode()
                 + "(" + status.getReasonPhrase() + ")");
+        this.status = status;
     }
 
     public MetadataServiceException(Exception e) {
         super(e);
     }
+
+    public ClientResponse.Status getStatus() {
+        return status;
+    }
 }