From a4af9e31fd3b534a2ca0717f59bda43b7b2a9353 Mon Sep 17 00:00:00 2001
From: Shwetha GS <sshivalingamurthy@hortonworks.com>
Date: Fri, 24 Apr 2015 13:24:23 +0530
Subject: [PATCH] auto registering hive model in hive hook
---
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java | 14 ++++++++------
addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java | 27 ++++++++++++++++-----------
addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java | 11 -----------
client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java | 18 ++++++++++++++++--
client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java | 7 +++++++
5 files changed, 47 insertions(+), 30 deletions(-)
diff --git a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
index 08f6db8..ab14744 100755
--- a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
@@ -351,13 +351,15 @@ public class HiveMetaStoreBridge {
return fieldsList;
}
- private void registerHiveDataModel() throws Exception {
+ public synchronized void registerHiveDataModel() throws Exception {
HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
- try {
- getMetadataServiceClient().createType(dataModelGenerator.getModelAsJson());
- } catch (Exception e) {
- //Ignore if type is already registered
- //TODO make createType idempotent
+ MetadataServiceClient dgiClient = getMetadataServiceClient();
+
+ //Register hive data model if its not already registered
+ if (dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()) == null ) {
+ dgiClient.createType(dataModelGenerator.getModelAsJson());
+ } else {
+ LOG.debug("Hive data model is already registered!");
}
}
diff --git a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
index 91611e2..abffe2e 100755
--- a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
@@ -38,8 +38,6 @@ package org.apache.hadoop.metadata.hive.hook;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.antlr.runtime.tree.Tree;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -65,6 +63,8 @@ import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
import org.apache.hadoop.metadata.typesystem.Referenceable;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.BufferedWriter;
import java.io.File;
@@ -86,7 +86,8 @@ import java.util.concurrent.TimeUnit;
*/
public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHook {
- private static final Log LOG = LogFactory.getLog(HiveHook.class.getName());
+ private static final Logger LOG = LoggerFactory.getLogger(HiveHook.class);
+
// wait time determines how long we wait before we exit the jvm on
// shutdown. Pending requests after that will not be sent.
private static final int WAIT_TIME = 3;
@@ -99,6 +100,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
private static final int minThreadsDefault = 5;
private static final int maxThreadsDefault = 5;
private static final long keepAliveTimeDefault = 10;
+ private static boolean typesRegistered = false;
static {
// anything shared should be initialized here and destroyed in the
@@ -174,14 +176,17 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
}
private void fireAndForget(HookContext hookContext, HiveConf conf) throws Exception {
- LOG.info("Entered DGI hook for query hook " + hookContext.getHookType());
- if (hookContext.getHookType() != HookContext.HookType.POST_EXEC_HOOK) {
- LOG.debug("No-op for query hook " + hookContext.getHookType());
- }
+ assert hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK : "Non-POST_EXEC_HOOK not supported!";
+
+ HiveOperation operation = HiveOperation.valueOf(hookContext.getOperationName());
+ LOG.info("Entered DGI hook for hook type {} operation {}", hookContext.getHookType(), operation);
HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(conf);
- HiveOperation operation = HiveOperation.valueOf(hookContext.getOperationName());
+ if (!typesRegistered) {
+ dgiBridge.registerHiveDataModel();
+ typesRegistered = true;
+ }
switch (operation) {
case CREATEDATABASE:
@@ -199,7 +204,8 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
if (entity.getType() == Entity.Type.TABLE) {
Table table = entity.getTable();
- //TODO table.getDbName().toLowerCase() is required as hive stores in lowercase, but table.getDbName() is not lowercase
+ //TODO table.getDbName().toLowerCase() is required as hive stores in lowercase,
+ // but table.getDbName() is not lowercase
Referenceable dbReferenceable = getDatabaseReference(dgiBridge, table.getDbName().toLowerCase());
dgiBridge.registerTable(dbReferenceable, table.getDbName(), table.getTableName());
}
@@ -215,8 +221,6 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
}
private void registerCTAS(HiveMetaStoreBridge dgiBridge, HookContext hookContext, HiveConf conf) throws Exception {
- LOG.debug("Registering CTAS");
-
Set<ReadEntity> inputs = hookContext.getInputs();
Set<WriteEntity> outputs = hookContext.getOutputs();
@@ -238,6 +242,7 @@ public class HiveHook implements ExecuteWithHookContext, HiveSemanticAnalyzerHoo
queryStartTime = plan.getQueryStartTime();
}
+ LOG.debug("Registering CTAS query: {}", queryStr);
Referenceable processReferenceable = new Referenceable(HiveDataTypes.HIVE_PROCESS.getName());
processReferenceable.set("processName", operation.getOperationName());
processReferenceable.set("startTime", queryStartTime);
diff --git a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
index bfe2a97..c47bdd5 100755
--- a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
@@ -40,16 +40,6 @@ public class HiveHookIT {
@BeforeClass
public void setUp() throws Exception {
- //Register hive types
- HiveDataModelGenerator hiveModel = new HiveDataModelGenerator();
- String typesAsJson = hiveModel.getModelAsJson();
- MetadataServiceClient dgiClient = new MetadataServiceClient(DGI_URL);
- try {
- dgiClient.createType(typesAsJson);
- } catch (Exception e) {
- //ignore if types are already defined
- }
-
//Set-up hive session
HiveConf conf = getHiveConf();
driver = new Driver(conf);
@@ -100,7 +90,6 @@ public class HiveHookIT {
//Create table where database doesn't exist, will create database instance as well
assertDatabaseIsRegistered("default");
-
}
@Test
diff --git a/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java b/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
index faa6a71..0f8f129 100755
--- a/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
+++ b/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
@@ -108,6 +108,21 @@ public class MetadataServiceClient {
}
}
+ public String getType(String typeName) throws MetadataServiceException {
+ WebResource resource = getResource(API.GET_TYPE, typeName);
+ try {
+ JSONObject response = callAPIWithResource(API.GET_TYPE, resource);
+ return response.getString("definition");
+ } catch (MetadataServiceException e) {
+ if (e.getStatus() == ClientResponse.Status.NOT_FOUND) {
+ return null;
+ }
+ throw e;
+ } catch (JSONException e) {
+ throw new MetadataServiceException(e);
+ }
+ }
+
/**
* Register the given type(meta model)
* @param typeAsJson type definition a jaon
@@ -216,8 +231,7 @@ public class MetadataServiceClient {
return resource;
}
- private JSONObject callAPIWithResource(API api,
- WebResource resource) throws MetadataServiceException {
+ private JSONObject callAPIWithResource(API api, WebResource resource) throws MetadataServiceException {
return callAPIWithResource(api, resource, null);
}
diff --git a/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java b/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java
index adb47eb..882e5bf 100755
--- a/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java
+++ b/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.metadata;
import com.sun.jersey.api.client.ClientResponse;
public class MetadataServiceException extends Exception {
+ private ClientResponse.Status status;
+
public MetadataServiceException(MetadataServiceClient.API api, Exception e) {
super("Metadata service API " + api + " failed", e);
}
@@ -28,9 +30,14 @@ public class MetadataServiceException extends Exception {
public MetadataServiceException(MetadataServiceClient.API api, ClientResponse.Status status) {
super("Metadata service API " + api + " failed with status " + status.getStatusCode()
+ "(" + status.getReasonPhrase() + ")");
+ this.status = status;
}
public MetadataServiceException(Exception e) {
super(e);
}
+
+ public ClientResponse.Status getStatus() {
+ return status;
+ }
}
--
libgit2 0.27.1