Commit 0596c9fc by kevalbhatt

ATLAS-1160 Update Atlas hive hook to read configuration from…

ATLAS-1160 Update Atlas hive hook to read configuration from atlas-application.properties instead of hive-site.xml (mneethiraj via kevalbhatt)
parent 4b2e6ba9
...@@ -97,7 +97,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ...@@ -97,7 +97,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
// wait time determines how long we wait before we exit the jvm on // wait time determines how long we wait before we exit the jvm on
// shutdown. Pending requests after that will not be sent. // shutdown. Pending requests after that will not be sent.
private static final int WAIT_TIME = 3; private static final int WAIT_TIME = 3;
private static ExecutorService executor; private static ExecutorService executor = null;
private static final int minThreadsDefault = 1; private static final int minThreadsDefault = 1;
private static final int maxThreadsDefault = 5; private static final int maxThreadsDefault = 5;
...@@ -110,28 +110,32 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ...@@ -110,28 +110,32 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
try { try {
// initialize the async facility to process hook calls. We don't // initialize the async facility to process hook calls. We don't
// want to do this inline since it adds plenty of overhead for the query. // want to do this inline since it adds plenty of overhead for the query.
int minThreads = atlasProperties.getInt(MIN_THREADS, minThreadsDefault); boolean isSync = atlasProperties.getBoolean(CONF_SYNC, Boolean.FALSE);
int maxThreads = atlasProperties.getInt(MAX_THREADS, maxThreadsDefault);
long keepAliveTime = atlasProperties.getLong(KEEP_ALIVE_TIME, keepAliveTimeDefault); if(!isSync) {
int queueSize = atlasProperties.getInt(QUEUE_SIZE, queueSizeDefault); int minThreads = atlasProperties.getInt(MIN_THREADS, minThreadsDefault);
int maxThreads = atlasProperties.getInt(MAX_THREADS, maxThreadsDefault);
executor = new ThreadPoolExecutor(minThreads, maxThreads, keepAliveTime, TimeUnit.MILLISECONDS, long keepAliveTime = atlasProperties.getLong(KEEP_ALIVE_TIME, keepAliveTimeDefault);
new LinkedBlockingQueue<Runnable>(queueSize), int queueSize = atlasProperties.getInt(QUEUE_SIZE, queueSizeDefault);
new ThreadFactoryBuilder().setNameFormat("Atlas Logger %d").build());
executor = new ThreadPoolExecutor(minThreads, maxThreads, keepAliveTime, TimeUnit.MILLISECONDS,
Runtime.getRuntime().addShutdownHook(new Thread() { new LinkedBlockingQueue<Runnable>(queueSize),
@Override new ThreadFactoryBuilder().setNameFormat("Atlas Logger %d").build());
public void run() {
try { Runtime.getRuntime().addShutdownHook(new Thread() {
executor.shutdown(); @Override
executor.awaitTermination(WAIT_TIME, TimeUnit.SECONDS); public void run() {
executor = null; try {
} catch (InterruptedException ie) { executor.shutdown();
LOG.info("Interrupt received in shutdown."); executor.awaitTermination(WAIT_TIME, TimeUnit.SECONDS);
executor = null;
} catch (InterruptedException ie) {
LOG.info("Interrupt received in shutdown.");
}
// shutdown client
} }
// shutdown client });
} }
});
setupOperationMap(); setupOperationMap();
} catch (Exception e) { } catch (Exception e) {
...@@ -174,8 +178,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { ...@@ -174,8 +178,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
event.setQueryStartTime(hookContext.getQueryPlan().getQueryStartTime()); event.setQueryStartTime(hookContext.getQueryPlan().getQueryStartTime());
event.setQueryType(hookContext.getQueryPlan().getQueryPlan().getQueryType()); event.setQueryType(hookContext.getQueryPlan().getQueryPlan().getQueryType());
boolean sync = conf.get(CONF_SYNC, "false").equals("true"); if (executor == null) {
if (sync) {
fireAndForget(event); fireAndForget(event);
} else { } else {
executor.submit(new Runnable() { executor.submit(new Runnable() {
......
...@@ -10,6 +10,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ...@@ -10,6 +10,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES: ALL CHANGES:
ATLAS-1160 Update Atlas hive hook to read configuration from atlas-application.properties instead of hive-site.xml (mneethiraj via kevalbhatt)
ATLAS-1154 Errors in Eclipse with web.xml (davidrad via dkantor) ATLAS-1154 Errors in Eclipse with web.xml (davidrad via dkantor)
ATLAS-1147 UI: column name doesn't show up in schema tab for hive table (Kalyanikashikar via kevalbhatt) ATLAS-1147 UI: column name doesn't show up in schema tab for hive table (Kalyanikashikar via kevalbhatt)
ATLAS-772 Ordering of columns is not maintained in schema query response, where as hive table entity response maintains the ordering (sarath.kum4r@gmail.com via kevalbhatt) ATLAS-772 Ordering of columns is not maintained in schema query response, where as hive table entity response maintains the ordering (sarath.kum4r@gmail.com via kevalbhatt)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment