ATLAS-1030 Atlas hooks should use properties from atlas-application.properties,…

addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java

@@ -89,7 +89,7 @@ public class FalconHookIT {
             return;
         }
 
-        HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasClient);
+        HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
         hiveMetaStoreBridge.registerHiveDataModel();
 
         FalconDataModelGenerator dataModelGenerator = new FalconDataModelGenerator();

addons/falcon-bridge/src/test/resources/hive-site.xml

@@ -42,11 +42,6 @@
     </property>
 
     <property>
-        <name>atlas.cluster.name</name>
-        <value>test</value>
-    </property>
-
-    <property>
         <name>fs.pfile.impl</name>
         <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
     </property>

addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java

@@ -99,16 +99,16 @@ public class HiveMetaStoreBridge {
      * Construct a HiveMetaStoreBridge.
      * @param hiveConf {@link HiveConf} for Hive component in the cluster
      */
-    public HiveMetaStoreBridge(HiveConf hiveConf) throws Exception {
-        this(hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), null);
+    public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf hiveConf) throws Exception {
+        this(atlasProperties, hiveConf, null);
     }
 
     /**
      * Construct a HiveMetaStoreBridge.
      * @param hiveConf {@link HiveConf} for Hive component in the cluster
      */
-    public HiveMetaStoreBridge(HiveConf hiveConf, AtlasClient atlasClient) throws Exception {
-        this(hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClient);
+    public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf hiveConf, AtlasClient atlasClient) throws Exception {
+        this(atlasProperties.getString(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClient);
     }
 
     AtlasClient getAtlasClient() {
@@ -660,7 +660,7 @@ public class HiveMetaStoreBridge {
             failOnError = true;
         }
 
-        HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasClient);
+        HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClient);
         hiveMetaStoreBridge.registerHiveDataModel();
         hiveMetaStoreBridge.importHiveMetadata(failOnError);
     }

addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java

@@ -200,7 +200,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
 
         LOG.info("Entered Atlas hook for hook type {} operation {}", event.getHookType(), event.getOperation());
 
-        HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(hiveConf);
+        HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(atlasProperties, hiveConf);
 
         switch (event.getOperation()) {
         case CREATEDATABASE:

addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java

@@ -87,7 +87,7 @@ public class HiveHookIT {
     private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(HiveHookIT.class);
 
     private static final String DGI_URL = "http://localhost:21000/";
-    private static final String CLUSTER_NAME = "test";
+    private static final String CLUSTER_NAME = "primary";
     public static final String DEFAULT_DB = "default";
     
     private static final String PART_FILE = "2015-01-01";
@@ -115,7 +115,7 @@ public class HiveHookIT {
         Configuration configuration = ApplicationProperties.get();
         atlasClient = new AtlasClient(configuration.getString(HiveMetaStoreBridge.ATLAS_ENDPOINT, DGI_URL));
 
-        hiveMetaStoreBridge = new HiveMetaStoreBridge(conf, atlasClient);
+        hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClient);
         hiveMetaStoreBridge.registerHiveDataModel();
     }
 
@@ -1624,7 +1624,7 @@ public class HiveHookIT {
         verifyEntityProperties(entityType, entityName, expectedProps, false);
 
         if (entityType != Entity.Type.DATABASE) {
-            //Database unset properties doesnt work strangely - alter database %s unset DBPROPERTIES doesnt work
+            //Database unset properties doesnt work - alter database %s unset DBPROPERTIES doesnt work
             //Unset all the props
             StringBuilder sb = new StringBuilder("'");
             query = String.format(fmtQuery, entityName, UNSET_OP, Joiner.on("','").skipNulls().appendTo(sb, expectedProps.keySet()).append('\''));
@@ -1928,7 +1928,7 @@ public class HiveHookIT {
                     fail("Assertions failed. Failing after waiting for timeout " + timeout + " msecs", e);
                 }
                 LOG.debug("Waiting up to " + (mustEnd - System.currentTimeMillis()) + " msec as assertion failed", e);
-                Thread.sleep(400);
+                Thread.sleep(5000);
             }
         }
     }

addons/hive-bridge/src/test/resources/hive-site.xml

@@ -57,11 +57,6 @@
     </property>
 
     <property>
-        <name>atlas.cluster.name</name>
-        <value>test</value>
-    </property>
-
-    <property>
         <name>fs.pfile.impl</name>
         <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
     </property>

addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java

@@ -163,8 +163,7 @@ public class SqoopHook extends SqoopJobDataPublisher {
     @Override
     public void publish(SqoopJobDataPublisher.Data data) throws Exception {
         Configuration atlasProperties = ApplicationProperties.get();
-        org.apache.hadoop.conf.Configuration sqoopConf = new org.apache.hadoop.conf.Configuration();
-        String clusterName = sqoopConf.get(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
+        String clusterName = atlasProperties.getString(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
 
         Referenceable dbStoreRef = createDBStoreInstance(data);
         Referenceable dbRef = createHiveDatabaseInstance(clusterName, data.getHiveDB());

addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java

@@ -54,7 +54,7 @@ public class SqoopHookIT {
 
     private void registerDataModels(AtlasClient client) throws Exception {
         // Make sure hive model exists
-        HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasClient);
+        HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
         hiveMetaStoreBridge.registerHiveDataModel();
         SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator();
 

addons/sqoop-bridge/src/test/resources/hive-site.xml

@@ -42,11 +42,6 @@
     </property>
 
     <property>
-        <name>atlas.cluster.name</name>
-        <value>test</value>
-    </property>
-
-    <property>
         <name>fs.pfile.impl</name>
         <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
     </property>

addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java

@@ -370,10 +370,6 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
     }
 
     private String getClusterName(Map stormConf) {
-        String clusterName = AtlasConstants.DEFAULT_CLUSTER_NAME;
-        if (stormConf.containsKey(AtlasConstants.CLUSTER_NAME_KEY)) {
-            clusterName = (String)stormConf.get(AtlasConstants.CLUSTER_NAME_KEY);
-        }
-        return clusterName;
+        return atlasProperties.getString(AtlasConstants.CLUSTER_NAME_KEY, AtlasConstants.DEFAULT_CLUSTER_NAME);
     }
 }

release-log.txt

@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
 
 
 ALL CHANGES:
+ATLAS-1027 Atlas hooks should use properties from atlas-application.properties, instead of component's configuration (mneethiraj via sumasai)
 ATLAS-1030 Add instrumentation to measure performance: REST API (mneethiraj via sumasai)
 ATLAS-996 DSL queries with comparsions of many primitive types fail (jnhagelb via shwethags)
 ATLAS-971 UI not displaying results for this query - Eg: "hive_table as t where qualifiedName = 'default.input@cl1' select t" (kevalbhatt18 via shwethags)