ATLAS-659 atlas_start fails on Windows (dkantor via shwethags)

distro/src/bin/atlas_config.py

@@ -326,13 +326,14 @@ def exist_pid(pid):
     elif IS_WINDOWS:
         #The os.kill approach does not work on Windows with python 2.7
         #the output from tasklist command is searched for the process id
-        command='tasklist /fi  "pid eq '+ pid + '"'
+        pidStr = str(pid)
+        command='tasklist /fi  "pid eq %s"' % pidStr
         sub_process=subprocess.Popen(command, stdout = subprocess.PIPE, shell=False)
         sub_process.communicate()
         output = subprocess.check_output(command)
         output=split(" *",output)
         for line in output:
-            if pid in line:
+            if pidStr in line:
                 return True
         return False
     #os other than nt or posix - not supported - need to delete the file to restart server if pid no longer exist
@@ -359,11 +360,23 @@ def is_hbase_local(confdir):
     confdir = os.path.join(confdir, CONF_FILE)
     return grep(confdir, HBASE_STORAGE_CONF_ENTRY) is not None and grep(confdir, HBASE_STORAGE_LOCAL_CONF_ENTRY) is not None
 
-def run_hbase(dir, action, hbase_conf_dir = None, logdir = None, wait=True):
-    if hbase_conf_dir is not None:
-        cmd = [os.path.join(dir, "hbase-daemon.sh"), '--config', hbase_conf_dir, action, 'master']
+def run_hbase_action(dir, action, hbase_conf_dir = None, logdir = None, wait=True):
+    if IS_WINDOWS:
+        if action == 'start':
+            hbaseScript = 'start-hbase.cmd'
+        else:
+            hbaseScript = 'stop-hbase.cmd'
+        if hbase_conf_dir is not None:
+            cmd = [os.path.join(dir, hbaseScript), '--config', hbase_conf_dir]
+        else:
+            cmd = [os.path.join(dir, hbaseScript)]
     else:
-        cmd = [os.path.join(dir, "hbase-daemon.sh"), action, 'master']
+        hbaseScript = 'hbase-daemon.sh'
+        if hbase_conf_dir is not None:
+            cmd = [os.path.join(dir, hbaseScript), '--config', hbase_conf_dir, action, 'master']
+        else:
+            cmd = [os.path.join(dir, hbaseScript), action, 'master']
+
 
     return runProcess(cmd, logdir, False, wait)
 
@@ -376,6 +389,11 @@ def configure_hbase(dir):
         hbase_conf_file = "hbase-site.xml"
 
         tmpl_file = os.path.join(tmpl_dir, hbase_conf_file + ".template")
+        if IS_WINDOWS:
+             url_prefix="file:///"
+        else:
+             url_prefix="file://"
+
         conf_file = os.path.join(conf_dir, hbase_conf_file)
         if os.path.exists(tmpl_file):
 
@@ -385,6 +403,7 @@ def configure_hbase(dir):
             f.close()
 
             config = template.replace("${hbase_home}", dir)
+            config = config.replace("${url_prefix}", url_prefix)
 
             f = open(conf_file,'w')
             f.write(config)

distro/src/bin/atlas_start.py

@@ -74,12 +74,6 @@ def main():
     #add hbase-site.xml to classpath
     hbase_conf_dir = mc.hbaseConfDir(atlas_home)
 
-    if mc.is_hbase_local(confdir):
-        print "configured for local hbase."
-        mc.configure_hbase(atlas_home)
-        mc.run_hbase(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir)
-        print "hbase started."
-
     p = os.pathsep
     atlas_classpath = confdir + p \
                        + os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \
@@ -111,6 +105,12 @@ def main():
        else:
            mc.server_pid_not_running(pid)
 
+    if mc.is_hbase_local(confdir):
+        print "configured for local hbase."
+        mc.configure_hbase(atlas_home)
+        mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir)
+        print "hbase started."
+
     web_app_path = os.path.join(web_app_dir, "atlas")
     if (mc.isCygwin()):
         web_app_path = mc.convertCygwinPath(web_app_path)

distro/src/bin/atlas_stop.py

@@ -56,7 +56,7 @@ def main():
 
     # stop hbase
     if mc.is_hbase_local(confdir):
-        mc.run_hbase(mc.hbaseBinDir(atlas_home), "stop", None, None, True)
+        mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "stop", None, None, True)
 
 if __name__ == '__main__':
     try:

distro/src/conf/hbase/hbase-site.xml.template

@@ -19,7 +19,7 @@
 <configuration>
   <property>
     <name>hbase.rootdir</name>
-    <value>file://${hbase_home}/root</value>
+    <value>${url_prefix}${hbase_home}/root</value>
   </property>
   <property>
     <name>hbase.zookeeper.property.dataDir</name>

distro/src/test/python/scripts/TestMetadata.py

@@ -53,7 +53,10 @@ class TestMetadata(unittest.TestCase):
 
     atlas.main()
     self.assertTrue(configure_hbase_mock.called)
-    runProcess_mock.assert_called_with(['atlas_home/hbase/bin/hbase-daemon.sh', '--config', 'atlas_home/hbase/conf', 'start', 'master'], 'atlas_home/logs', False, True)
+    if IS_WINDOWS:
+      runProcess_mock.assert_called_with(['atlas_home\\hbase\\bin\\start-hbase.cmd', '--config', 'atlas_home\\hbase\\conf'], 'atlas_home\\logs', False, True)
+    else:
+      runProcess_mock.assert_called_with(['atlas_home/hbase/bin/hbase-daemon.sh', '--config', 'atlas_home/hbase/conf', 'start', 'master'], 'atlas_home/logs', False, True)
     self.assertTrue(java_mock.called)
     if IS_WINDOWS:
       

release-log.txt

@@ -18,6 +18,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-659 atlas_start fails on Windows (dkantor via shwethags)
 ATLAS-732 Dashboard v2 build fails on Windows (vmadugun via yhemanth)
 ATLAS-602 Hooks stuck in case of failure (svimal2106 via shwethags)
 ATLAS-631 Introduce Versioning to Atlas Notification Payload (tbeerbower via shwethags)