ATLAS-4011: Dockerfile: updated to support storing data (solr, berkeleydb, kafka) on host directory

dev-support/atlas-docker/Dockerfile.atlas

@@ -26,8 +26,12 @@ RUN tar xfz /tmp/apache-atlas-${ATLAS_VERSION}-bin.tar.gz --directory=/opt/ && \
     rm -f /tmp/apache-atlas-${ATLAS_VERSION}-bin.tar.gz && \
     mkdir -p /var/run/atlas && \
     mkdir -p /var/log/atlas && \
+    mkdir -p /home/atlas/data && \
     rm -rf ${ATLAS_HOME}/logs && \
     ln -s /var/log/atlas ${ATLAS_HOME}/logs && \
+    ln -s /home/atlas/data ${ATLAS_HOME}/data && \
     chown -R atlas:atlas ${ATLAS_HOME}/ /var/run/atlas/ /var/log/atlas/
 
+VOLUME /home/atlas/data
+
 ENTRYPOINT [ "/home/atlas/scripts/atlas.sh" ]

dev-support/atlas-docker/README.md

@@ -67,9 +67,9 @@ Docker files in this folder create docker images and run them to build Apache At
 
    This might take about 10 minutes to complete.
 
-   5.8. Execute following command to install and run Atlas services in a container:
+   5.5. Execute following command to install and run Atlas services in a container:
 
-        docker run -it -d --name atlas --hostname atlas.example.com -p 21000:21000 atlas
+        docker run -it -d --name atlas --hostname atlas.example.com -p 21000:21000 -v $(pwd)/data:/home/atlas/data atlas
 
    This might take few minutes to complete.
 

dev-support/atlas-docker/data/.gitignore

+*

dev-support/atlas-docker/docker-compose.atlas.yml

@@ -13,6 +13,8 @@ services:
     tty: true
     networks:
       - atlas
+    volumes:
+      - ./data:/home/atlas/data
     ports:
       - "21000:21000"
     depends_on:

dev-support/atlas-docker/scripts/atlas.sh

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -x
 
 if [ ! -e ${ATLAS_HOME}/.setupDone ]
 then
@@ -36,6 +37,7 @@ then
 fi
 
 su -c "cd ${ATLAS_HOME}/bin && ./atlas_start.py" atlas
+ATLAS_PID=`ps -ef  | grep -v grep | grep -i "org.apache.atlas.Atlas" | awk '{print $2}'`
 
 # prevent the container from exiting
-/bin/bash
+tail --pid=$ATLAS_PID -f /dev/null

distro/src/bin/atlas_config.py

@@ -50,7 +50,9 @@ MANAGE_EMBEDDED_CASSANDRA = "MANAGE_EMBEDDED_CASSANDRA"
 MANAGE_LOCAL_ELASTICSEARCH = "MANAGE_LOCAL_ELASTICSEARCH"
 SOLR_BIN = "SOLR_BIN"
 SOLR_CONF = "SOLR_CONF"
+SOLR_HOME = "SOLR_HOME"
 SOLR_PORT = "SOLR_PORT"
+SOLR_DIR = "SOLR_DIR"
 DEFAULT_SOLR_PORT = "9838"
 SOLR_SHARDS = "SOLR_SHARDS"
 DEFAULT_SOLR_SHARDS = "1"
@@ -59,7 +61,7 @@ DEFAULT_SOLR_REPLICATION_FACTOR = "1"
 
 ENV_KEYS = ["JAVA_HOME", ATLAS_OPTS, ATLAS_SERVER_OPTS, ATLAS_SERVER_HEAP, ATLAS_LOG, ATLAS_PID, ATLAS_CONF,
             "ATLASCPPATH", ATLAS_DATA, ATLAS_HOME, ATLAS_WEBAPP, HBASE_CONF_DIR, SOLR_PORT, MANAGE_LOCAL_HBASE,
-            MANAGE_LOCAL_SOLR, MANAGE_EMBEDDED_CASSANDRA, MANAGE_LOCAL_ELASTICSEARCH]
+            MANAGE_LOCAL_SOLR, MANAGE_EMBEDDED_CASSANDRA, MANAGE_LOCAL_ELASTICSEARCH, SOLR_HOME, SOLR_DIR]
 IS_WINDOWS = platform.system() == "Windows"
 ON_POSIX = 'posix' in sys.builtin_module_names
 CONF_FILE="atlas-application.properties"
@@ -107,12 +109,21 @@ def hbaseConfDir(dir):
 def zookeeperBinDir(dir):
     return os.environ.get(SOLR_BIN, os.path.join(dir, "zk", BIN))
 
+def solrDir():
+    return os.environ.get(SOLR_DIR, os.path.join(atlasDir(), "solr"))
+
+def solrServerDir():
+    return os.path.join(solrDir(), "server")
+
 def solrBinDir(dir):
     return os.environ.get(SOLR_BIN, os.path.join(dir, "solr", BIN))
 
 def elasticsearchBinDir(dir):
     return os.environ.get(SOLR_BIN, os.path.join(dir, "elasticsearch", BIN))
 
+def solrHomeDir(dir):
+    return os.environ.get(SOLR_HOME, os.path.join(dir, "data", "solr"))
+
 def solrConfDir(dir):
     return os.environ.get(SOLR_CONF, os.path.join(dir, "solr", CONFIG_SETS_CONF))
 
@@ -565,7 +576,7 @@ def start_elasticsearch(dir, logdir = None, wait=True):
     sleep(6)
     return processVal
 
-def run_solr(dir, action, zk_url = None, port = None, logdir = None, wait=True):
+def run_solr(dir, action, zk_url = None, port = None, logdir = None, wait=True, homedir = None):
 
     solrScript = "solr"
 
@@ -583,6 +594,19 @@ def run_solr(dir, action, zk_url = None, port = None, logdir = None, wait=True):
         else:
             cmd = [os.path.join(dir, solrScript), action, '-z', zk_url, '-p', port]
 
+    if homedir is not None:
+        if not os.path.exists(homedir) :
+            os.makedirs(homedir)
+        #Copy solr.xml from installation directory to the solr home directory
+        srcSolrXmlPath = os.path.join(solrServerDir(), "solr", "solr.xml")
+        destSolrXmlPath = os.path.join(homedir, "solr.xml")
+        if not os.path.exists(destSolrXmlPath) :
+            print "solr.xml doesn't exist in " + homedir + ", copying from " + srcSolrXmlPath
+            copyCmd = ["cp", srcSolrXmlPath, homedir]
+            runProcess(copyCmd, logdir, False, True)
+        cmd.append('-s')
+        cmd.append(homedir)
+
     return runProcess(cmd, logdir, False, wait)
 
 def create_solr_collection(dir, confdir, index, logdir = None, wait=True):

distro/src/bin/atlas_start.py

@@ -128,7 +128,7 @@ def main():
             mc.run_zookeeper(mc.zookeeperBinDir(atlas_home), "start", logdir)
             print "zookeeper started."
 
-        mc.run_solr(mc.solrBinDir(atlas_home), "start", mc.get_solr_zk_url(confdir), mc.solrPort(), logdir)
+        mc.run_solr(mc.solrBinDir(atlas_home), "start", mc.get_solr_zk_url(confdir), mc.solrPort(), logdir, True, mc.solrHomeDir(atlas_home))
         print "solr started."
 
         print "setting up solr collections..."

distro/src/bin/atlas_stop.py

@@ -68,7 +68,7 @@ def main():
 
     # stop solr
     if mc.is_solr_local(confdir):
-        mc.run_solr(mc.solrBinDir(atlas_home), "stop", None, mc.solrPort(), None, True)
+        mc.run_solr(mc.solrBinDir(atlas_home), "stop", None, mc.solrPort(), None, True, mc.solrHomeDir(atlas_home))
 
     if mc.is_zookeeper_local(confdir):
         mc.run_zookeeper(mc.zookeeperBinDir(atlas_home), "stop")

distro/src/test/python/scripts/TestMetadata.py

@@ -106,7 +106,7 @@ class TestMetadata(unittest.TestCase):
             calls = [
                 call(['atlas_home/hbase/bin/hbase-daemon.sh', '--config', 'atlas_home/hbase/conf', 'start', 'master'],
                      'atlas_home/logs', False, True),
-                call(['atlas_home/solr/bin/solr', 'start', '-z', 'localhost:9838', '-p', '9838'], 'atlas_home/logs',
+                call(['atlas_home/solr/bin/solr', 'start', '-z', 'localhost:9838', '-p', '9838', '-s', 'atlas_home/data/solr'], 'atlas_home/logs',
                      False, True),
                 call(['atlas_home/solr/bin/solr', 'create', '-c', 'vertex_index', '-d',
                       'atlas_home/solr/server/solr/configsets/_default/conf', '-shards', '1', '-replicationFactor',