ATLAS-3934: docker scripts to build and run Apache Atlas in containers

dev-support/atlas-docker/.dockerignore

+*
+!config
+!dist/apache-atlas-*-bin.tar.gz
+!scripts/*

dev-support/atlas-docker/Dockerfile.atlas

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM atlas-base:latest
+
+
+COPY ./scripts/atlas.sh                              ${ATLAS_SCRIPTS}/
+COPY ./dist/apache-atlas-${ATLAS_VERSION}-bin.tar.gz /tmp/
+
+RUN tar xfz /tmp/apache-atlas-${ATLAS_VERSION}-bin.tar.gz --directory=/opt/ && \
+    ln -s /opt/apache-atlas-${ATLAS_VERSION} ${ATLAS_HOME} && \
+    rm -f /tmp/apache-atlas-${ATLAS_VERSION}-bin.tar.gz && \
+    mkdir -p /var/run/atlas && \
+    mkdir -p /var/log/atlas && \
+    chown -R atlas:atlas ${ATLAS_HOME}/ /var/run/atlas/ /var/log/atlas/
+
+ENTRYPOINT [ "/home/atlas/scripts/atlas.sh" ]

dev-support/atlas-docker/Dockerfile.atlas-base

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:20.04
+
+
+ENV ATLAS_VERSION 3.0.0-SNAPSHOT
+
+# Install curl, wget, tzdata, Python, Java
+RUN apt-get update && \
+    DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \
+    python python3 python3-pip openjdk-8-jdk bc iputils-ping ssh pdsh
+
+# Set environment variables
+ENV JAVA_HOME      /usr/lib/jvm/java-8-openjdk-amd64
+ENV ATLAS_DIST    /home/atlas/dist
+ENV ATLAS_HOME    /opt/atlas
+ENV ATLAS_SCRIPTS /home/atlas/scripts
+ENV PATH          /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+
+# setup groups, users, directories
+RUN groupadd atlas && \
+    useradd -g atlas -ms /bin/bash atlas && \
+    mkdir -p /home/atlas/dist && \
+    mkdir -p /home/atlas/scripts && \
+    chown -R atlas:atlas /home/atlas
+
+ENTRYPOINT [ "/bin/bash" ]

dev-support/atlas-docker/Dockerfile.atlas-build

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM atlas-base:latest
+
+
+# Install necessary packages to build Atlas
+RUN apt-get update && apt-get -y install git maven
+
+# Set environment variables
+ENV MAVEN_HOME /usr/share/maven
+ENV PATH       /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/apache-maven/bin
+
+# setup atlas group, and users
+RUN mkdir -p /home/atlas/git && \
+    mkdir -p /home/atlas/.m2 && \
+    mkdir -p /home/atlas/scripts && \
+    mkdir -p /home/atlas/patches && \
+    mkdir -p /home/atlas/dist && \
+	chown -R atlas:atlas /home/atlas
+
+COPY ./scripts/atlas-build.sh /home/atlas/scripts/
+
+VOLUME /home/atlas/.m2
+VOLUME /home/atlas/scripts
+VOLUME /home/atlas/patches
+VOLUME /home/atlas/dist
+VOLUME /home/atlas/src
+
+USER atlas
+
+WORKDIR /home/atlas/git
+
+RUN git clone https://github.com/apache/atlas.git
+
+ENTRYPOINT [ "/home/atlas/scripts/atlas-build.sh" ]

dev-support/atlas-docker/README.md

@@ -17,31 +17,62 @@ specific language governing permissions and limitations
 under the License.
 -->
 
-# atlas_docker
-
 ## Overview
 
-The Dockerfile in this folder can be used to build a Docker image running
-the latest Atlas master branch in standalone mode. It does this by setting
-up necessary dependencies, checking out the master branch of Atlas from
-GitHub, and then building Atlas. By default, this image will start the Atlas
-on port 21000.
+Docker files in this folder create docker images and run them to build Apache Atlas, deploy Apache Atlas and dependent services in containers.
 
 ## Usage
 
+1. Ensure that you have recent version of Docker installed from [docker.io](http://www.docker.io) (as of this writing: Engine 19.03, Compose 1.26.2).
 
-1. Ensure that you have a recent version of Docker installed from
-   [docker.io](http://www.docker.io).
 2. Set this folder as your working directory.
-3. Type `docker build -t atlas_docker .` to build a Docker image called **atlas_docker**.
-   This may take 20 minutes or more the first time you run the command since it will
-   create a Maven repository inside the image as well as checkout the master branch
-   of Atlas. Note that by default unit tests are skipped, to run unit tests within
-   the image remove the '-DskipTests' in the Dockerfile
-4. When this completes successfully, you can run `docker run -it -p 21000:21000 atlas_docker`
-   to access an Atlas server running inside of a container created from the
-   **atlas_docker** image. Alternatively, you can type `docker run -it atlas_docker
-   bash` to start a container without a running Atlas. Within this environment,
-   Atlas is built in /root/atlas-bin.
-5. When you run command `docker run -it -p 21000:21000 atlas_docker` successfully, you can
-   access Atlas UI from a browser at http://localhost:21000.
+
+3. Using docker-compose is the simpler way to build and deploy Apache Atlas in containers.
+
+   3.1. Execute following command to build Apache Atlas:
+
+        docker-compose -f docker-compose.atlas-base.yml -f docker-compose.atlas-build.yml up
+
+   Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
+
+   3.2. Execute following command to install and start Atlas in a container:
+
+        docker-compose -f docker-compose.atlas-base.yml -f docker-compose.atlas.yml up -d
+
+   Apache Atlas will be installed at /opt/atlas/, and logs are at /opt/atlas/logs directory.
+
+4. Alternatively docker command can be used to build and deploy Apache Atlas.
+
+   4.1. Execute following command to build Docker image **atlas-base**:
+
+        docker build -f Dockerfile.atlas-base -t atlas-base .
+
+   This might take about 10 minutes to complete.
+
+   4.2. Execute following command to build Docker image **atlas-build**:
+
+        docker build -f Dockerfile.atlas-build -t atlas-build .
+
+   4.3. Build Apache Atlas in a container with one of the following commands:
+
+        # to build from a specific branch
+        docker run -it --rm -v ${HOME}/.m2:/home/atlas/.m2 -v $(pwd)/scripts:/home/atlas/scripts -v $(pwd)/patches:/home/atlas/patches -v $(pwd)/dist:/home/atlas/dist -e BRANCH=master -e PROFILE=dist,berkeley-solr -e SKIPTESTS=true atlas-build
+
+        # to build from local sources
+        docker run -it --rm -v ${HOME}/.m2:/home/atlas/.m2 -v $(pwd)/scripts:/home/atlas/scripts -v $(pwd)/../..:/home/atlas/src -v $(pwd)/dist:/home/atlas/dist -e PROFILE=dist,berkeley-solr -e SKIPTESTS=true atlas-build
+
+   Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
+
+   4.4. Execute following command to build Docker image **atlas**:
+
+        docker build -f Dockerfile.atlas -t atlas .
+
+   This might take about 10 minutes to complete.
+
+   4.8. Execute following command to install and run Atlas services in a container:
+
+        docker run -it -d --name atlas --hostname atlas.example.com -p 21000:21000 atlas
+
+   This might take few minutes to complete.
+
+5. Atlas Admin can be accessed at http://localhost:21000 (admin/atlasR0cks!)

dev-support/atlas-docker/dist/.gitignore

+*

dev-support/atlas-docker/docker-compose.atlas-base.yml

+version: '3'
+services:
+  atlas-base:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-base
+    image: atlas-base
+    networks:
+      - atlas
+
+networks:
+  atlas:

dev-support/atlas-docker/docker-compose.atlas-build.yml

+version: '3'
+services:
+  atlas-build:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas-build
+    image: atlas-build
+    container_name: atlas-build
+    hostname: atlas-build.example.com
+    stdin_open: true
+    tty: true
+    networks:
+      - atlas
+    volumes:
+      - ~/.m2:/home/atlas/.m2
+      - ./scripts:/home/atlas/scripts
+      - ./patches:/home/atlas/patches
+      - ./dist:/home/atlas/dist
+    depends_on:
+      - atlas-base
+    environment:
+      BRANCH: 'master'
+      SKIPTESTS: 'true'
+      PROFILE: 'dist,berkeley-solr'
+
+networks:
+  atlas:

dev-support/atlas-docker/docker-compose.atlas.yml

+version: '3'
+services:
+  atlas:
+    build:
+      context: .
+      dockerfile: Dockerfile.atlas
+    image: atlas:latest
+    container_name: atlas
+    hostname: atlas.example.com
+    stdin_open: true
+    tty: true
+    networks:
+      - atlas
+    ports:
+      - "21000:21000"
+    depends_on:
+      - atlas-base
+    command:
+      - /home/atlas/scripts/atlas.sh
+
+networks:
+  atlas:

dev-support/atlas-docker/patches/.gitignore

+*

dev-support/atlas-docker/scripts/atlas-build.sh

+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "${BRANCH}" == "" ]
+then
+  BRANCH=master
+fi
+
+if [ "${PROFILE}" != "" ]
+then
+  ARG_PROFILES="-P${PROFILE}"
+fi
+
+if [ "${SKIPTESTS}" == "" ]
+then
+  ARG_SKIPTESTS="-DskipTests"
+else
+  ARG_SKIPTESTS="-DskipTests=${SKIPTESTS}"
+fi
+
+export MAVEN_OPTS="-Xms2g -Xmx2g"
+export M2=/home/atlas/.m2
+
+
+if [ -f /home/atlas/src/pom.xml ]
+then
+  echo "Building from /home/atlas/src"
+
+  cd /home/atlas/src
+else
+  echo "Building from /home/atlas/git/atlas"
+
+  cd /home/atlas/git/atlas
+
+  git checkout ${BRANCH}
+  git pull
+
+  for patch in `ls -1 /home/atlas/patches | sort`
+  do
+    echo "applying patch /home/atlas/patches/${patch}"
+    git apply /home/atlas/patches/${patch}
+  done
+fi
+
+mvn ${ARG_PROFILES} ${ARG_SKIPTESTS} -DskipDocs clean package
+
+mv -f distro/target/apache-atlas-${ATLAS_VERSION}-bin.tar.gz /home/atlas/dist/

dev-support/atlas-docker/scripts/atlas.sh

+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+if [ ! -e ${ATLAS_HOME}/.setupDone ]
+then
+  SETUP_ATLAS=true
+else
+  SETUP_ATLAS=false
+fi
+
+if [ "${SETUP_ATLAS}" == "true" ]
+then
+  encryptedPwd=$(${ATLAS_HOME}/bin/cputil.py -g -u admin -p atlasR0cks! -s)
+
+  echo "admin=ADMIN::${encryptedPwd}" > ${ATLAS_HOME}/conf/users-credentials.properties
+
+  chown -R atlas:atlas ${ATLAS_HOME}/
+
+  touch ${ATLAS_HOME}/.setupDone
+fi
+
+su -c "cd ${ATLAS_HOME}/bin && ./atlas_start.py" atlas
+
+# prevent the container from exiting
+/bin/bash