ATLAS-616 Resolve OOM - Zookeeper throws exceptions when trying to fire DSL…

distro/src/bin/atlas_config.py

@@ -32,15 +32,18 @@ CONF = "conf"
 LOG = "logs"
 WEBAPP = "server" + os.sep + "webapp"
 DATA = "data"
-ENV_KEYS = ["JAVA_HOME", "ATLAS_OPTS", "ATLAS_LOG_DIR", "ATLAS_PID_DIR", "ATLAS_CONF", "ATLASCPPATH", "ATLAS_DATA_DIR", "ATLAS_HOME_DIR", "ATLAS_EXPANDED_WEBAPP_DIR", "HBASE_CONF_DIR"]
 ATLAS_CONF = "ATLAS_CONF"
 ATLAS_LOG = "ATLAS_LOG_DIR"
 ATLAS_PID = "ATLAS_PID_DIR"
 ATLAS_WEBAPP = "ATLAS_EXPANDED_WEBAPP_DIR"
+ATLAS_SERVER_OPTS = "ATLAS_SERVER_OPTS"
 ATLAS_OPTS = "ATLAS_OPTS"
+ATLAS_SERVER_HEAP = "ATLAS_SERVER_HEAP"
 ATLAS_DATA = "ATLAS_DATA_DIR"
 ATLAS_HOME = "ATLAS_HOME_DIR"
 HBASE_CONF_DIR = "HBASE_CONF_DIR"
+ENV_KEYS = ["JAVA_HOME", ATLAS_OPTS, ATLAS_SERVER_OPTS, ATLAS_SERVER_HEAP, ATLAS_LOG, ATLAS_PID, ATLAS_CONF,
+            "ATLASCPPATH", ATLAS_DATA, ATLAS_HOME, ATLAS_WEBAPP, HBASE_CONF_DIR]
 IS_WINDOWS = platform.system() == "Windows"
 ON_POSIX = 'posix' in sys.builtin_module_names
 CONF_FILE="atlas-application.properties"

distro/src/bin/atlas_start.py

@@ -24,7 +24,8 @@ import atlas_config as mc
 ATLAS_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=%s.log"
 ATLAS_COMMAND_OPTS="-Datlas.home=%s"
 ATLAS_CONFIG_OPTS="-Datlas.conf=%s"
-DEFAULT_JVM_OPTS="-Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true"
+DEFAULT_JVM_HEAP_OPTS="-Xmx1024m -XX:MaxPermSize=512m"
+DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true"
 
 def main():
 
@@ -56,8 +57,14 @@ def main():
     config_opts = (ATLAS_CONFIG_OPTS % jvm_confdir)
     jvm_opts_list.extend(config_opts.split())
 
-    default_jvm_opts = DEFAULT_JVM_OPTS
-    atlas_jvm_opts = os.environ.get(mc.ATLAS_OPTS, default_jvm_opts)
+    atlas_server_heap_opts = os.environ.get(mc.ATLAS_SERVER_HEAP, DEFAULT_JVM_HEAP_OPTS)
+    jvm_opts_list.extend(atlas_server_heap_opts.split())
+
+    atlas_server_jvm_opts = os.environ.get(mc.ATLAS_SERVER_OPTS)
+    if atlas_server_jvm_opts:
+        jvm_opts_list.extend(atlas_server_jvm_opts.split())
+
+    atlas_jvm_opts = os.environ.get(mc.ATLAS_OPTS, DEFAULT_JVM_OPTS)
     jvm_opts_list.extend(atlas_jvm_opts.split())
 
     #expand web app dir

distro/src/conf/atlas-env.sh

@@ -30,9 +30,17 @@
 # any additional opts you want to set for atlas service.
 #export ATLAS_SERVER_OPTS=
 
+# indicative values for large number of metadata entities (equal or more than 10,000s)
+#export ATLAS_SERVER_OPTS="-server -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+PrintTenuringDistribution -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dumps/atlas_server.hprof -Xloggc:logs/gc-worker.log -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1m -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCTimeStamps"
+
 # java heap size we want to set for the atlas server. Default is 1024MB
 #export ATLAS_SERVER_HEAP=
 
+# indicative values for large number of metadata entities (equal or more than 10,000s) for JDK 7
+#export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=3072m -XX:PermSize=100M -XX:MaxPermSize=512m"
+# indicative values for large number of metadata entities (equal or more than 10,000s) for JDK 8
+#export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=5120m -XX:MetaspaceSize=100M -XX:MaxMetaspaceSize=512m"
+
 # What is is considered as atlas home dir. Default is the base locaion of the installed software
 #export ATLAS_HOME_DIR=
 

docs/src/site/twiki/InstallationSteps.twiki

@@ -121,6 +121,28 @@ executed. The following environment variables are available to set.
 #export ATLAS_EXPANDED_WEBAPP_DIR=
 </verbatim>
 
+*Settings to support large number of metadata objects*
+
+If you plan to store several tens of thousands of metadata objects, it is recommended that you use values
+tuned for better GC performance of the JVM.
+
+The following values are common server side options:
+<verbatim>
+export ATLAS_SERVER_OPTS="-server -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+PrintTenuringDistribution -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dumps/atlas_server.hprof -Xloggc:logs/gc-worker.log -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1m -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCTimeStamps"
+</verbatim>
+
+The =-XX:SoftRefLRUPolicyMSPerMB= option was found to be particularly helpful to regulate GC performance for
+query heavy workloads with many concurrent users.
+
+The following values are recommended for JDK 7:
+<verbatim>
+export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=3072m -XX:PermSize=100M -XX:MaxPermSize=512m"
+</verbatim>
+
+The following values are recommended for JDK 8:
+<verbatim>
+export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=5120m -XX:MetaspaceSize=100M -XX:MaxMetaspaceSize=512m"
+</verbatim>
 
 *NOTE for Mac OS users*
 If you are using a Mac OS, you will need to configure the ATLAS_SERVER_OPTS (explained above).

release-log.txt

@@ -17,6 +17,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-616 Resolve OOM - Zookeeper throws exceptions when trying to fire DSL queries at Atlas at large scale. (yhemanth via sumasai)
 ATLAS-530 Add table information to column class (sumasai)
 ATLAS-538 Rename table should retain traits/tags assigned to columns/storage descriptors (sumasai)
 ATLAS-628 Starting two Atlas instances at the same time causes exceptions in HA mode (yhemanth via sumasai)