ATLAS-833 Make default build profile for External HBase and Solr (tbeerbower via yhemanth)

distro/pom.xml

@@ -28,9 +28,28 @@
     <artifactId>atlas-distro</artifactId>
     <description>Apache Atlas Distribution</description>
     <name>Apache Atlas Distribution</name>
+
+    <!-- by default configure hbase and solr with the distribution -->
     <properties>
-        <titan.storage.backend>berkeleyje</titan.storage.backend>
-        <titan.index.backend>elasticsearch</titan.index.backend>
+
+        <titan.storage.backend>hbase</titan.storage.backend>
+        <titan.storage.properties>#Hbase
+#For standalone mode , specify localhost
+#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
+atlas.graph.storage.hostname=
+atlas.graph.storage.hbase.regions-per-server=1
+atlas.graph.storage.lock.wait-time=10000
+        </titan.storage.properties>
+        <titan.index.backend>solr5</titan.index.backend>
+        <titan.index.properties>#Solr
+#Solr cloud mode properties
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=
+
+#Solr http mode properties
+#atlas.graph.index.search.solr.mode=http
+#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
+        </titan.index.properties>
         <hbase.embedded>false</hbase.embedded>
         <solr.embedded>false</solr.embedded>
     </properties>
@@ -93,15 +112,59 @@
                 </plugins>
             </build>
         </profile>
-        <!-- embedded_services profile to package and configure embedded hbase and solr with the distribution -->
+        <!-- profile to configure berkeley and elasticsearch with the distribution -->
         <profile>
-            <id>embedded_services</id>
+            <id>berkeley-elasticsearch</id>
             <activation>
                 <activeByDefault>false</activeByDefault>
             </activation>
             <properties>
-                <titan.storage.backend>hbase</titan.storage.backend>
-                <titan.index.backend>solr5</titan.index.backend>
+                <titan.storage.backend>berkeleyje</titan.storage.backend>
+                <titan.storage.properties>#Berkeley
+atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
+                </titan.storage.properties>
+                <titan.index.backend>elasticsearch</titan.index.backend>
+                <titan.index.properties>#ElasticSearch
+atlas.graph.index.search.directory=${sys:atlas.home}/data/es
+atlas.graph.index.search.elasticsearch.client-only=false
+atlas.graph.index.search.elasticsearch.local-mode=true
+atlas.graph.index.search.elasticsearch.create.sleep=2000
+                </titan.index.properties>
+            </properties>
+        </profile>
+
+        <!-- profile to configure berkeley and elasticsearch with the distribution -->
+        <profile>
+            <id>external-hbase-solr</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+        </profile>
+
+        <!-- profile to package and configure embedded hbase and solr with the distribution -->
+        <profile>
+            <id>embedded-hbase-solr</id>
+            <activation>
+                <activeByDefault>false</activeByDefault>
+            </activation>
+            <properties>
+                <titan.storage.properties>#Hbase
+#For standalone mode , specify localhost
+#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
+atlas.graph.storage.hostname=localhost
+atlas.graph.storage.hbase.regions-per-server=1
+atlas.graph.storage.lock.wait-time=10000
+                </titan.storage.properties>
+                <titan.index.properties>#Solr
+#Solr cloud mode properties
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=localhost:2181
+
+#Solr http mode properties
+#atlas.graph.index.search.solr.mode=http
+#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
+                </titan.index.properties>
+
                 <hbase.embedded>true</hbase.embedded>
                 <solr.embedded>true</solr.embedded>
 

distro/src/conf/atlas-application.properties

@@ -19,33 +19,13 @@
 #########  Graph Database Configs  #########
 # Graph Storage
 atlas.graph.storage.backend=${titan.storage.backend}
-atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
 
-#Hbase as storage backend
-#atlas.graph.storage.backend=hbase
-#For standalone mode , specify localhost
-#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
-atlas.graph.storage.hostname=localhost
-atlas.graph.storage.hbase.regions-per-server=1
-atlas.graph.storage.lock.wait-time=10000
+${titan.storage.properties}
 
-#Solr
+# Graph Search Index
 atlas.graph.index.search.backend=${titan.index.backend}
 
-# Solr cloud mode properties
-atlas.graph.index.search.solr.mode=cloud
-atlas.graph.index.search.solr.zookeeper-url=localhost:2181
-
-#Solr http mode properties
-#atlas.graph.index.search.solr.mode=http
-#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
-
-# Graph Search Index
-#ElasticSearch
-atlas.graph.index.search.directory=${sys:atlas.home}/data/es
-atlas.graph.index.search.elasticsearch.client-only=false
-atlas.graph.index.search.elasticsearch.local-mode=true
-atlas.graph.index.search.elasticsearch.create.sleep=2000
+${titan.index.properties}
 
 
 #########  Notification Configs  #########

distro/src/conf/atlas-env.sh

@@ -57,5 +57,8 @@
 # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
 #export ATLAS_EXPANDED_WEBAPP_DIR=
 
+# indicates whether or not a local instance of HBase should be started for Atlas
 export MANAGE_LOCAL_HBASE=${hbase.embedded}
+
+# indicates whether or not a local instance of Solr should be started for Atlas
 export MANAGE_LOCAL_SOLR=${solr.embedded}

docs/src/site/twiki/Configuration.twiki

@@ -53,10 +53,10 @@ Without Ranger, HBase shell can be used to set the permissions.
    echo "grant 'atlas', 'RWXCA', 'titan'" | hbase shell
 </verbatim>
 
-Note that if the embedded_services profile is used then HBase is included in the distribution so that a standalone
-instance of HBase can be started as the default storage backend for the graph repository.  Using the embedded_services
+Note that if the embedded-hbase-solr profile is used then HBase is included in the distribution so that a standalone
+instance of HBase can be started as the default storage backend for the graph repository.  Using the embedded-hbase-solr
 profile will configure Atlas so that HBase instance will be started and stopped along with the Atlas server by default.
-To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
+To use the embedded-hbase-solr profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
 section.
 
 ---+++ Graph Search Index
@@ -80,10 +80,10 @@ Please note that Solr installation in Cloud mode is a prerequisite before config
  atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181
 </verbatim>
 
-Also note that if the embedded_services profile is used then Solr is included in the distribution so that a standalone
-instance of Solr can be started as the default search indexing backend. Using the embedded_services profile will
+Also note that if the embedded-hbase-solr profile is used then Solr is included in the distribution so that a standalone
+instance of Solr can be started as the default search indexing backend. Using the embedded-hbase-solr profile will
 configure Atlas so that the standalone Solr instance will be started and stopped along with the Atlas server by default.
-To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
+To use the embedded-hbase-solr profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
 section.
 
 ---+++ Choosing between Persistence and Indexing Backends

docs/src/site/twiki/InstallationSteps.twiki

@@ -18,15 +18,29 @@ mvn clean package -Pdist
 
 </verbatim>
 
-To build a distribution that packages HBase and Solr, build with the embedded_services profile.
+To build a distribution that configures Atlas for external HBase and Solr, build with the external-hbase-solr profile.
 
 <verbatim>
 
-mvn clean package -Pdist,embedded_services
+mvn clean package -Pdist,external-hbase-solr
 
 </verbatim>
 
-Using the embedded_services profile will configure Atlas so that an HBase instance and a Solr instance will be started
+Note that when the external-hbase-solr profile is used the following steps need to be completed to make Atlas functional.
+   * Configure atlas.graph.storage.hostname (see "Graph persistence engine - HBase" in the [[Configuration][Configuration]] section).
+   * Configure atlas.graph.index.search.solr.zookeeper-url (see "Graph Search Index - Solr" in the [[Configuration][Configuration]] section).
+   * Set HBASE_CONF_DIR to point to a valid HBase config directory (see "Graph persistence engine - HBase" in the [[Configuration][Configuration]] section).
+   * Create the SOLR indices (see "Graph Search Index - Solr" in the [[Configuration][Configuration]] section).
+
+To build a distribution that packages HBase and Solr, build with the embedded-hbase-solr profile.
+
+<verbatim>
+
+mvn clean package -Pdist,embedded-hbase-solr
+
+</verbatim>
+
+Using the embedded-hbase-solr profile will configure Atlas so that an HBase instance and a Solr instance will be started
 and stopped along with the Atlas server by default.
 
 Tar can be found in atlas/distro/target/apache-atlas-${project.version}-bin.tar.gz
@@ -75,7 +89,7 @@ Tar is structured as follows
 
 </verbatim>
 
-Note that if the embedded_services profile is specified for the build then HBase and Solr are included in the
+Note that if the embedded-hbase-solr profile is specified for the build then HBase and Solr are included in the
 distribution.
 
 In this case, a standalone instance of HBase can be started as the default storage backend for the graph repository.

release-log.txt

@@ -22,6 +22,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-833 Make default build profile for External HBase and Solr (tbeerbower via yhemanth)
 ATLAS-841 mvn clean install of Atlas is failing python unit tests (yhemanth)
 ATLAS-834 Handle exceptions from HiveHook executor.submit() (sumasai)
 ATLAS-635 Process showing old entity name where as actual entity is renamed ( svimal2106 via sumasai )