Skip to content

A
atlas
Commit 208e90b1 by Hemanth Yamijala
Options

ATLAS-833 Make default build profile for External HBase and Solr (tbeerbower via yhemanth)

parent 42d5e37e
Showing with 103 additions and 42 deletions
distro/pom.xml
......@@ -28,9 +28,28 @@
<artifactId>atlas-distro</artifactId>
<description>Apache Atlas Distribution</description>
<name>Apache Atlas Distribution</name>
<!-- by default configure hbase and solr with the distribution -->
<properties>
<titan.storage.backend>berkeleyje</titan.storage.backend>
<titan.index.backend>elasticsearch</titan.index.backend>
<titan.storage.backend>hbase</titan.storage.backend>
<titan.storage.properties>#Hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=
atlas.graph.storage.hbase.regions-per-server=1
atlas.graph.storage.lock.wait-time=10000
</titan.storage.properties>
<titan.index.backend>solr5</titan.index.backend>
<titan.index.properties>#Solr
#Solr cloud mode properties
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
</titan.index.properties>
<hbase.embedded>false</hbase.embedded>
<solr.embedded>false</solr.embedded>
</properties>
......@@ -93,15 +112,59 @@
</plugins>
</build>
</profile>
<!-- embedded_services profile to package and configure embedded hbase and solr with the distribution -->
<!-- profile to configure berkeley and elasticsearch with the distribution -->
<profile>
<id>berkeley-elasticsearch</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<titan.storage.backend>berkeleyje</titan.storage.backend>
<titan.storage.properties>#Berkeley
atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
</titan.storage.properties>
<titan.index.backend>elasticsearch</titan.index.backend>
<titan.index.properties>#ElasticSearch
atlas.graph.index.search.directory=${sys:atlas.home}/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
</titan.index.properties>
</properties>
</profile>
<!-- profile to configure berkeley and elasticsearch with the distribution -->
<profile>
<id>embedded_services</id>
<id>external-hbase-solr</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
</profile>
<!-- profile to package and configure embedded hbase and solr with the distribution -->
<profile>
<id>embedded-hbase-solr</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<titan.storage.backend>hbase</titan.storage.backend>
<titan.index.backend>solr5</titan.index.backend>
<titan.storage.properties>#Hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=localhost
atlas.graph.storage.hbase.regions-per-server=1
atlas.graph.storage.lock.wait-time=10000
</titan.storage.properties>
<titan.index.properties>#Solr
#Solr cloud mode properties
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=localhost:2181
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
</titan.index.properties>
<hbase.embedded>true</hbase.embedded>
<solr.embedded>true</solr.embedded>
......@@ -193,9 +256,9 @@
<resource>
<directory>src/bin</directory>
<filtering>true</filtering>
<includes>
<include>**/*.py</include>
</includes>
<includes>
<include>**/*.py</include>
</includes>
<targetPath>bin</targetPath>
</resource>
<resource>
......
distro/src/conf/atlas-application.properties
......@@ -19,33 +19,13 @@
######### Graph Database Configs #########
# Graph Storage
atlas.graph.storage.backend=${titan.storage.backend}
atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
#Hbase as storage backend
#atlas.graph.storage.backend=hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=localhost
atlas.graph.storage.hbase.regions-per-server=1
atlas.graph.storage.lock.wait-time=10000
${titan.storage.properties}
#Solr
# Graph Search Index
atlas.graph.index.search.backend=${titan.index.backend}
# Solr cloud mode properties
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=localhost:2181
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
# Graph Search Index
#ElasticSearch
atlas.graph.index.search.directory=${sys:atlas.home}/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
${titan.index.properties}
######### Notification Configs #########
......
distro/src/conf/atlas-env.sh
......@@ -57,5 +57,8 @@
# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
#export ATLAS_EXPANDED_WEBAPP_DIR=
# indicates whether or not a local instance of HBase should be started for Atlas
export MANAGE_LOCAL_HBASE=${hbase.embedded}
# indicates whether or not a local instance of Solr should be started for Atlas
export MANAGE_LOCAL_SOLR=${solr.embedded}
docs/src/site/twiki/Configuration.twiki
......@@ -53,10 +53,10 @@ Without Ranger, HBase shell can be used to set the permissions.
echo "grant 'atlas', 'RWXCA', 'titan'" | hbase shell
</verbatim>
Note that if the embedded_services profile is used then HBase is included in the distribution so that a standalone
instance of HBase can be started as the default storage backend for the graph repository. Using the embedded_services
Note that if the embedded-hbase-solr profile is used then HBase is included in the distribution so that a standalone
instance of HBase can be started as the default storage backend for the graph repository. Using the embedded-hbase-solr
profile will configure Atlas so that HBase instance will be started and stopped along with the Atlas server by default.
To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
To use the embedded-hbase-solr profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
section.
---+++ Graph Search Index
......@@ -80,10 +80,10 @@ Please note that Solr installation in Cloud mode is a prerequisite before config
atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181
</verbatim>
Also note that if the embedded_services profile is used then Solr is included in the distribution so that a standalone
instance of Solr can be started as the default search indexing backend. Using the embedded_services profile will
Also note that if the embedded-hbase-solr profile is used then Solr is included in the distribution so that a standalone
instance of Solr can be started as the default search indexing backend. Using the embedded-hbase-solr profile will
configure Atlas so that the standalone Solr instance will be started and stopped along with the Atlas server by default.
To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
To use the embedded-hbase-solr profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
section.
---+++ Choosing between Persistence and Indexing Backends
......
docs/src/site/twiki/InstallationSteps.twiki
......@@ -18,15 +18,29 @@ mvn clean package -Pdist
</verbatim>
To build a distribution that packages HBase and Solr, build with the embedded_services profile.
To build a distribution that configures Atlas for external HBase and Solr, build with the external-hbase-solr profile.
<verbatim>
mvn clean package -Pdist,embedded_services
mvn clean package -Pdist,external-hbase-solr
</verbatim>
Using the embedded_services profile will configure Atlas so that an HBase instance and a Solr instance will be started
Note that when the external-hbase-solr profile is used the following steps need to be completed to make Atlas functional.
* Configure atlas.graph.storage.hostname (see "Graph persistence engine - HBase" in the [[Configuration][Configuration]] section).
* Configure atlas.graph.index.search.solr.zookeeper-url (see "Graph Search Index - Solr" in the [[Configuration][Configuration]] section).
* Set HBASE_CONF_DIR to point to a valid HBase config directory (see "Graph persistence engine - HBase" in the [[Configuration][Configuration]] section).
* Create the SOLR indices (see "Graph Search Index - Solr" in the [[Configuration][Configuration]] section).
To build a distribution that packages HBase and Solr, build with the embedded-hbase-solr profile.
<verbatim>
mvn clean package -Pdist,embedded-hbase-solr
</verbatim>
Using the embedded-hbase-solr profile will configure Atlas so that an HBase instance and a Solr instance will be started
and stopped along with the Atlas server by default.
Tar can be found in atlas/distro/target/apache-atlas-${project.version}-bin.tar.gz
......@@ -75,7 +89,7 @@ Tar is structured as follows
</verbatim>
Note that if the embedded_services profile is specified for the build then HBase and Solr are included in the
Note that if the embedded-hbase-solr profile is specified for the build then HBase and Solr are included in the
distribution.
In this case, a standalone instance of HBase can be started as the default storage backend for the graph repository.
......
release-log.txt
......@@ -22,6 +22,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-833 Make default build profile for External HBase and Solr (tbeerbower via yhemanth)
ATLAS-841 mvn clean install of Atlas is failing python unit tests (yhemanth)
ATLAS-834 Handle exceptions from HiveHook executor.submit() (sumasai)
ATLAS-635 Process showing old entity name where as actual entity is renamed ( svimal2106 via sumasai )
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment