Commit 266d7cc0 by Shwetha GS

ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr…

ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
parent b832faf0
...@@ -59,6 +59,13 @@ ...@@ -59,6 +59,13 @@
<artifactId>hive-metastore</artifactId> <artifactId>hive-metastore</artifactId>
<version>${hive.version}</version> <version>${hive.version}</version>
<scope>provided</scope> <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
...@@ -73,6 +80,17 @@ ...@@ -73,6 +80,17 @@
<artifactId>hive-cli</artifactId> <artifactId>hive-cli</artifactId>
<version>${hive.version}</version> <version>${hive.version}</version>
<scope>test</scope> <scope>test</scope>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
...@@ -116,8 +134,16 @@ ...@@ -116,8 +134,16 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mortbay.jetty</groupId> <groupId>org.apache.atlas</groupId>
<artifactId>jetty</artifactId> <artifactId>atlas-webapp</artifactId>
<type>test-jar</type>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
</dependencies> </dependencies>
...@@ -229,21 +255,22 @@ ...@@ -229,21 +255,22 @@
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.mortbay.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>maven-jetty-plugin</artifactId> <artifactId>jetty-maven-plugin</artifactId>
<version>${jetty.version}</version>
<configuration> <configuration>
<skip>${skipTests}</skip> <skip>${skipTests}</skip>
<!--only skip int tests --> <!--only skip int tests -->
<connectors> <httpConnector>
<connector implementation="org.mortbay.jetty.nio.SelectChannelConnector">
<port>21000</port> <port>21000</port>
<maxIdleTime>60000</maxIdleTime> <idleTimeout>60000</idleTimeout>
</connector> </httpConnector>
</connectors> <war>../../webapp/target/atlas-webapp-${project.version}.war</war>
<webApp>../../webapp/target/atlas-webapp-${project.version}.war</webApp> <daemon>true</daemon>
<webApp>
<contextPath>/</contextPath> <contextPath>/</contextPath>
<useTestClasspath>true</useTestClasspath> <descriptor>../../webapp/src/test/webapp/WEB-INF/web.xml</descriptor>
</webApp>
<useTestScope>true</useTestScope>
<systemProperties> <systemProperties>
<systemProperty> <systemProperty>
<name>atlas.log.dir</name> <name>atlas.log.dir</name>
...@@ -251,7 +278,7 @@ ...@@ -251,7 +278,7 @@
</systemProperty> </systemProperty>
<systemProperty> <systemProperty>
<name>atlas.conf</name> <name>atlas.conf</name>
<value>addons/hive-bridge/src/test/resources</value> <value>${project.build.directory}/test-classes</value>
</systemProperty> </systemProperty>
</systemProperties> </systemProperties>
<stopKey>atlas-stop</stopKey> <stopKey>atlas-stop</stopKey>
......
...@@ -16,8 +16,7 @@ ...@@ -16,8 +16,7 @@
*/ */
package org.apache.atlas.hive.hook; package org.apache.atlas.hive.hook;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.security.BaseSecurityTest;
import org.apache.atlas.web.service.SecureEmbeddedServer; import org.apache.atlas.web.service.SecureEmbeddedServer;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
...@@ -26,7 +25,7 @@ import org.apache.hadoop.fs.Path; ...@@ -26,7 +25,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.mortbay.jetty.Server; import org.eclipse.jetty.server.Server;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
......
...@@ -29,7 +29,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; ...@@ -29,7 +29,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier; import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.mortbay.jetty.webapp.WebAppContext; import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
......
...@@ -34,7 +34,7 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider; ...@@ -34,7 +34,7 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier; import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.mortbay.jetty.webapp.WebAppContext; import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
......
...@@ -37,8 +37,8 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider; ...@@ -37,8 +37,8 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier; import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.mortbay.jetty.Server; import org.eclipse.jetty.server.Server;
import org.mortbay.jetty.webapp.WebAppContext; import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
......
...@@ -17,23 +17,45 @@ ...@@ -17,23 +17,45 @@
# #
######### Graph Database Configs ######### ######### Graph Database Configs #########
#Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html
# Graph Storage # Graph Storage
atlas.graph.storage.backend=inmemory atlas.graph.storage.backend=${titan.storage.backend}
# Graph Search Index #Berkeley storage directory
atlas.graph.index.search.backend=lucene atlas.graph.storage.directory=target/data/berkley
atlas.graph.index.search.directory=target/data/lucene
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${titan.storage.hostname}
# Graph Search Index Backend
atlas.graph.index.search.backend=${titan.index.backend}
#lucene
#atlas.graph.index.search.directory=target/data/lucene
#elasticsearch
atlas.graph.index.search.directory=./target/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
#solr in cloud mode
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
#solr in http mode
atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
######### Hive Lineage Configs ######### ######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process
#atlas.lineage.hive.table.type.name=DataSet #atlas.lineage.hive.table.type.name=DataSet
#atlas.lineage.hive.process.type.name=Process #atlas.lineage.hive.process.type.name=Process
#atlas.lineage.hive.process.inputs.name=inputs #atlas.lineage.hive.process.inputs.name=inputs
#atlas.lineage.hive.process.outputs.name=outputs #atlas.lineage.hive.process.outputs.name=outputs
## Schema ## Schema
#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns #atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
######### Security Properties ######### ######### Security Properties #########
......
...@@ -322,11 +322,13 @@ ...@@ -322,11 +322,13 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<slf4j.version>1.7.7</slf4j.version> <slf4j.version>1.7.7</slf4j.version>
<jetty.version>6.1.26</jetty.version> <jetty.version>9.2.12.v20150709</jetty.version>
<jersey.version>1.9</jersey.version> <jersey.version>1.9</jersey.version>
<jackson.version>1.8.3</jackson.version>
<tinkerpop.version>2.6.0</tinkerpop.version> <tinkerpop.version>2.6.0</tinkerpop.version>
<titan.version>0.5.4</titan.version> <titan.version>0.5.4</titan.version>
<hadoop.version>2.6.0</hadoop.version> <hadoop.version>2.6.0</hadoop.version>
<hbase.version>0.98.9-hadoop2</hbase.version>
<!-- scala versions --> <!-- scala versions -->
<scala.version>2.10.4</scala.version> <scala.version>2.10.4</scala.version>
...@@ -351,6 +353,8 @@ ...@@ -351,6 +353,8 @@
<!-- skips checkstyle and find bugs --> <!-- skips checkstyle and find bugs -->
<skipCheck>false</skipCheck> <skipCheck>false</skipCheck>
<titan.storage.backend>berkeleyje</titan.storage.backend>
<titan.index.backend>elasticsearch</titan.index.backend>
</properties> </properties>
<profiles> <profiles>
...@@ -381,8 +385,22 @@ ...@@ -381,8 +385,22 @@
</python.path.l> </python.path.l>
</properties> </properties>
</profile> </profile>
</profiles>
<!-- Turning on this profile affects only tests and does not affect packaging -->
<profile>
<id>distributed</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<titan.storage.backend>hbase</titan.storage.backend>
<titan.index.backend>solr</titan.index.backend>
<solr.zk.address>localhost:9983</solr.zk.address>
<titan.storage.hostname>localhost</titan.storage.hostname>
</properties>
</profile>
</profiles>
<modules> <modules>
<module>typesystem</module> <module>typesystem</module>
<module>client</module> <module>client</module>
...@@ -455,9 +473,15 @@ ...@@ -455,9 +473,15 @@
<dependencyManagement> <dependencyManagement>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.mortbay.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>jsp-2.1</artifactId> <artifactId>jetty-jsp</artifactId>
<version>6.0.0</version> <version>${jetty.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<!-- Logging --> <!-- Logging -->
...@@ -507,25 +531,68 @@ ...@@ -507,25 +531,68 @@
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId> <artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version> <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.htrace</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId> <artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version> <version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId> <artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version> <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId> <artifactId>hadoop-annotations</artifactId>
<version>${hadoop.version}</version> <version>${hadoop.version}</version>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
...@@ -615,6 +682,12 @@ ...@@ -615,6 +682,12 @@
<groupId>com.sun.jersey</groupId> <groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId> <artifactId>jersey-json</artifactId>
<version>${jersey.version}</version> <version>${jersey.version}</version>
<exclusions>
<exclusion>
<groupId>javax.xml.stream</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
...@@ -639,19 +712,19 @@ ...@@ -639,19 +712,19 @@
<dependency> <dependency>
<groupId>org.codehaus.jackson</groupId> <groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId> <artifactId>jackson-core-asl</artifactId>
<version>1.5.2</version> <version>${jackson.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.codehaus.jackson</groupId> <groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId> <artifactId>jackson-mapper-asl</artifactId>
<version>1.5.2</version> <version>${jackson.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.codehaus.jettison</groupId> <groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId> <artifactId>jettison</artifactId>
<version>1.3</version> <version>1.3.7</version>
</dependency> </dependency>
<dependency> <dependency>
...@@ -668,15 +741,15 @@ ...@@ -668,15 +741,15 @@
<!-- Jetty --> <!-- Jetty -->
<dependency> <dependency>
<groupId>org.mortbay.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>jetty</artifactId> <artifactId>jetty-server</artifactId>
<version>${jetty.version}</version> <version>${jetty.version}</version>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mortbay.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-plus</artifactId> <artifactId>jetty-webapp</artifactId>
<version>${jetty.version}</version> <version>${jetty.version}</version>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
...@@ -736,12 +809,80 @@ ...@@ -736,12 +809,80 @@
<dependency> <dependency>
<groupId>com.thinkaurelius.titan</groupId> <groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-hbase</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>jruby-complete</artifactId>
<groupId>org.jruby</groupId>
</exclusion>
<exclusion>
<artifactId>asm</artifactId>
<groupId>asm</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.mortbay.jetty</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-es</artifactId> <artifactId>titan-es</artifactId>
<version>${titan.version}</version> <version>${titan.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.thinkaurelius.titan</groupId> <groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-solr</artifactId>
<version>${titan.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.jruby.joni</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.eclipse.jetty.orbit</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.restlet.jee</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.ow2.asm</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-lucene</artifactId> <artifactId>titan-lucene</artifactId>
<version>${titan.version}</version> <version>${titan.version}</version>
<!--<scope>test</scope>--> <!--<scope>test</scope>-->
...@@ -982,6 +1123,7 @@ ...@@ -982,6 +1123,7 @@
<testResources> <testResources>
<testResource> <testResource>
<directory>src/test/resources</directory> <directory>src/test/resources</directory>
<filtering>true</filtering>
</testResource> </testResource>
</testResources> </testResources>
...@@ -1072,6 +1214,15 @@ ...@@ -1072,6 +1214,15 @@
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.7</version>
<configuration>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId> <groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId> <artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version> <version>3.2.0</version>
...@@ -1222,12 +1373,12 @@ ...@@ -1222,12 +1373,12 @@
<!-- Run the application using "mvn jetty:run" --> <!-- Run the application using "mvn jetty:run" -->
<plugin> <plugin>
<groupId>org.mortbay.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>maven-jetty-plugin</artifactId> <artifactId>jetty-maven-plugin</artifactId>
<version>6.1.16</version> <version>${jetty.version}</version>
<configuration> <configuration>
<!-- Log to the console. --> <!-- Log to the console. -->
<requestLog implementation="org.mortbay.jetty.NCSARequestLog"> <requestLog implementation="org.eclipse.jetty.server.NCSARequestLog">
<!-- This doesn't do anything for Jetty, but is a workaround for a Maven bug <!-- This doesn't do anything for Jetty, but is a workaround for a Maven bug
that prevents the requestLog from being set. --> that prevents the requestLog from being set. -->
<append>true</append> <append>true</append>
......
...@@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags) ...@@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags) ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES: ALL CHANGES:
ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found (dossett@gmail.com via shwethags) ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found (dossett@gmail.com via shwethags)
ATLAS-45 Entity submit fails (suma.shivaprasad via shwethags) ATLAS-45 Entity submit fails (suma.shivaprasad via shwethags)
ATLAS-46 Different data directory with restart (shwethags) ATLAS-46 Different data directory with restart (shwethags)
......
...@@ -101,11 +101,26 @@ ...@@ -101,11 +101,26 @@
<dependency> <dependency>
<groupId>com.thinkaurelius.titan</groupId> <groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-solr</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId> <artifactId>titan-berkeleyje</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.thinkaurelius.titan</groupId> <groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-hbase</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-lucene</artifactId> <artifactId>titan-lucene</artifactId>
</dependency> </dependency>
...@@ -172,6 +187,7 @@ ...@@ -172,6 +187,7 @@
</dependencies> </dependencies>
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
......
...@@ -22,6 +22,7 @@ import com.google.inject.matcher.Matchers; ...@@ -22,6 +22,7 @@ import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder; import com.google.inject.multibindings.Multibinder;
import com.google.inject.throwingproviders.ThrowingProviderBinder; import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import org.aopalliance.intercept.MethodInterceptor; import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.DiscoveryService; import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.HiveLineageService; import org.apache.atlas.discovery.HiveLineageService;
......
...@@ -22,7 +22,6 @@ import com.google.inject.throwingproviders.CheckedProvider; ...@@ -22,7 +22,6 @@ import com.google.inject.throwingproviders.CheckedProvider;
import com.tinkerpop.blueprints.Graph; import com.tinkerpop.blueprints.Graph;
public interface GraphProvider<T extends Graph> extends CheckedProvider<T> { public interface GraphProvider<T extends Graph> extends CheckedProvider<T> {
@Override @Override
T get(); T get();
} }
...@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; ...@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Singleton; import javax.inject.Singleton;
import java.util.Iterator; import java.util.Iterator;
import java.util.Properties;
/** /**
* Default implementation for Graph Provider that doles out Titan Graph. * Default implementation for Graph Provider that doles out Titan Graph.
...@@ -43,11 +44,17 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> { ...@@ -43,11 +44,17 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
*/ */
private static final String ATLAS_PREFIX = "atlas.graph."; private static final String ATLAS_PREFIX = "atlas.graph.";
private static TitanGraph graphInstance;
private static Configuration getConfiguration() throws AtlasException { private static Configuration getConfiguration() throws AtlasException {
PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties(); PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties();
Configuration graphConfig = new PropertiesConfiguration(); Configuration graphConfig = new PropertiesConfiguration();
Properties sysProperties = System.getProperties();
LOG.info("System properties: ");
LOG.info(sysProperties.toString());
final Iterator<String> iterator = configProperties.getKeys(); final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) { while (iterator.hasNext()) {
String key = iterator.next(); String key = iterator.next();
...@@ -66,6 +73,9 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> { ...@@ -66,6 +73,9 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
@Singleton @Singleton
@Provides @Provides
public TitanGraph get() { public TitanGraph get() {
if(graphInstance == null) {
synchronized (TitanGraphProvider.class) {
if(graphInstance == null) {
Configuration config; Configuration config;
try { try {
config = getConfiguration(); config = getConfiguration();
...@@ -73,6 +83,10 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> { ...@@ -73,6 +83,10 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
return TitanFactory.open(config); graphInstance = TitanFactory.open(config);
}
}
}
return graphInstance;
} }
} }
...@@ -82,6 +82,7 @@ public class DefaultMetadataService implements MetadataService { ...@@ -82,6 +82,7 @@ public class DefaultMetadataService implements MetadataService {
@Inject @Inject
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final Collection<Provider<TypesChangeListener>> typeChangeListeners) throws AtlasException { final Collection<Provider<TypesChangeListener>> typeChangeListeners) throws AtlasException {
this.typeStore = typeStore; this.typeStore = typeStore;
this.typeSystem = TypeSystem.getInstance(); this.typeSystem = TypeSystem.getInstance();
this.repository = repository; this.repository = repository;
......
...@@ -19,8 +19,11 @@ ...@@ -19,8 +19,11 @@
package org.apache.atlas; package org.apache.atlas;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.repository.graph.GraphProvider; import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
......
...@@ -20,6 +20,7 @@ package org.apache.atlas.discovery; ...@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RepositoryMetadataModule;
...@@ -117,6 +118,12 @@ public class GraphBackedDiscoveryServiceTest { ...@@ -117,6 +118,12 @@ public class GraphBackedDiscoveryServiceTest {
@AfterClass @AfterClass
public void tearDown() throws Exception { public void tearDown() throws Exception {
TypeSystem.getInstance().reset(); TypeSystem.getInstance().reset();
graphProvider.get().shutdown();
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
} }
@Test @Test
......
...@@ -20,9 +20,12 @@ package org.apache.atlas.discovery; ...@@ -20,9 +20,12 @@ package org.apache.atlas.discovery;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.EntityNotFoundException; import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.TypesDef;
...@@ -38,17 +41,22 @@ import org.apache.atlas.typesystem.types.IDataType; ...@@ -38,17 +41,22 @@ import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity; import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils; import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider; import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import javax.inject.Inject; import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List; import java.util.List;
/** /**
...@@ -58,25 +66,36 @@ import java.util.List; ...@@ -58,25 +66,36 @@ import java.util.List;
public class HiveLineageServiceTest { public class HiveLineageServiceTest {
@Inject @Inject
private GraphBackedDiscoveryService discoveryService;
@Inject
private DefaultMetadataService metadataService; private DefaultMetadataService metadataService;
@Inject @Inject
private GraphBackedDiscoveryService discoveryService; private GraphProvider<TitanGraph> graphProvider;
@Inject @Inject
private HiveLineageService hiveLineageService; private HiveLineageService hiveLineageService;
// @Inject
// private GraphProvider<TitanGraph> graphProvider;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
setUpTypes(); setUpTypes();
setupInstances(); setupInstances();
// TestUtils.dumpGraph(graphProvider.get()); // TestUtils.dumpGraph(graphProvider.get());
} }
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
graphProvider.get().shutdown();
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
}
@DataProvider(name = "dslQueriesProvider") @DataProvider(name = "dslQueriesProvider")
private Object[][] createDSLQueries() { private Object[][] createDSLQueries() {
return new String[][]{ return new String[][]{
......
...@@ -20,9 +20,11 @@ package org.apache.atlas.repository.graph; ...@@ -20,9 +20,11 @@ package org.apache.atlas.repository.graph;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Compare; import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery; import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils; import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
...@@ -51,6 +53,7 @@ import org.apache.commons.lang.RandomStringUtils; ...@@ -51,6 +53,7 @@ import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject; import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
...@@ -105,12 +108,23 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -105,12 +108,23 @@ public class GraphBackedMetadataRepositoryTest {
createHiveTypes(); createHiveTypes();
} }
/*
@AfterMethod @AfterClass
public void tearDown() throws Exception { public void tearDown() throws Exception {
TestUtils.dumpGraph(graphProvider.get()); TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
} }
*/ }
@Test @Test
public void testSubmitEntity() throws Exception { public void testSubmitEntity() throws Exception {
...@@ -189,7 +203,8 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -189,7 +203,8 @@ public class GraphBackedMetadataRepositoryTest {
System.out.println("*** table = " + table); System.out.println("*** table = " + table);
} }
private String getGUID() { @GraphTransaction
String getGUID() {
Vertex tableVertex = getTableEntityVertex(); Vertex tableVertex = getTableEntityVertex();
String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY); String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
...@@ -199,7 +214,8 @@ public class GraphBackedMetadataRepositoryTest { ...@@ -199,7 +214,8 @@ public class GraphBackedMetadataRepositoryTest {
return guid; return guid;
} }
private Vertex getTableEntityVertex() { @GraphTransaction
Vertex getTableEntityVertex() {
TitanGraph graph = graphProvider.get(); TitanGraph graph = graphProvider.get();
GraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE); GraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator(); Iterator<Vertex> results = query.vertices().iterator();
......
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
package org.apache.atlas.repository.typestore; package org.apache.atlas.repository.typestore;
import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.Vertex;
...@@ -39,6 +40,7 @@ import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; ...@@ -39,6 +40,7 @@ import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice; import org.testng.annotations.Guice;
import org.testng.annotations.Test; import org.testng.annotations.Test;
...@@ -63,6 +65,17 @@ public class GraphBackedTypeStoreTest { ...@@ -63,6 +65,17 @@ public class GraphBackedTypeStoreTest {
TestUtils.defineDeptEmployeeTypes(ts); TestUtils.defineDeptEmployeeTypes(ts);
} }
@AfterClass
public void tearDown() throws Exception {
ts.reset();
graphProvider.get().shutdown();
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
}
@Test @Test
@GraphTransaction @GraphTransaction
public void testStore() throws AtlasException { public void testStore() throws AtlasException {
......
...@@ -19,15 +19,35 @@ ...@@ -19,15 +19,35 @@
######### Graph Database Configs ######### ######### Graph Database Configs #########
#Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html #Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html
# Graph Storage # Graph Storage
atlas.graph.storage.backend=inmemory
atlas.graph.storage.backend=${titan.storage.backend}
# Graph Search Index #Berkeley storage directory
atlas.graph.index.search.backend=elasticsearch atlas.graph.storage.directory=target/data/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${titan.storage.hostname}
# Graph Search Index Backend
atlas.graph.index.search.backend=${titan.index.backend}
#lucene
#atlas.graph.index.search.directory=target/data/lucene
#elasticsearch
atlas.graph.index.search.directory=./target/data/es atlas.graph.index.search.directory=./target/data/es
atlas.graph.index.search.elasticsearch.client-only=false atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
#solr in cloud mode
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
#solr in http mode
atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
######### Hive Lineage Configs ######### ######### Hive Lineage Configs #########
#atlas.lineage.hive.table.type.name=DataSet #atlas.lineage.hive.table.type.name=DataSet
......
...@@ -21,6 +21,23 @@ ...@@ -21,6 +21,23 @@
atlas.graph.storage.backend=berkeleyje atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.directory=data/berkley atlas.graph.storage.directory=data/berkley
#Hbase as stoarge backend
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#atlas.graph.storage.hostname=localhost
#Solr
#atlas.graph.index.search.backend=solr
# Solr cloud mode properties
#atlas.graph.index.search.solr.mode=cloud
#atlas.graph.index.search.solr.zookeeper-url=localhost:2181
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
# Graph Search Index # Graph Search Index
atlas.graph.index.search.backend=elasticsearch atlas.graph.index.search.backend=elasticsearch
atlas.graph.index.search.directory=data/es atlas.graph.index.search.directory=data/es
......
...@@ -116,13 +116,13 @@ ...@@ -116,13 +116,13 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mortbay.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>jetty</artifactId> <artifactId>jetty-server</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mortbay.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-plus</artifactId> <artifactId>jetty-webapp</artifactId>
</dependency> </dependency>
<dependency> <dependency>
...@@ -171,8 +171,13 @@ ...@@ -171,8 +171,13 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mortbay.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>jsp-2.1</artifactId> <artifactId>jetty-jsp</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</dependency> </dependency>
<dependency> <dependency>
...@@ -245,7 +250,17 @@ ...@@ -245,7 +250,17 @@
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId> <artifactId>maven-war-plugin</artifactId>
...@@ -260,7 +275,16 @@ ...@@ -260,7 +275,16 @@
<directory>src/main/webapp/WEB-INF</directory> <directory>src/main/webapp/WEB-INF</directory>
<targetPath>WEB-INF</targetPath> <targetPath>WEB-INF</targetPath>
</resource> </resource>
<resource>
<directory>${project.build.directory}/test-classes</directory>
<targetPath>WEB-INF/classes</targetPath>
</resource>
</webResources> </webResources>
<archive>
<manifest>
<addClasspath>true</addClasspath>
</manifest>
</archive>
</configuration> </configuration>
</plugin> </plugin>
...@@ -312,30 +336,24 @@ ...@@ -312,30 +336,24 @@
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.mortbay.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>maven-jetty-plugin</artifactId> <artifactId>jetty-maven-plugin</artifactId>
<version>${jetty.version}</version>
<configuration> <configuration>
<skip>${skipTests}</skip> <skip>${skipTests}</skip>
<!--only skip int tests --> <!--only skip int tests -->
<connectors> <httpConnector>
<!--
<connector implementation="org.mortbay.jetty.security.SslSocketConnector">
<port>21443</port>
<maxIdleTime>60000</maxIdleTime>
<keystore>${project.build.directory}/../../webapp/target/atlas.keystore</keystore>
<keyPassword>atlas-passwd</keyPassword>
<password>atlas-passwd</password>
</connector>
-->
<connector implementation="org.mortbay.jetty.nio.SelectChannelConnector">
<port>21000</port> <port>21000</port>
<maxIdleTime>60000</maxIdleTime> <idleTimeout>60000</idleTimeout>
</connector> </httpConnector>
</connectors> <war>${project.build.directory}/atlas-webapp-${project.version}.war</war>
<webApp>${project.build.directory}/atlas-webapp-${project.version}</webApp> <daemon>true</daemon>
<!--<webAppSourceDirectory>webapp/src/test/webapp</webAppSourceDirectory>-->
<webApp>
<contextPath>/</contextPath> <contextPath>/</contextPath>
<useTestClasspath>true</useTestClasspath> <descriptor>webapp/src/test/webapp/WEB-INF/web.xml</descriptor>
<!-- ${project.build.directory}/atlas-webapp-${project.version} -->
</webApp>
<useTestScope>true</useTestScope>
<systemProperties> <systemProperties>
<systemProperty> <systemProperty>
<name>atlas.log.dir</name> <name>atlas.log.dir</name>
...@@ -364,11 +382,8 @@ ...@@ -364,11 +382,8 @@
<id>start-jetty</id> <id>start-jetty</id>
<phase>pre-integration-test</phase> <phase>pre-integration-test</phase>
<goals> <goals>
<goal>run</goal> <goal>deploy-war</goal>
</goals> </goals>
<configuration>
<daemon>true</daemon>
</configuration>
</execution> </execution>
<execution> <execution>
<id>stop-jetty</id> <id>stop-jetty</id>
......
...@@ -20,14 +20,20 @@ package org.apache.atlas.web.listeners; ...@@ -20,14 +20,20 @@ package org.apache.atlas.web.listeners;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.Injector; import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.google.inject.servlet.GuiceServletContextListener; import com.google.inject.servlet.GuiceServletContextListener;
import com.sun.jersey.api.core.PackagesResourceConfig; import com.sun.jersey.api.core.PackagesResourceConfig;
import com.sun.jersey.guice.JerseyServletModule; import com.sun.jersey.guice.JerseyServletModule;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.PropertiesUtil; import org.apache.atlas.PropertiesUtil;
import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.web.filters.AtlasAuthenticationFilter; import org.apache.atlas.web.filters.AtlasAuthenticationFilter;
import org.apache.atlas.web.filters.AuditFilter; import org.apache.atlas.web.filters.AuditFilter;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
...@@ -35,6 +41,7 @@ import org.apache.commons.configuration.PropertiesConfiguration; ...@@ -35,6 +41,7 @@ import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextEvent;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
...@@ -45,7 +52,7 @@ public class GuiceServletConfig extends GuiceServletContextListener { ...@@ -45,7 +52,7 @@ public class GuiceServletConfig extends GuiceServletContextListener {
private static final String GUICE_CTX_PARAM = "guice.packages"; private static final String GUICE_CTX_PARAM = "guice.packages";
static final String HTTP_AUTHENTICATION_ENABLED = "atlas.http.authentication.enabled"; static final String HTTP_AUTHENTICATION_ENABLED = "atlas.http.authentication.enabled";
private Injector injector; protected Injector injector;
@Override @Override
protected Injector getInjector() { protected Injector getInjector() {
...@@ -106,5 +113,11 @@ public class GuiceServletConfig extends GuiceServletContextListener { ...@@ -106,5 +113,11 @@ public class GuiceServletConfig extends GuiceServletContextListener {
@Override @Override
public void contextDestroyed(ServletContextEvent servletContextEvent) { public void contextDestroyed(ServletContextEvent servletContextEvent) {
super.contextDestroyed(servletContextEvent); super.contextDestroyed(servletContextEvent);
if(injector != null) {
TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {};
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
final Graph graph = graphProvider.get().get();
graph.shutdown();
}
} }
} }
\ No newline at end of file
...@@ -20,10 +20,13 @@ package org.apache.atlas.web.service; ...@@ -20,10 +20,13 @@ package org.apache.atlas.web.service;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.mortbay.jetty.Connector; import org.eclipse.jetty.server.Connector;
import org.mortbay.jetty.Server; import org.eclipse.jetty.server.HttpConfiguration;
import org.mortbay.jetty.bio.SocketConnector; import org.eclipse.jetty.server.HttpConnectionFactory;
import org.mortbay.jetty.webapp.WebAppContext; import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppClassLoader;
import org.eclipse.jetty.webapp.WebAppContext;
import java.io.IOException; import java.io.IOException;
...@@ -40,6 +43,7 @@ public class EmbeddedServer { ...@@ -40,6 +43,7 @@ public class EmbeddedServer {
server.addConnector(connector); server.addConnector(connector);
WebAppContext application = new WebAppContext(path, "/"); WebAppContext application = new WebAppContext(path, "/");
application.setClassLoader(Thread.currentThread().getContextClassLoader());
server.setHandler(application); server.setHandler(application);
} }
...@@ -52,19 +56,21 @@ public class EmbeddedServer { ...@@ -52,19 +56,21 @@ public class EmbeddedServer {
} }
protected Connector getConnector(int port) throws IOException { protected Connector getConnector(int port) throws IOException {
Connector connector = new SocketConnector();
connector.setPort(port);
connector.setHost("0.0.0.0");
HttpConfiguration http_config = new HttpConfiguration();
// this is to enable large header sizes when Kerberos is enabled with AD // this is to enable large header sizes when Kerberos is enabled with AD
final Integer bufferSize = getBufferSize(); final int bufferSize = getBufferSize();
connector.setHeaderBufferSize(bufferSize); http_config.setResponseHeaderSize(bufferSize);
connector.setRequestBufferSize(bufferSize); http_config.setRequestHeaderSize(bufferSize);
ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(http_config));
connector.setPort(port);
connector.setHost("0.0.0.0");
server.addConnector(connector);
return connector; return connector;
} }
private Integer getBufferSize() { protected Integer getBufferSize() {
try { try {
PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties"); PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties");
return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE); return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE);
......
...@@ -23,8 +23,14 @@ import org.apache.commons.configuration.PropertiesConfiguration; ...@@ -23,8 +23,14 @@ import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.mortbay.jetty.Connector; import org.eclipse.jetty.server.Connector;
import org.mortbay.jetty.security.SslSocketConnector; import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.http.HttpVersion;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -54,18 +60,38 @@ public class SecureEmbeddedServer extends EmbeddedServer { ...@@ -54,18 +60,38 @@ public class SecureEmbeddedServer extends EmbeddedServer {
protected Connector getConnector(int port) throws IOException { protected Connector getConnector(int port) throws IOException {
PropertiesConfiguration config = getConfiguration(); PropertiesConfiguration config = getConfiguration();
SslSocketConnector connector = new SslSocketConnector(); SslContextFactory sslContextFactory = new SslContextFactory();
connector.setPort(port); sslContextFactory.setKeyStorePath(config.getString(KEYSTORE_FILE_KEY,
connector.setHost("0.0.0.0");
connector.setKeystore(config.getString(KEYSTORE_FILE_KEY,
System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION))); System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
connector.setKeyPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY)); sslContextFactory.setKeyStorePassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY, sslContextFactory.setKeyManagerPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
sslContextFactory.setTrustStorePath(config.getString(TRUSTSTORE_FILE_KEY,
System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION))); System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY)); sslContextFactory.setTrustStorePassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
connector.setPassword(getPassword(config, KEYSTORE_PASSWORD_KEY)); sslContextFactory.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
return connector; // SSL HTTP Configuration
// HTTP Configuration
HttpConfiguration http_config = new HttpConfiguration();
http_config.setSecureScheme("https");
final int bufferSize = getBufferSize();
http_config.setSecurePort(port);
http_config.setRequestHeaderSize(bufferSize);
http_config.setResponseHeaderSize(bufferSize);
http_config.setSendServerVersion(true);
http_config.setSendDateHeader(false);
HttpConfiguration https_config = new HttpConfiguration(http_config);
https_config.addCustomizer(new SecureRequestCustomizer());
// SSL Connector
ServerConnector sslConnector = new ServerConnector(server,
new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
new HttpConnectionFactory(https_config));
sslConnector.setPort(port);
server.addConnector(sslConnector);
return sslConnector;
} }
/** /**
......
...@@ -18,16 +18,29 @@ ...@@ -18,16 +18,29 @@
######### Graph Database Configs ######### ######### Graph Database Configs #########
# Graph Storage # Graph Storage
atlas.graph.storage.backend=berkeleyje atlas.graph.storage.backend=${titan.storage.backend}
# Graph Search Index Backend
atlas.graph.index.search.backend=${titan.index.backend}
#Berkeley storage directory
atlas.graph.storage.directory=target/data/berkley atlas.graph.storage.directory=target/data/berkley
# Graph Search Index #hbase
atlas.graph.index.search.backend=elasticsearch #For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${titan.storage.hostname}
#ElasticSearch
atlas.graph.index.search.directory=target/data/es atlas.graph.index.search.directory=target/data/es
atlas.graph.index.search.elasticsearch.client-only=false atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000 atlas.graph.index.search.elasticsearch.create.sleep=2000
# Solr cloud mode properties
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
######### Hive Lineage Configs ######### ######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process # This models reflects the base super types for Data and Process
#atlas.lineage.hive.table.type.name=DataSet #atlas.lineage.hive.table.type.name=DataSet
......
...@@ -16,12 +16,12 @@ ...@@ -16,12 +16,12 @@
*/ */
package org.apache.atlas.web.filters; package org.apache.atlas.web.filters;
import org.apache.atlas.security.BaseSecurityTest; import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.web.service.EmbeddedServer; import org.apache.atlas.web.service.EmbeddedServer;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.mortbay.jetty.Server; import org.eclipse.jetty.server.Server;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.Test; import org.testng.annotations.Test;
......
...@@ -16,10 +16,10 @@ ...@@ -16,10 +16,10 @@
*/ */
package org.apache.atlas.web.filters; package org.apache.atlas.web.filters;
import org.apache.atlas.security.BaseSecurityTest; import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.web.service.EmbeddedServer; import org.apache.atlas.web.service.EmbeddedServer;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.mortbay.jetty.Server; import org.eclipse.jetty.server.Server;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.Test; import org.testng.annotations.Test;
......
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
*/ */
package org.apache.atlas.web.listeners; package org.apache.atlas.web.listeners;
import org.apache.atlas.security.BaseSecurityTest; import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.listeners;
import com.google.inject.Key;
import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.repository.graph.GraphProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContextEvent;
public class TestGuiceServletConfig extends GuiceServletConfig {
private static final Logger LOG = LoggerFactory.getLogger(TestGuiceServletConfig.class);
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
LOG.info("Initializing test servlet listener");
super.contextInitialized(servletContextEvent);
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
if(injector != null) {
TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {};
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
TitanGraph graph = graphProvider.get().get();
LOG.info("Clearing graph store");
try {
graph.shutdown();
TitanCleanup.clear(graph);
} catch (Exception e) {
LOG.warn("Clearing graph store failed ", e);
}
}
}
}
...@@ -14,14 +14,14 @@ ...@@ -14,14 +14,14 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.atlas.security; package org.apache.atlas.web.security;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.zookeeper.Environment; import org.apache.zookeeper.Environment;
import org.mortbay.jetty.Server; import org.eclipse.jetty.server.Server;
import org.mortbay.jetty.webapp.WebAppContext; import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert; import org.testng.Assert;
import java.io.File; import java.io.File;
......
...@@ -38,6 +38,7 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase { ...@@ -38,6 +38,7 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase {
SecureEmbeddedServer secureEmbeddedServer = null; SecureEmbeddedServer secureEmbeddedServer = null;
try { try {
String appPath = System.getProperty("user.dir") + getWarPath(); String appPath = System.getProperty("user.dir") + getWarPath();
secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) { secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) {
@Override @Override
protected PropertiesConfiguration getConfiguration() { protected PropertiesConfiguration getConfiguration() {
...@@ -53,6 +54,8 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase { ...@@ -53,6 +54,8 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase {
// test to see whether server is up and root page can be served // test to see whether server is up and root page can be served
Assert.assertEquals(connection.getResponseCode(), 200); Assert.assertEquals(connection.getResponseCode(), 200);
} catch(Throwable e) {
Assert.fail("War deploy failed", e);
} finally { } finally {
secureEmbeddedServer.server.stop(); secureEmbeddedServer.server.stop();
} }
......
...@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.Path; ...@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.mortbay.jetty.webapp.WebAppContext; import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert; import org.testng.Assert;
import org.testng.TestListenerAdapter; import org.testng.TestListenerAdapter;
import org.testng.TestNG; import org.testng.TestNG;
......
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
"http://java.sun.com/dtd/web-app_2_3.dtd">
<web-app>
<display-name>Apache Atlas</display-name>
<description>Metadata Management and Data Governance Platform over Hadoop</description>
<context-param>
<param-name>guice.packages</param-name>
<param-value>
org.apache.atlas.web.resources,org.apache.atlas.web.params
</param-value>
</context-param>
<!--
More information can be found here:
https://jersey.java.net/nonav/apidocs/1.11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary.html
-->
<filter>
<filter-name>guiceFilter</filter-name>
<filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>guiceFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<listener>
<listener-class>org.apache.atlas.web.listeners.TestGuiceServletConfig</listener-class>
</listener>
</web-app>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment