Skip to content

A
atlas
Commit 266d7cc0 by Shwetha GS
Options

ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr…

ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
parent b832faf0
Showing with 469 additions and 107 deletions
addons/hive-bridge/pom.xml
......@@ -59,6 +59,13 @@
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -73,6 +80,17 @@
<artifactId>hive-cli</artifactId>
<version>${hive.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -116,8 +134,16 @@
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<type>test-jar</type>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
......@@ -229,21 +255,22 @@
</plugin>
<plugin>
<groupId>org.mortbay.jetty</groupId>
<artifactId>maven-jetty-plugin</artifactId>
<version>${jetty.version}</version>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-maven-plugin</artifactId>
<configuration>
<skip>${skipTests}</skip>
<!--only skip int tests -->
<connectors>
<connector implementation="org.mortbay.jetty.nio.SelectChannelConnector">
<httpConnector>
<port>21000</port>
<maxIdleTime>60000</maxIdleTime>
</connector>
</connectors>
<webApp>../../webapp/target/atlas-webapp-${project.version}.war</webApp>
<idleTimeout>60000</idleTimeout>
</httpConnector>
<war>../../webapp/target/atlas-webapp-${project.version}.war</war>
<daemon>true</daemon>
<webApp>
<contextPath>/</contextPath>
<useTestClasspath>true</useTestClasspath>
<descriptor>../../webapp/src/test/webapp/WEB-INF/web.xml</descriptor>
</webApp>
<useTestScope>true</useTestScope>
<systemProperties>
<systemProperty>
<name>atlas.log.dir</name>
......@@ -251,7 +278,7 @@
</systemProperty>
<systemProperty>
<name>atlas.conf</name>
<value>addons/hive-bridge/src/test/resources</value>
<value>${project.build.directory}/test-classes</value>
</systemProperty>
</systemProperties>
<stopKey>atlas-stop</stopKey>
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java
......@@ -16,8 +16,7 @@
*/
package org.apache.atlas.hive.hook;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.security.BaseSecurityTest;
import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.web.service.SecureEmbeddedServer;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
......@@ -26,7 +25,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.mortbay.jetty.Server;
import org.eclipse.jetty.server.Server;
import java.io.File;
import java.io.IOException;
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java
......@@ -29,7 +29,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java
......@@ -34,7 +34,7 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java
......@@ -37,8 +37,8 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......
addons/hive-bridge/src/test/resources/application.properties
......@@ -17,23 +17,45 @@
#
######### Graph Database Configs #########
#Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html
# Graph Storage
atlas.graph.storage.backend=inmemory
atlas.graph.storage.backend=${titan.storage.backend}
# Graph Search Index
atlas.graph.index.search.backend=lucene
atlas.graph.index.search.directory=target/data/lucene
#Berkeley storage directory
atlas.graph.storage.directory=target/data/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${titan.storage.hostname}
# Graph Search Index Backend
atlas.graph.index.search.backend=${titan.index.backend}
#lucene
#atlas.graph.index.search.directory=target/data/lucene
#elasticsearch
atlas.graph.index.search.directory=./target/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
#solr in cloud mode
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
#solr in http mode
atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process
#atlas.lineage.hive.table.type.name=DataSet
#atlas.lineage.hive.process.type.name=Process
#atlas.lineage.hive.process.inputs.name=inputs
#atlas.lineage.hive.process.outputs.name=outputs
## Schema
#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
#atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
######### Security Properties #########
......
release-log.txt
......@@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES:
ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found (dossett@gmail.com via shwethags)
ATLAS-45 Entity submit fails (suma.shivaprasad via shwethags)
ATLAS-46 Different data directory with restart (shwethags)
......
repository/pom.xml
......@@ -101,11 +101,26 @@
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-solr</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-hbase</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-lucene</artifactId>
</dependency>
......@@ -172,6 +187,7 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -22,6 +22,7 @@ import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.HiveLineageService;
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java
......@@ -22,7 +22,6 @@ import com.google.inject.throwingproviders.CheckedProvider;
import com.tinkerpop.blueprints.Graph;
public interface GraphProvider<T extends Graph> extends CheckedProvider<T> {
@Override
T get();
}
repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java
......@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Singleton;
import java.util.Iterator;
import java.util.Properties;
/**
* Default implementation for Graph Provider that doles out Titan Graph.
......@@ -43,11 +44,17 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
*/
private static final String ATLAS_PREFIX = "atlas.graph.";
private static TitanGraph graphInstance;
private static Configuration getConfiguration() throws AtlasException {
PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties();
Configuration graphConfig = new PropertiesConfiguration();
Properties sysProperties = System.getProperties();
LOG.info("System properties: ");
LOG.info(sysProperties.toString());
final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) {
String key = iterator.next();
......@@ -66,6 +73,9 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
@Singleton
@Provides
public TitanGraph get() {
if(graphInstance == null) {
synchronized (TitanGraphProvider.class) {
if(graphInstance == null) {
Configuration config;
try {
config = getConfiguration();
......@@ -73,6 +83,10 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
throw new RuntimeException(e);
}
return TitanFactory.open(config);
graphInstance = TitanFactory.open(config);
}
}
}
return graphInstance;
}
}
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -82,6 +82,7 @@ public class DefaultMetadataService implements MetadataService {
@Inject
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final Collection<Provider<TypesChangeListener>> typeChangeListeners) throws AtlasException {
this.typeStore = typeStore;
this.typeSystem = TypeSystem.getInstance();
this.repository = repository;
......
repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
......@@ -19,8 +19,11 @@
package org.apache.atlas;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.RepositoryMetadataModule;
......@@ -117,6 +118,12 @@ public class GraphBackedDiscoveryServiceTest {
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
graphProvider.get().shutdown();
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
}
@Test
......
repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
......@@ -20,9 +20,12 @@ package org.apache.atlas.discovery;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
......@@ -38,17 +41,22 @@ import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
/**
......@@ -58,25 +66,36 @@ import java.util.List;
public class HiveLineageServiceTest {
@Inject
private GraphBackedDiscoveryService discoveryService;
@Inject
private DefaultMetadataService metadataService;
@Inject
private GraphBackedDiscoveryService discoveryService;
private GraphProvider<TitanGraph> graphProvider;
@Inject
private HiveLineageService hiveLineageService;
// @Inject
// private GraphProvider<TitanGraph> graphProvider;
@BeforeClass
public void setUp() throws Exception {
setUpTypes();
setupInstances();
// TestUtils.dumpGraph(graphProvider.get());
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
graphProvider.get().shutdown();
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
}
@DataProvider(name = "dslQueriesProvider")
private Object[][] createDSLQueries() {
return new String[][]{
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -20,9 +20,11 @@ package org.apache.atlas.repository.graph;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
......@@ -51,6 +53,7 @@ import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -105,12 +108,23 @@ public class GraphBackedMetadataRepositoryTest {
createHiveTypes();
}
/*
@AfterMethod
@AfterClass
public void tearDown() throws Exception {
TestUtils.dumpGraph(graphProvider.get());
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
*/
}
@Test
public void testSubmitEntity() throws Exception {
......@@ -189,7 +203,8 @@ public class GraphBackedMetadataRepositoryTest {
System.out.println("*** table = " + table);
}
private String getGUID() {
@GraphTransaction
String getGUID() {
Vertex tableVertex = getTableEntityVertex();
String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
......@@ -199,7 +214,8 @@ public class GraphBackedMetadataRepositoryTest {
return guid;
}
private Vertex getTableEntityVertex() {
@GraphTransaction
Vertex getTableEntityVertex() {
TitanGraph graph = graphProvider.get();
GraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
......
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.repository.typestore;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
......@@ -39,6 +40,7 @@ import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -63,6 +65,17 @@ public class GraphBackedTypeStoreTest {
TestUtils.defineDeptEmployeeTypes(ts);
}
@AfterClass
public void tearDown() throws Exception {
ts.reset();
graphProvider.get().shutdown();
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
}
@Test
@GraphTransaction
public void testStore() throws AtlasException {
......
repository/src/test/resources/application.properties
......@@ -19,15 +19,35 @@
######### Graph Database Configs #########
#Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html
# Graph Storage
atlas.graph.storage.backend=inmemory
atlas.graph.storage.backend=${titan.storage.backend}
# Graph Search Index
atlas.graph.index.search.backend=elasticsearch
#Berkeley storage directory
atlas.graph.storage.directory=target/data/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${titan.storage.hostname}
# Graph Search Index Backend
atlas.graph.index.search.backend=${titan.index.backend}
#lucene
#atlas.graph.index.search.directory=target/data/lucene
#elasticsearch
atlas.graph.index.search.directory=./target/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
#solr in cloud mode
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
#solr in http mode
atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
######### Hive Lineage Configs #########
#atlas.lineage.hive.table.type.name=DataSet
......
src/conf/application.properties
......@@ -21,6 +21,23 @@
atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.directory=data/berkley
#Hbase as stoarge backend
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#atlas.graph.storage.hostname=localhost
#Solr
#atlas.graph.index.search.backend=solr
# Solr cloud mode properties
#atlas.graph.index.search.solr.mode=cloud
#atlas.graph.index.search.solr.zookeeper-url=localhost:2181
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
# Graph Search Index
atlas.graph.index.search.backend=elasticsearch
atlas.graph.index.search.directory=data/es
......
webapp/pom.xml
......@@ -116,13 +116,13 @@
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-plus</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
</dependency>
<dependency>
......@@ -171,8 +171,13 @@
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-jsp</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</dependency>
<dependency>
......@@ -245,7 +250,17 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId>
......@@ -260,7 +275,16 @@
<directory>src/main/webapp/WEB-INF</directory>
<targetPath>WEB-INF</targetPath>
</resource>
<resource>
<directory>${project.build.directory}/test-classes</directory>
<targetPath>WEB-INF/classes</targetPath>
</resource>
</webResources>
<archive>
<manifest>
<addClasspath>true</addClasspath>
</manifest>
</archive>
</configuration>
</plugin>
......@@ -312,30 +336,24 @@
</plugin>
<plugin>
<groupId>org.mortbay.jetty</groupId>
<artifactId>maven-jetty-plugin</artifactId>
<version>${jetty.version}</version>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-maven-plugin</artifactId>
<configuration>
<skip>${skipTests}</skip>
<!--only skip int tests -->
<connectors>
<!--
<connector implementation="org.mortbay.jetty.security.SslSocketConnector">
<port>21443</port>
<maxIdleTime>60000</maxIdleTime>
<keystore>${project.build.directory}/../../webapp/target/atlas.keystore</keystore>
<keyPassword>atlas-passwd</keyPassword>
<password>atlas-passwd</password>
</connector>
-->
<connector implementation="org.mortbay.jetty.nio.SelectChannelConnector">
<httpConnector>
<port>21000</port>
<maxIdleTime>60000</maxIdleTime>
</connector>
</connectors>
<webApp>${project.build.directory}/atlas-webapp-${project.version}</webApp>
<idleTimeout>60000</idleTimeout>
</httpConnector>
<war>${project.build.directory}/atlas-webapp-${project.version}.war</war>
<daemon>true</daemon>
<!--<webAppSourceDirectory>webapp/src/test/webapp</webAppSourceDirectory>-->
<webApp>
<contextPath>/</contextPath>
<useTestClasspath>true</useTestClasspath>
<descriptor>webapp/src/test/webapp/WEB-INF/web.xml</descriptor>
<!-- ${project.build.directory}/atlas-webapp-${project.version} -->
</webApp>
<useTestScope>true</useTestScope>
<systemProperties>
<systemProperty>
<name>atlas.log.dir</name>
......@@ -364,11 +382,8 @@
<id>start-jetty</id>
<phase>pre-integration-test</phase>
<goals>
<goal>run</goal>
<goal>deploy-war</goal>
</goals>
<configuration>
<daemon>true</daemon>
</configuration>
</execution>
<execution>
<id>stop-jetty</id>
......
webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java
......@@ -20,14 +20,20 @@ package org.apache.atlas.web.listeners;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.google.inject.servlet.GuiceServletContextListener;
import com.sun.jersey.api.core.PackagesResourceConfig;
import com.sun.jersey.guice.JerseyServletModule;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.PropertiesUtil;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.web.filters.AtlasAuthenticationFilter;
import org.apache.atlas.web.filters.AuditFilter;
import org.apache.commons.configuration.ConfigurationException;
......@@ -35,6 +41,7 @@ import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.servlet.ServletContextEvent;
import java.util.HashMap;
import java.util.Map;
......@@ -45,7 +52,7 @@ public class GuiceServletConfig extends GuiceServletContextListener {
private static final String GUICE_CTX_PARAM = "guice.packages";
static final String HTTP_AUTHENTICATION_ENABLED = "atlas.http.authentication.enabled";
private Injector injector;
protected Injector injector;
@Override
protected Injector getInjector() {
......@@ -106,5 +113,11 @@ public class GuiceServletConfig extends GuiceServletContextListener {
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
super.contextDestroyed(servletContextEvent);
if(injector != null) {
TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {};
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
final Graph graph = graphProvider.get().get();
graph.shutdown();
}
}
}
\ No newline at end of file
webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
......@@ -20,10 +20,13 @@ package org.apache.atlas.web.service;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppClassLoader;
import org.eclipse.jetty.webapp.WebAppContext;
import java.io.IOException;
......@@ -40,6 +43,7 @@ public class EmbeddedServer {
server.addConnector(connector);
WebAppContext application = new WebAppContext(path, "/");
application.setClassLoader(Thread.currentThread().getContextClassLoader());
server.setHandler(application);
}
......@@ -52,19 +56,21 @@ public class EmbeddedServer {
}
protected Connector getConnector(int port) throws IOException {
Connector connector = new SocketConnector();
connector.setPort(port);
connector.setHost("0.0.0.0");
HttpConfiguration http_config = new HttpConfiguration();
// this is to enable large header sizes when Kerberos is enabled with AD
final Integer bufferSize = getBufferSize();
connector.setHeaderBufferSize(bufferSize);
connector.setRequestBufferSize(bufferSize);
final int bufferSize = getBufferSize();
http_config.setResponseHeaderSize(bufferSize);
http_config.setRequestHeaderSize(bufferSize);
ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(http_config));
connector.setPort(port);
connector.setHost("0.0.0.0");
server.addConnector(connector);
return connector;
}
private Integer getBufferSize() {
protected Integer getBufferSize() {
try {
PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties");
return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE);
......
webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
......@@ -23,8 +23,14 @@ import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.security.SslSocketConnector;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.http.HttpVersion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -54,18 +60,38 @@ public class SecureEmbeddedServer extends EmbeddedServer {
protected Connector getConnector(int port) throws IOException {
PropertiesConfiguration config = getConfiguration();
SslSocketConnector connector = new SslSocketConnector();
connector.setPort(port);
connector.setHost("0.0.0.0");
connector.setKeystore(config.getString(KEYSTORE_FILE_KEY,
SslContextFactory sslContextFactory = new SslContextFactory();
sslContextFactory.setKeyStorePath(config.getString(KEYSTORE_FILE_KEY,
System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
connector.setKeyPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY,
sslContextFactory.setKeyStorePassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
sslContextFactory.setKeyManagerPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
sslContextFactory.setTrustStorePath(config.getString(TRUSTSTORE_FILE_KEY,
System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
connector.setPassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
return connector;
sslContextFactory.setTrustStorePassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
sslContextFactory.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
// SSL HTTP Configuration
// HTTP Configuration
HttpConfiguration http_config = new HttpConfiguration();
http_config.setSecureScheme("https");
final int bufferSize = getBufferSize();
http_config.setSecurePort(port);
http_config.setRequestHeaderSize(bufferSize);
http_config.setResponseHeaderSize(bufferSize);
http_config.setSendServerVersion(true);
http_config.setSendDateHeader(false);
HttpConfiguration https_config = new HttpConfiguration(http_config);
https_config.addCustomizer(new SecureRequestCustomizer());
// SSL Connector
ServerConnector sslConnector = new ServerConnector(server,
new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
new HttpConnectionFactory(https_config));
sslConnector.setPort(port);
server.addConnector(sslConnector);
return sslConnector;
}
/**
......
webapp/src/main/resources/application.properties
......@@ -18,16 +18,29 @@
######### Graph Database Configs #########
# Graph Storage
atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.backend=${titan.storage.backend}
# Graph Search Index Backend
atlas.graph.index.search.backend=${titan.index.backend}
#Berkeley storage directory
atlas.graph.storage.directory=target/data/berkley
# Graph Search Index
atlas.graph.index.search.backend=elasticsearch
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${titan.storage.hostname}
#ElasticSearch
atlas.graph.index.search.directory=target/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
# Solr cloud mode properties
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process
#atlas.lineage.hive.table.type.name=DataSet
......
webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java
......@@ -16,12 +16,12 @@
*/
package org.apache.atlas.web.filters;
import org.apache.atlas.security.BaseSecurityTest;
import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.web.service.EmbeddedServer;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.mortbay.jetty.Server;
import org.eclipse.jetty.server.Server;
import org.testng.Assert;
import org.testng.annotations.Test;
......
webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java
......@@ -16,10 +16,10 @@
*/
package org.apache.atlas.web.filters;
import org.apache.atlas.security.BaseSecurityTest;
import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.web.service.EmbeddedServer;
import org.apache.commons.configuration.ConfigurationException;
import org.mortbay.jetty.Server;
import org.eclipse.jetty.server.Server;
import org.testng.Assert;
import org.testng.annotations.Test;
......
webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java
......@@ -16,7 +16,7 @@
*/
package org.apache.atlas.web.listeners;
import org.apache.atlas.security.BaseSecurityTest;
import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
......
webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.listeners;
import com.google.inject.Key;
import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.repository.graph.GraphProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContextEvent;
public class TestGuiceServletConfig extends GuiceServletConfig {
private static final Logger LOG = LoggerFactory.getLogger(TestGuiceServletConfig.class);
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
LOG.info("Initializing test servlet listener");
super.contextInitialized(servletContextEvent);
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
if(injector != null) {
TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {};
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
TitanGraph graph = graphProvider.get().get();
LOG.info("Clearing graph store");
try {
graph.shutdown();
TitanCleanup.clear(graph);
} catch (Exception e) {
LOG.warn("Clearing graph store failed ", e);
}
}
}
}
client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java → webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java
......@@ -14,14 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.security;
package org.apache.atlas.web.security;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.zookeeper.Environment;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import java.io.File;
......
webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java
......@@ -38,6 +38,7 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase {
SecureEmbeddedServer secureEmbeddedServer = null;
try {
String appPath = System.getProperty("user.dir") + getWarPath();
secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) {
@Override
protected PropertiesConfiguration getConfiguration() {
......@@ -53,6 +54,8 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase {
// test to see whether server is up and root page can be served
Assert.assertEquals(connection.getResponseCode(), 200);
} catch(Throwable e) {
Assert.fail("War deploy failed", e);
} finally {
secureEmbeddedServer.server.stop();
}
......
webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java
......@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.TestListenerAdapter;
import org.testng.TestNG;
......
webapp/src/test/webapp/WEB-INF/web.xml 0 → 100755
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
"http://java.sun.com/dtd/web-app_2_3.dtd">
<web-app>
<display-name>Apache Atlas</display-name>
<description>Metadata Management and Data Governance Platform over Hadoop</description>
<context-param>
<param-name>guice.packages</param-name>
<param-value>
org.apache.atlas.web.resources,org.apache.atlas.web.params
</param-value>
</context-param>
<!--
More information can be found here:
https://jersey.java.net/nonav/apidocs/1.11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary.html
-->
<filter>
<filter-name>guiceFilter</filter-name>
<filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>guiceFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<listener>
<listener-class>org.apache.atlas.web.listeners.TestGuiceServletConfig</listener-class>
</listener>
</web-app>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment