Commit 266d7cc0 by Shwetha GS

ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr…

ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
parent b832faf0
......@@ -59,6 +59,13 @@
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -73,6 +80,17 @@
<artifactId>hive-cli</artifactId>
<version>${hive.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -116,8 +134,16 @@
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<type>test-jar</type>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
......@@ -229,21 +255,22 @@
</plugin>
<plugin>
<groupId>org.mortbay.jetty</groupId>
<artifactId>maven-jetty-plugin</artifactId>
<version>${jetty.version}</version>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-maven-plugin</artifactId>
<configuration>
<skip>${skipTests}</skip>
<!--only skip int tests -->
<connectors>
<connector implementation="org.mortbay.jetty.nio.SelectChannelConnector">
<port>21000</port>
<maxIdleTime>60000</maxIdleTime>
</connector>
</connectors>
<webApp>../../webapp/target/atlas-webapp-${project.version}.war</webApp>
<contextPath>/</contextPath>
<useTestClasspath>true</useTestClasspath>
<httpConnector>
<port>21000</port>
<idleTimeout>60000</idleTimeout>
</httpConnector>
<war>../../webapp/target/atlas-webapp-${project.version}.war</war>
<daemon>true</daemon>
<webApp>
<contextPath>/</contextPath>
<descriptor>../../webapp/src/test/webapp/WEB-INF/web.xml</descriptor>
</webApp>
<useTestScope>true</useTestScope>
<systemProperties>
<systemProperty>
<name>atlas.log.dir</name>
......@@ -251,7 +278,7 @@
</systemProperty>
<systemProperty>
<name>atlas.conf</name>
<value>addons/hive-bridge/src/test/resources</value>
<value>${project.build.directory}/test-classes</value>
</systemProperty>
</systemProperties>
<stopKey>atlas-stop</stopKey>
......
......@@ -16,8 +16,7 @@
*/
package org.apache.atlas.hive.hook;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.security.BaseSecurityTest;
import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.web.service.SecureEmbeddedServer;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
......@@ -26,7 +25,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.mortbay.jetty.Server;
import org.eclipse.jetty.server.Server;
import java.io.File;
import java.io.IOException;
......
......@@ -29,7 +29,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......
......@@ -34,7 +34,7 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......
......@@ -37,8 +37,8 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
......
......@@ -17,23 +17,45 @@
#
######### Graph Database Configs #########
#Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html
# Graph Storage
atlas.graph.storage.backend=inmemory
atlas.graph.storage.backend=${titan.storage.backend}
# Graph Search Index
atlas.graph.index.search.backend=lucene
atlas.graph.index.search.directory=target/data/lucene
#Berkeley storage directory
atlas.graph.storage.directory=target/data/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${titan.storage.hostname}
# Graph Search Index Backend
atlas.graph.index.search.backend=${titan.index.backend}
#lucene
#atlas.graph.index.search.directory=target/data/lucene
#elasticsearch
atlas.graph.index.search.directory=./target/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
#solr in cloud mode
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
#solr in http mode
atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process
#atlas.lineage.hive.table.type.name=DataSet
#atlas.lineage.hive.process.type.name=Process
#atlas.lineage.hive.process.inputs.name=inputs
#atlas.lineage.hive.process.outputs.name=outputs
## Schema
#atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
#atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
######### Security Properties #########
......
......@@ -322,11 +322,13 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<slf4j.version>1.7.7</slf4j.version>
<jetty.version>6.1.26</jetty.version>
<jetty.version>9.2.12.v20150709</jetty.version>
<jersey.version>1.9</jersey.version>
<jackson.version>1.8.3</jackson.version>
<tinkerpop.version>2.6.0</tinkerpop.version>
<titan.version>0.5.4</titan.version>
<hadoop.version>2.6.0</hadoop.version>
<hbase.version>0.98.9-hadoop2</hbase.version>
<!-- scala versions -->
<scala.version>2.10.4</scala.version>
......@@ -351,6 +353,8 @@
<!-- skips checkstyle and find bugs -->
<skipCheck>false</skipCheck>
<titan.storage.backend>berkeleyje</titan.storage.backend>
<titan.index.backend>elasticsearch</titan.index.backend>
</properties>
<profiles>
......@@ -381,8 +385,22 @@
</python.path.l>
</properties>
</profile>
</profiles>
<!-- Turning on this profile affects only tests and does not affect packaging -->
<profile>
<id>distributed</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<titan.storage.backend>hbase</titan.storage.backend>
<titan.index.backend>solr</titan.index.backend>
<solr.zk.address>localhost:9983</solr.zk.address>
<titan.storage.hostname>localhost</titan.storage.hostname>
</properties>
</profile>
</profiles>
<modules>
<module>typesystem</module>
<module>client</module>
......@@ -455,9 +473,15 @@
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
<version>6.0.0</version>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-jsp</artifactId>
<version>${jetty.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Logging -->
......@@ -507,25 +531,68 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.htrace</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<artifactId>hadoop-annotations</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
......@@ -615,6 +682,12 @@
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<version>${jersey.version}</version>
<exclusions>
<exclusion>
<groupId>javax.xml.stream</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -639,19 +712,19 @@
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>1.5.2</version>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>1.5.2</version>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
<version>1.3</version>
<version>1.3.7</version>
</dependency>
<dependency>
......@@ -668,15 +741,15 @@
<!-- Jetty -->
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>${jetty.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-plus</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
<version>${jetty.version}</version>
<scope>compile</scope>
</dependency>
......@@ -736,12 +809,80 @@
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-hbase</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>jruby-complete</artifactId>
<groupId>org.jruby</groupId>
</exclusion>
<exclusion>
<artifactId>asm</artifactId>
<groupId>asm</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.mortbay.jetty</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-es</artifactId>
<version>${titan.version}</version>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-solr</artifactId>
<version>${titan.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.jruby.joni</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.eclipse.jetty.orbit</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.restlet.jee</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.ow2.asm</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-lucene</artifactId>
<version>${titan.version}</version>
<!--<scope>test</scope>-->
......@@ -982,6 +1123,7 @@
<testResources>
<testResource>
<directory>src/test/resources</directory>
<filtering>true</filtering>
</testResource>
</testResources>
......@@ -1072,6 +1214,15 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.7</version>
<configuration>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
......@@ -1222,12 +1373,12 @@
<!-- Run the application using "mvn jetty:run" -->
<plugin>
<groupId>org.mortbay.jetty</groupId>
<artifactId>maven-jetty-plugin</artifactId>
<version>6.1.16</version>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-maven-plugin</artifactId>
<version>${jetty.version}</version>
<configuration>
<!-- Log to the console. -->
<requestLog implementation="org.mortbay.jetty.NCSARequestLog">
<requestLog implementation="org.eclipse.jetty.server.NCSARequestLog">
<!-- This doesn't do anything for Jetty, but is a workaround for a Maven bug
that prevents the requestLog from being set. -->
<append>true</append>
......
......@@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES:
ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found (dossett@gmail.com via shwethags)
ATLAS-45 Entity submit fails (suma.shivaprasad via shwethags)
ATLAS-46 Different data directory with restart (shwethags)
......
......@@ -101,11 +101,26 @@
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-solr</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-berkeleyje</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-hbase</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
</dependency>
<dependency>
<groupId>com.thinkaurelius.titan</groupId>
<artifactId>titan-lucene</artifactId>
</dependency>
......@@ -172,6 +187,7 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
......
......@@ -22,6 +22,7 @@ import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.HiveLineageService;
......
......@@ -22,7 +22,6 @@ import com.google.inject.throwingproviders.CheckedProvider;
import com.tinkerpop.blueprints.Graph;
public interface GraphProvider<T extends Graph> extends CheckedProvider<T> {
@Override
T get();
}
......@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Singleton;
import java.util.Iterator;
import java.util.Properties;
/**
* Default implementation for Graph Provider that doles out Titan Graph.
......@@ -43,11 +44,17 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
*/
private static final String ATLAS_PREFIX = "atlas.graph.";
private static TitanGraph graphInstance;
private static Configuration getConfiguration() throws AtlasException {
PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties();
Configuration graphConfig = new PropertiesConfiguration();
Properties sysProperties = System.getProperties();
LOG.info("System properties: ");
LOG.info(sysProperties.toString());
final Iterator<String> iterator = configProperties.getKeys();
while (iterator.hasNext()) {
String key = iterator.next();
......@@ -66,13 +73,20 @@ public class TitanGraphProvider implements GraphProvider<TitanGraph> {
@Singleton
@Provides
public TitanGraph get() {
Configuration config;
try {
config = getConfiguration();
} catch (AtlasException e) {
throw new RuntimeException(e);
}
if(graphInstance == null) {
synchronized (TitanGraphProvider.class) {
if(graphInstance == null) {
Configuration config;
try {
config = getConfiguration();
} catch (AtlasException e) {
throw new RuntimeException(e);
}
return TitanFactory.open(config);
graphInstance = TitanFactory.open(config);
}
}
}
return graphInstance;
}
}
......@@ -81,7 +81,8 @@ public class DefaultMetadataService implements MetadataService {
@Inject
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final Collection<Provider<TypesChangeListener>> typeChangeListeners) throws AtlasException {
final Collection<Provider<TypesChangeListener>> typeChangeListeners) throws AtlasException {
this.typeStore = typeStore;
this.typeSystem = TypeSystem.getInstance();
this.repository = repository;
......
......@@ -19,8 +19,11 @@
package org.apache.atlas;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......
......@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.RepositoryMetadataModule;
......@@ -117,6 +118,12 @@ public class GraphBackedDiscoveryServiceTest {
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
graphProvider.get().shutdown();
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
}
@Test
......
......@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
......@@ -20,9 +20,12 @@ package org.apache.atlas.discovery;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
......@@ -38,17 +41,22 @@ import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
/**
......@@ -58,65 +66,76 @@ import java.util.List;
public class HiveLineageServiceTest {
@Inject
private GraphBackedDiscoveryService discoveryService;
@Inject
private DefaultMetadataService metadataService;
@Inject
private GraphBackedDiscoveryService discoveryService;
private GraphProvider<TitanGraph> graphProvider;
@Inject
private HiveLineageService hiveLineageService;
// @Inject
// private GraphProvider<TitanGraph> graphProvider;
@BeforeClass
public void setUp() throws Exception {
setUpTypes();
setupInstances();
// TestUtils.dumpGraph(graphProvider.get());
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
graphProvider.get().shutdown();
try {
TitanCleanup.clear(graphProvider.get());
} catch (Exception e) {
e.printStackTrace();
}
}
@DataProvider(name = "dslQueriesProvider")
private Object[][] createDSLQueries() {
return new String[][]{
// joins
{"hive_table where name=\"sales_fact\", columns"},
{"hive_table where name=\"sales_fact\", columns select name, dataType, comment"},
{"hive_table where name=\"sales_fact\", columns as c select c.name, c.dataType, c.comment"},
// {"hive_db as db where (db.name=\"Reporting\"), hive_table as table select db.name,
// table.name"},
{"from hive_db"}, {"hive_db"}, {"hive_db where hive_db.name=\"Reporting\""},
{"hive_db hive_db.name = \"Reporting\""},
{"hive_db where hive_db.name=\"Reporting\" select name, owner"}, {"hive_db has name"},
// {"hive_db, hive_table"},
// {"hive_db, hive_process has name"},
// {"hive_db as db1, hive_table where db1.name = \"Reporting\""},
// {"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System
// .currentTimeMillis()},
{"from hive_table"}, {"hive_table"}, {"hive_table is Dimension"},
{"hive_column where hive_column isa PII"},
// {"hive_column where hive_column isa PII select hive_column.name"},
{"hive_column select hive_column.name"}, {"hive_column select name"},
{"hive_column where hive_column.name=\"customer_id\""}, {"from hive_table select hive_table.name"},
{"hive_db where (name = \"Reporting\")"},
{"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1"},
{"hive_db where hive_db has name"},
// {"hive_db hive_table"},
{"hive_db where hive_db has name"},
// {"hive_db as db1 hive_table where (db1.name = \"Reporting\")"},
{"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
// {"hive_db where (name = \"Reporting\") and ((createTime + 1) > 0)"},
// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name =
// \"Reporting\") select db1.name as dbName, tab.name as tabName"},
// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) or (db1.name =
// \"Reporting\") select db1.name as dbName, tab.name as tabName"},
// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name =
// \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name =
// \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
// trait searches
{"Dimension"}, {"Fact"}, {"ETL"}, {"Metric"}, {"PII"},};
// joins
{"hive_table where name=\"sales_fact\", columns"},
{"hive_table where name=\"sales_fact\", columns select name, dataType, comment"},
{"hive_table where name=\"sales_fact\", columns as c select c.name, c.dataType, c.comment"},
// {"hive_db as db where (db.name=\"Reporting\"), hive_table as table select db.name,
// table.name"},
{"from hive_db"}, {"hive_db"}, {"hive_db where hive_db.name=\"Reporting\""},
{"hive_db hive_db.name = \"Reporting\""},
{"hive_db where hive_db.name=\"Reporting\" select name, owner"}, {"hive_db has name"},
// {"hive_db, hive_table"},
// {"hive_db, hive_process has name"},
// {"hive_db as db1, hive_table where db1.name = \"Reporting\""},
// {"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System
// .currentTimeMillis()},
{"from hive_table"}, {"hive_table"}, {"hive_table is Dimension"},
{"hive_column where hive_column isa PII"},
// {"hive_column where hive_column isa PII select hive_column.name"},
{"hive_column select hive_column.name"}, {"hive_column select name"},
{"hive_column where hive_column.name=\"customer_id\""}, {"from hive_table select hive_table.name"},
{"hive_db where (name = \"Reporting\")"},
{"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1"},
{"hive_db where hive_db has name"},
// {"hive_db hive_table"},
{"hive_db where hive_db has name"},
// {"hive_db as db1 hive_table where (db1.name = \"Reporting\")"},
{"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
// {"hive_db where (name = \"Reporting\") and ((createTime + 1) > 0)"},
// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name =
// \"Reporting\") select db1.name as dbName, tab.name as tabName"},
// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) or (db1.name =
// \"Reporting\") select db1.name as dbName, tab.name as tabName"},
// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name =
// \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name =
// \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
// trait searches
{"Dimension"}, {"Fact"}, {"ETL"}, {"Metric"}, {"PII"},};
}
@Test(dataProvider = "dslQueriesProvider")
......@@ -242,7 +261,7 @@ public class HiveLineageServiceTest {
@DataProvider(name = "tableNamesProvider")
private Object[][] tableNames() {
return new String[][]{{"sales_fact", "4"}, {"time_dim", "3"}, {"sales_fact_daily_mv", "4"},
{"sales_fact_monthly_mv", "4"}};
{"sales_fact_monthly_mv", "4"}};
}
@Test(dataProvider = "tableNamesProvider")
......@@ -296,45 +315,45 @@ public class HiveLineageServiceTest {
private TypesDef createTypeDefinitions() {
HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil
.createClassTypeDef(DATABASE_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE),
attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE));
.createClassTypeDef(DATABASE_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE),
attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE));
HierarchicalTypeDefinition<ClassType> storageDescClsDef = TypesUtil
.createClassTypeDef(STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE),
attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE),
attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null));
.createClassTypeDef(STORAGE_DESC_TYPE, null, attrDef("location", DataTypes.STRING_TYPE),
attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE),
attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null));
HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil
.createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE));
.createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> tblClsDef = TypesUtil
.createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"),
attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE),
attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE),
attrDef("temporary", DataTypes.BOOLEAN_TYPE),
new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
// todo - uncomment this, something is broken
// new AttributeDefinition("sd", STORAGE_DESC_TYPE,
// Multiplicity.REQUIRED, true, null),
new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE),
Multiplicity.COLLECTION, true, null));
.createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"),
attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE),
attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE),
attrDef("temporary", DataTypes.BOOLEAN_TYPE),
new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
// todo - uncomment this, something is broken
// new AttributeDefinition("sd", STORAGE_DESC_TYPE,
// Multiplicity.REQUIRED, true, null),
new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE),
Multiplicity.COLLECTION, true, null));
HierarchicalTypeDefinition<ClassType> loadProcessClsDef = TypesUtil
.createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"),
attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE),
attrDef("endTime", DataTypes.LONG_TYPE),
attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED));
.createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"),
attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE),
attrDef("endTime", DataTypes.LONG_TYPE),
attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED));
HierarchicalTypeDefinition<ClassType> viewClsDef = TypesUtil
.createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE),
Multiplicity.COLLECTION, false, null));
.createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE),
Multiplicity.COLLECTION, false, null));
HierarchicalTypeDefinition<TraitType> dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", null);
......@@ -349,8 +368,8 @@ public class HiveLineageServiceTest {
HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null);
return TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef));
ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef));
}
AttributeDefinition attrDef(String name, IDataType dT) {
......@@ -362,7 +381,7 @@ public class HiveLineageServiceTest {
}
AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m, boolean isComposite,
String reverseAttributeName) {
String reverseAttributeName) {
Preconditions.checkNotNull(name);
Preconditions.checkNotNull(dT);
return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
......@@ -372,62 +391,62 @@ public class HiveLineageServiceTest {
Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
Referenceable sd =
storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true);
storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true);
List<Referenceable> salesFactColumns = ImmutableList
.of(column("time_id", "int", "time id"), column("product_id", "int", "product id"),
column("customer_id", "int", "customer id", "PII"),
column("sales", "double", "product id", "Metric"));
.of(column("time_id", "int", "time id"), column("product_id", "int", "product id"),
column("customer_id", "int", "customer id", "PII"),
column("sales", "double", "product id", "Metric"));
Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
List<Referenceable> timeDimColumns = ImmutableList
.of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"),
column("weekDay", "int", "week Day"));
.of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"),
column("weekDay", "int", "week Day"));
Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns,
"Dimension");
"Dimension");
Id reportingDB =
database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting");
database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting");
Id salesFactDaily =
table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed",
salesFactColumns, "Metric");
table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed",
salesFactColumns, "Metric");
loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact, timeDim),
ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL");
ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL");
List<Referenceable> productDimColumns = ImmutableList
.of(column("product_id", "int", "product id"), column("product_name", "string", "product name"),
column("brand_name", "int", "brand name"));
.of(column("product_id", "int", "product id"), column("product_name", "string", "product name"),
column("brand_name", "int", "brand name"));
Id productDim =
table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns,
"Dimension");
table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns,
"Dimension");
view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess");
List<Referenceable> customerDimColumns = ImmutableList.of(column("customer_id", "int", "customer id", "PII"),
column("name", "string", "customer name", "PII"),
column("address", "string", "customer address", "PII"));
column("name", "string", "customer name", "PII"),
column("address", "string", "customer address", "PII"));
Id customerDim =
table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns,
"Dimension");
table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns,
"Dimension");
view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
Id salesFactMonthly =
table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI",
"Managed", salesFactColumns, "Metric");
table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI",
"Managed", salesFactColumns, "Metric");
loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL", ImmutableList.of(salesFactDaily),
ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL");
ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL");
}
Id database(String name, String description, String owner, String locationUri, String... traitNames)
throws Exception {
throws Exception {
Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
referenceable.set("name", name);
referenceable.set("description", description);
......@@ -439,7 +458,7 @@ public class HiveLineageServiceTest {
}
Referenceable storageDescriptor(String location, String inputFormat, String outputFormat, boolean compressed)
throws Exception {
throws Exception {
Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
referenceable.set("location", location);
referenceable.set("inputFormat", inputFormat);
......@@ -459,7 +478,7 @@ public class HiveLineageServiceTest {
}
Id table(String name, String description, Id dbId, Referenceable sd, String owner, String tableType,
List<Referenceable> columns, String... traitNames) throws Exception {
List<Referenceable> columns, String... traitNames) throws Exception {
Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames);
referenceable.set("name", name);
referenceable.set("description", description);
......@@ -479,8 +498,8 @@ public class HiveLineageServiceTest {
}
Id loadProcess(String name, String description, String user, List<Id> inputTables, List<Id> outputTables,
String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames)
throws Exception {
String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames)
throws Exception {
Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames);
referenceable.set("name", name);
referenceable.set("description", description);
......
......@@ -20,9 +20,11 @@ package org.apache.atlas.repository.graph;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
......@@ -51,6 +53,7 @@ import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -105,12 +108,23 @@ public class GraphBackedMetadataRepositoryTest {
createHiveTypes();
}
/*
@AfterMethod
@AfterClass
public void tearDown() throws Exception {
TestUtils.dumpGraph(graphProvider.get());
TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
} catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
}
*/
@Test
public void testSubmitEntity() throws Exception {
......@@ -189,7 +203,8 @@ public class GraphBackedMetadataRepositoryTest {
System.out.println("*** table = " + table);
}
private String getGUID() {
@GraphTransaction
String getGUID() {
Vertex tableVertex = getTableEntityVertex();
String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
......@@ -199,7 +214,8 @@ public class GraphBackedMetadataRepositoryTest {
return guid;
}
private Vertex getTableEntityVertex() {
@GraphTransaction
Vertex getTableEntityVertex() {
TitanGraph graph = graphProvider.get();
GraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
Iterator<Vertex> results = query.vertices().iterator();
......
......@@ -19,6 +19,7 @@
package org.apache.atlas.repository.typestore;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
......@@ -39,6 +40,7 @@ import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -63,6 +65,17 @@ public class GraphBackedTypeStoreTest {
TestUtils.defineDeptEmployeeTypes(ts);
}
@AfterClass
public void tearDown() throws Exception {
ts.reset();
graphProvider.get().shutdown();
try {
TitanCleanup.clear(graphProvider.get());
} catch(Exception e) {
e.printStackTrace();
}
}
@Test
@GraphTransaction
public void testStore() throws AtlasException {
......
......@@ -19,15 +19,35 @@
######### Graph Database Configs #########
#Refer http://s3.thinkaurelius.com/docs/titan/0.5.1/titan-config-ref.html
# Graph Storage
atlas.graph.storage.backend=inmemory
atlas.graph.storage.backend=${titan.storage.backend}
# Graph Search Index
atlas.graph.index.search.backend=elasticsearch
#Berkeley storage directory
atlas.graph.storage.directory=target/data/berkley
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${titan.storage.hostname}
# Graph Search Index Backend
atlas.graph.index.search.backend=${titan.index.backend}
#lucene
#atlas.graph.index.search.directory=target/data/lucene
#elasticsearch
atlas.graph.index.search.directory=./target/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
#solr in cloud mode
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
#solr in http mode
atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
######### Hive Lineage Configs #########
#atlas.lineage.hive.table.type.name=DataSet
......
......@@ -21,6 +21,23 @@
atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.directory=data/berkley
#Hbase as stoarge backend
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
#atlas.graph.storage.hostname=localhost
#Solr
#atlas.graph.index.search.backend=solr
# Solr cloud mode properties
#atlas.graph.index.search.solr.mode=cloud
#atlas.graph.index.search.solr.zookeeper-url=localhost:2181
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
# Graph Search Index
atlas.graph.index.search.backend=elasticsearch
atlas.graph.index.search.directory=data/es
......
......@@ -116,13 +116,13 @@
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-plus</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
</dependency>
<dependency>
......@@ -171,8 +171,13 @@
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-jsp</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</dependency>
<dependency>
......@@ -192,36 +197,36 @@
<workingDirectory>../dashboard/v2/</workingDirectory>
</configuration>
<executions>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>v0.10.30</nodeVersion>
<npmVersion>1.4.3</npmVersion>
</configuration>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>grunt dist</id>
<goals>
<goal>grunt</goal>
</goals>
<configuration>
<arguments>build</arguments>
</configuration>
</execution>
<execution>
<id>install node and npm</id>
<goals>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>v0.10.30</nodeVersion>
<npmVersion>1.4.3</npmVersion>
</configuration>
</execution>
<execution>
<id>npm install</id>
<goals>
<goal>npm</goal>
</goals>
<configuration>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>grunt dist</id>
<goals>
<goal>grunt</goal>
</goals>
<configuration>
<arguments>build</arguments>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
......@@ -245,7 +250,17 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId>
......@@ -260,7 +275,16 @@
<directory>src/main/webapp/WEB-INF</directory>
<targetPath>WEB-INF</targetPath>
</resource>
<resource>
<directory>${project.build.directory}/test-classes</directory>
<targetPath>WEB-INF/classes</targetPath>
</resource>
</webResources>
<archive>
<manifest>
<addClasspath>true</addClasspath>
</manifest>
</archive>
</configuration>
</plugin>
......@@ -312,30 +336,24 @@
</plugin>
<plugin>
<groupId>org.mortbay.jetty</groupId>
<artifactId>maven-jetty-plugin</artifactId>
<version>${jetty.version}</version>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-maven-plugin</artifactId>
<configuration>
<skip>${skipTests}</skip>
<!--only skip int tests -->
<connectors>
<!--
<connector implementation="org.mortbay.jetty.security.SslSocketConnector">
<port>21443</port>
<maxIdleTime>60000</maxIdleTime>
<keystore>${project.build.directory}/../../webapp/target/atlas.keystore</keystore>
<keyPassword>atlas-passwd</keyPassword>
<password>atlas-passwd</password>
</connector>
-->
<connector implementation="org.mortbay.jetty.nio.SelectChannelConnector">
<port>21000</port>
<maxIdleTime>60000</maxIdleTime>
</connector>
</connectors>
<webApp>${project.build.directory}/atlas-webapp-${project.version}</webApp>
<contextPath>/</contextPath>
<useTestClasspath>true</useTestClasspath>
<httpConnector>
<port>21000</port>
<idleTimeout>60000</idleTimeout>
</httpConnector>
<war>${project.build.directory}/atlas-webapp-${project.version}.war</war>
<daemon>true</daemon>
<!--<webAppSourceDirectory>webapp/src/test/webapp</webAppSourceDirectory>-->
<webApp>
<contextPath>/</contextPath>
<descriptor>webapp/src/test/webapp/WEB-INF/web.xml</descriptor>
<!-- ${project.build.directory}/atlas-webapp-${project.version} -->
</webApp>
<useTestScope>true</useTestScope>
<systemProperties>
<systemProperty>
<name>atlas.log.dir</name>
......@@ -364,11 +382,8 @@
<id>start-jetty</id>
<phase>pre-integration-test</phase>
<goals>
<goal>run</goal>
<goal>deploy-war</goal>
</goals>
<configuration>
<daemon>true</daemon>
</configuration>
</execution>
<execution>
<id>stop-jetty</id>
......
......@@ -20,14 +20,20 @@ package org.apache.atlas.web.listeners;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.google.inject.servlet.GuiceServletContextListener;
import com.sun.jersey.api.core.PackagesResourceConfig;
import com.sun.jersey.guice.JerseyServletModule;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.PropertiesUtil;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.web.filters.AtlasAuthenticationFilter;
import org.apache.atlas.web.filters.AuditFilter;
import org.apache.commons.configuration.ConfigurationException;
......@@ -35,6 +41,7 @@ import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.servlet.ServletContextEvent;
import java.util.HashMap;
import java.util.Map;
......@@ -45,7 +52,7 @@ public class GuiceServletConfig extends GuiceServletContextListener {
private static final String GUICE_CTX_PARAM = "guice.packages";
static final String HTTP_AUTHENTICATION_ENABLED = "atlas.http.authentication.enabled";
private Injector injector;
protected Injector injector;
@Override
protected Injector getInjector() {
......@@ -106,5 +113,11 @@ public class GuiceServletConfig extends GuiceServletContextListener {
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
super.contextDestroyed(servletContextEvent);
if(injector != null) {
TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {};
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
final Graph graph = graphProvider.get().get();
graph.shutdown();
}
}
}
\ No newline at end of file
......@@ -20,10 +20,13 @@ package org.apache.atlas.web.service;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppClassLoader;
import org.eclipse.jetty.webapp.WebAppContext;
import java.io.IOException;
......@@ -40,6 +43,7 @@ public class EmbeddedServer {
server.addConnector(connector);
WebAppContext application = new WebAppContext(path, "/");
application.setClassLoader(Thread.currentThread().getContextClassLoader());
server.setHandler(application);
}
......@@ -52,19 +56,21 @@ public class EmbeddedServer {
}
protected Connector getConnector(int port) throws IOException {
Connector connector = new SocketConnector();
connector.setPort(port);
connector.setHost("0.0.0.0");
HttpConfiguration http_config = new HttpConfiguration();
// this is to enable large header sizes when Kerberos is enabled with AD
final Integer bufferSize = getBufferSize();
connector.setHeaderBufferSize(bufferSize);
connector.setRequestBufferSize(bufferSize);
final int bufferSize = getBufferSize();
http_config.setResponseHeaderSize(bufferSize);
http_config.setRequestHeaderSize(bufferSize);
ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(http_config));
connector.setPort(port);
connector.setHost("0.0.0.0");
server.addConnector(connector);
return connector;
}
private Integer getBufferSize() {
protected Integer getBufferSize() {
try {
PropertiesConfiguration configuration = new PropertiesConfiguration("application.properties");
return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE);
......
......@@ -23,8 +23,14 @@ import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.security.SslSocketConnector;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.http.HttpVersion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -54,18 +60,38 @@ public class SecureEmbeddedServer extends EmbeddedServer {
protected Connector getConnector(int port) throws IOException {
PropertiesConfiguration config = getConfiguration();
SslSocketConnector connector = new SslSocketConnector();
connector.setPort(port);
connector.setHost("0.0.0.0");
connector.setKeystore(config.getString(KEYSTORE_FILE_KEY,
System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
connector.setKeyPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY,
System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
connector.setPassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
return connector;
SslContextFactory sslContextFactory = new SslContextFactory();
sslContextFactory.setKeyStorePath(config.getString(KEYSTORE_FILE_KEY,
System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
sslContextFactory.setKeyStorePassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
sslContextFactory.setKeyManagerPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
sslContextFactory.setTrustStorePath(config.getString(TRUSTSTORE_FILE_KEY,
System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
sslContextFactory.setTrustStorePassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
sslContextFactory.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
// SSL HTTP Configuration
// HTTP Configuration
HttpConfiguration http_config = new HttpConfiguration();
http_config.setSecureScheme("https");
final int bufferSize = getBufferSize();
http_config.setSecurePort(port);
http_config.setRequestHeaderSize(bufferSize);
http_config.setResponseHeaderSize(bufferSize);
http_config.setSendServerVersion(true);
http_config.setSendDateHeader(false);
HttpConfiguration https_config = new HttpConfiguration(http_config);
https_config.addCustomizer(new SecureRequestCustomizer());
// SSL Connector
ServerConnector sslConnector = new ServerConnector(server,
new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
new HttpConnectionFactory(https_config));
sslConnector.setPort(port);
server.addConnector(sslConnector);
return sslConnector;
}
/**
......
......@@ -18,16 +18,29 @@
######### Graph Database Configs #########
# Graph Storage
atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.backend=${titan.storage.backend}
# Graph Search Index Backend
atlas.graph.index.search.backend=${titan.index.backend}
#Berkeley storage directory
atlas.graph.storage.directory=target/data/berkley
# Graph Search Index
atlas.graph.index.search.backend=elasticsearch
#hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${titan.storage.hostname}
#ElasticSearch
atlas.graph.index.search.directory=target/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
# Solr cloud mode properties
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
######### Hive Lineage Configs #########
# This models reflects the base super types for Data and Process
#atlas.lineage.hive.table.type.name=DataSet
......
......@@ -16,12 +16,12 @@
*/
package org.apache.atlas.web.filters;
import org.apache.atlas.security.BaseSecurityTest;
import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.web.service.EmbeddedServer;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.mortbay.jetty.Server;
import org.eclipse.jetty.server.Server;
import org.testng.Assert;
import org.testng.annotations.Test;
......
......@@ -16,10 +16,10 @@
*/
package org.apache.atlas.web.filters;
import org.apache.atlas.security.BaseSecurityTest;
import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.web.service.EmbeddedServer;
import org.apache.commons.configuration.ConfigurationException;
import org.mortbay.jetty.Server;
import org.eclipse.jetty.server.Server;
import org.testng.Assert;
import org.testng.annotations.Test;
......
......@@ -16,7 +16,7 @@
*/
package org.apache.atlas.web.listeners;
import org.apache.atlas.security.BaseSecurityTest;
import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web.listeners;
import com.google.inject.Key;
import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Graph;
import org.apache.atlas.repository.graph.GraphProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContextEvent;
public class TestGuiceServletConfig extends GuiceServletConfig {
private static final Logger LOG = LoggerFactory.getLogger(TestGuiceServletConfig.class);
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
LOG.info("Initializing test servlet listener");
super.contextInitialized(servletContextEvent);
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
if(injector != null) {
TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {};
Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType));
TitanGraph graph = graphProvider.get().get();
LOG.info("Clearing graph store");
try {
graph.shutdown();
TitanCleanup.clear(graph);
} catch (Exception e) {
LOG.warn("Clearing graph store failed ", e);
}
}
}
}
......@@ -14,14 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.security;
package org.apache.atlas.web.security;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.zookeeper.Environment;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import java.io.File;
......
......@@ -38,6 +38,7 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase {
SecureEmbeddedServer secureEmbeddedServer = null;
try {
String appPath = System.getProperty("user.dir") + getWarPath();
secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) {
@Override
protected PropertiesConfiguration getConfiguration() {
......@@ -53,6 +54,8 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase {
// test to see whether server is up and root page can be served
Assert.assertEquals(connection.getResponseCode(), 200);
} catch(Throwable e) {
Assert.fail("War deploy failed", e);
} finally {
secureEmbeddedServer.server.stop();
}
......
......@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.mortbay.jetty.webapp.WebAppContext;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.TestListenerAdapter;
import org.testng.TestNG;
......
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
"http://java.sun.com/dtd/web-app_2_3.dtd">
<web-app>
<display-name>Apache Atlas</display-name>
<description>Metadata Management and Data Governance Platform over Hadoop</description>
<context-param>
<param-name>guice.packages</param-name>
<param-value>
org.apache.atlas.web.resources,org.apache.atlas.web.params
</param-value>
</context-param>
<!--
More information can be found here:
https://jersey.java.net/nonav/apidocs/1.11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary.html
-->
<filter>
<filter-name>guiceFilter</filter-name>
<filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>guiceFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<listener>
<listener-class>org.apache.atlas.web.listeners.TestGuiceServletConfig</listener-class>
</listener>
</web-app>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment