Skip to content

A
atlas
Commit b93fe4a4 by Shwetha GS
Options

ATLAS-86 Jenkins build failing as of build #41 (shwethags)

parent 9d303743
Showing with 186 additions and 428 deletions
addons/hive-bridge/pom.xml
...@@ -38,11 +38,6 @@ ...@@ -38,11 +38,6 @@
</properties> </properties>
<dependencies> <dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
</dependency>
<!-- Logging --> <!-- Logging -->
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
...@@ -95,21 +90,20 @@ ...@@ -95,21 +90,20 @@
<dependency> <dependency>
<groupId>org.apache.atlas</groupId> <groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId> <artifactId>atlas-client</artifactId>
<version>${version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
<scope>runtime</scope>
<type>test-jar</type>
</dependency> </dependency>
<!-- to bring up atlas server for integration tests -->
<dependency> <dependency>
<groupId>org.apache.atlas</groupId> <groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId> <artifactId>atlas-webapp</artifactId>
<type>war</type>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
...@@ -128,20 +122,6 @@ ...@@ -128,20 +122,6 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<classifier>classes</classifier>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<type>test-jar</type>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId> <artifactId>jetty-server</artifactId>
<scope>test</scope> <scope>test</scope>
...@@ -329,13 +309,6 @@ ...@@ -329,13 +309,6 @@
<skip>false</skip> <skip>false</skip>
</configuration> </configuration>
</plugin> </plugin>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<inherited>true</inherited>
<extensions>true</extensions>
</plugin>
</plugins> </plugins>
</build> </build>
</project> </project>
client/pom.xml
...@@ -77,28 +77,4 @@ ...@@ -77,28 +77,4 @@
<artifactId>testng</artifactId> <artifactId>testng</artifactId>
</dependency> </dependency>
</dependencies> </dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<inherited>true</inherited>
<extensions>true</extensions>
</plugin>
</plugins>
</build>
</project> </project>
client/src/main/java/org/apache/atlas/ApplicationProperties.java
...@@ -17,8 +17,10 @@ ...@@ -17,8 +17,10 @@
package org.apache.atlas; package org.apache.atlas;
import org.apache.commons.configuration.AbstractConfiguration;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.ConfigurationUtils;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -59,66 +61,26 @@ public class ApplicationProperties extends PropertiesConfiguration { ...@@ -59,66 +61,26 @@ public class ApplicationProperties extends PropertiesConfiguration {
: new File(confLocation, fileName).toURI().toURL(); : new File(confLocation, fileName).toURI().toURL();
LOG.info("Loading {} from {}", fileName, url); LOG.info("Loading {} from {}", fileName, url);
ApplicationProperties configuration = new ApplicationProperties(url); Configuration configuration = new ApplicationProperties(url).interpolatedConfiguration();
Iterator<String> keys = configuration.getKeys(); logConfiguration(configuration);
LOG.debug("Configuration loaded:");
while(keys.hasNext()) {
String key = keys.next();
LOG.debug("{} = {}", key, configuration.getProperty(key));
}
return configuration; return configuration;
} catch (Exception e) { } catch (Exception e) {
throw new AtlasException("Failed to load application properties", e); throw new AtlasException("Failed to load application properties", e);
} }
} }
public static final Configuration getSubsetConfiguration(Configuration inConf, String prefix) { private static void logConfiguration(Configuration configuration) {
return inConf.subset(prefix); if (LOG.isDebugEnabled()) {
} Iterator<String> keys = configuration.getKeys();
LOG.debug("Configuration loaded:");
@Override while (keys.hasNext()) {
public Object getProperty(String key) { String key = keys.next();
Object value = super.getProperty(key); LOG.debug("{} = {}", key, configuration.getProperty(key));
if (value instanceof String) {
value = substituteVars((String) value);
}
return value;
}
private static final Pattern VAR_PATTERN = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
private static final int MAX_SUBST = 20;
private String substituteVars(String expr) {
if (expr == null) {
return null;
}
Matcher match = VAR_PATTERN.matcher("");
String eval = expr;
for(int s = 0; s < MAX_SUBST; s++) {
match.reset(eval);
if (!match.find()) {
return eval;
}
String var = match.group();
var = var.substring(2, var.length() - 1); // remove ${ .. }
String val = null;
try {
val = System.getProperty(var);
} catch(SecurityException se) {
LOG.warn("Unexpected SecurityException in Configuration", se);
} }
if (val == null) {
val = getString(var);
} }
if (val == null) {
return eval; // return literal ${var}: var is unbound
} }
// substitute public static final Configuration getSubsetConfiguration(Configuration inConf, String prefix) {
eval = eval.substring(0, match.start()) + val + eval.substring(match.end()); return inConf.subset(prefix);
}
throw new IllegalStateException("Variable substitution depth too large: " + MAX_SUBST + " " + expr);
} }
} }
client/src/main/java/org/apache/atlas/security/SecureClientUtils.java
...@@ -64,7 +64,7 @@ public class SecureClientUtils { ...@@ -64,7 +64,7 @@ public class SecureClientUtils {
final UserGroupInformation ugi) { final UserGroupInformation ugi) {
config.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true); config.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, "ssl-client.xml")); conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, SecurityProperties.SSL_CLIENT_PROPERTIES));
UserGroupInformation.setConfiguration(conf); UserGroupInformation.setConfiguration(conf);
final ConnectionConfigurator connConfigurator = newConnConfigurator(conf); final ConnectionConfigurator connConfigurator = newConnConfigurator(conf);
String authType = "simple"; String authType = "simple";
......
client/src/test/resources/application.properties
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
# #
#system property #system property
atlas.data=/var/data/${user.name}/atlas atlas.data=/var/data/${sys:user.name}/atlas
#re-use existing property #re-use existing property
atlas.graph.data=${atlas.data}/graph atlas.graph.data=${atlas.data}/graph
......
pom.xml
...@@ -353,6 +353,8 @@ ...@@ -353,6 +353,8 @@
<!-- skips checkstyle and find bugs --> <!-- skips checkstyle and find bugs -->
<skipCheck>false</skipCheck> <skipCheck>false</skipCheck>
<skipTests>false</skipTests>
<projectBaseDir>${project.basedir}</projectBaseDir>
<titan.storage.backend>berkeleyje</titan.storage.backend> <titan.storage.backend>berkeleyje</titan.storage.backend>
<titan.index.backend>elasticsearch</titan.index.backend> <titan.index.backend>elasticsearch</titan.index.backend>
</properties> </properties>
...@@ -909,23 +911,15 @@ ...@@ -909,23 +911,15 @@
<dependency> <dependency>
<groupId>org.apache.atlas</groupId> <groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<version>${project.version}</version>
<classifier>classes</classifier>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId> <artifactId>atlas-client</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.atlas</groupId> <groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId> <artifactId>atlas-webapp</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
<type>test-jar</type> <type>war</type>
<scope>test</scope>
</dependency> </dependency>
<!--Scala dependencies--> <!--Scala dependencies-->
...@@ -1303,6 +1297,14 @@ ...@@ -1303,6 +1297,14 @@
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<version>2.5.4</version>
<inherited>true</inherited>
<extensions>true</extensions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId> <artifactId>maven-assembly-plugin</artifactId>
<inherited>false</inherited> <inherited>false</inherited>
<configuration> <configuration>
...@@ -1349,6 +1351,9 @@ ...@@ -1349,6 +1351,9 @@
<artifactId>maven-failsafe-plugin</artifactId> <artifactId>maven-failsafe-plugin</artifactId>
<version>2.18.1</version> <version>2.18.1</version>
<configuration> <configuration>
<systemPropertyVariables>
<projectBaseDir>${projectBaseDir}</projectBaseDir>
</systemPropertyVariables>
<redirectTestOutputToFile>true</redirectTestOutputToFile> <redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.awt.headless=true -Dproject.version=${project.version} <argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
-Dhadoop.tmp.dir=${project.build.directory}/tmp-hadoop-${user.name} -Dhadoop.tmp.dir=${project.build.directory}/tmp-hadoop-${user.name}
......
release-log.txt
...@@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags) ...@@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags) ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES: ALL CHANGES:
ATLAS-86 Jenkins build failing as of build #41 (shwethags)
ATLAS-80 Support for variables in application properties (shwethags) ATLAS-80 Support for variables in application properties (shwethags)
ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags) ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found (dossett@gmail.com via shwethags) ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found (dossett@gmail.com via shwethags)
......
src/conf/application.properties
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
######### Graph Database Configs ######### ######### Graph Database Configs #########
# Graph Storage # Graph Storage
atlas.graph.storage.backend=berkeleyje atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.directory=${atlas.home}/data/berkley atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
#Hbase as stoarge backend #Hbase as stoarge backend
#hbase #hbase
...@@ -40,7 +40,7 @@ atlas.graph.storage.directory=${atlas.home}/data/berkley ...@@ -40,7 +40,7 @@ atlas.graph.storage.directory=${atlas.home}/data/berkley
# Graph Search Index # Graph Search Index
atlas.graph.index.search.backend=elasticsearch atlas.graph.index.search.backend=elasticsearch
atlas.graph.index.search.directory=${atlas.home}/data/es atlas.graph.index.search.directory=${sys:atlas.home}/data/es
atlas.graph.index.search.elasticsearch.client-only=false atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000 atlas.graph.index.search.elasticsearch.create.sleep=2000
......
webapp/pom.xml
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
<packaging>war</packaging> <packaging>war</packaging>
<properties> <properties>
<projectBaseDir>${project.basedir}/..</projectBaseDir>
<debug.jetty.daemon>true</debug.jetty.daemon> <debug.jetty.daemon>true</debug.jetty.daemon>
</properties> </properties>
...@@ -51,7 +52,6 @@ ...@@ -51,7 +52,6 @@
<dependency> <dependency>
<groupId>org.apache.atlas</groupId> <groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId> <artifactId>atlas-client</artifactId>
<type>test-jar</type>
</dependency> </dependency>
<dependency> <dependency>
...@@ -252,17 +252,6 @@ ...@@ -252,17 +252,6 @@
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId> <artifactId>maven-war-plugin</artifactId>
<configuration> <configuration>
<attachClasses>true</attachClasses> <attachClasses>true</attachClasses>
...@@ -395,12 +384,6 @@ ...@@ -395,12 +384,6 @@
</executions> </executions>
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<inherited>true</inherited>
<extensions>true</extensions>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId> <groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId> <artifactId>scala-maven-plugin</artifactId>
</plugin> </plugin>
......
webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
...@@ -18,6 +18,8 @@ ...@@ -18,6 +18,8 @@
package org.apache.atlas.web.service; package org.apache.atlas.web.service;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
...@@ -58,7 +60,7 @@ public class SecureEmbeddedServer extends EmbeddedServer { ...@@ -58,7 +60,7 @@ public class SecureEmbeddedServer extends EmbeddedServer {
} }
protected Connector getConnector(int port) throws IOException { protected Connector getConnector(int port) throws IOException {
PropertiesConfiguration config = getConfiguration(); org.apache.commons.configuration.Configuration config = getConfiguration();
SslContextFactory sslContextFactory = new SslContextFactory(); SslContextFactory sslContextFactory = new SslContextFactory();
sslContextFactory.setKeyStorePath(config.getString(KEYSTORE_FILE_KEY, sslContextFactory.setKeyStorePath(config.getString(KEYSTORE_FILE_KEY,
...@@ -102,7 +104,7 @@ public class SecureEmbeddedServer extends EmbeddedServer { ...@@ -102,7 +104,7 @@ public class SecureEmbeddedServer extends EmbeddedServer {
* @return the password. * @return the password.
* @throws IOException * @throws IOException
*/ */
private String getPassword(PropertiesConfiguration config, String key) throws IOException { private String getPassword(org.apache.commons.configuration.Configuration config, String key) throws IOException {
String password; String password;
...@@ -131,10 +133,10 @@ public class SecureEmbeddedServer extends EmbeddedServer { ...@@ -131,10 +133,10 @@ public class SecureEmbeddedServer extends EmbeddedServer {
* Returns the application configuration. * Returns the application configuration.
* @return * @return
*/ */
protected PropertiesConfiguration getConfiguration() { protected org.apache.commons.configuration.Configuration getConfiguration() {
try { try {
return new PropertiesConfiguration("application.properties"); return ApplicationProperties.get();
} catch (ConfigurationException e) { } catch (AtlasException e) {
throw new RuntimeException("Unable to load configuration: application.properties"); throw new RuntimeException("Unable to load configuration: application.properties");
} }
} }
......
webapp/src/test/java/org/apache/atlas/web/TestUtils.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileWriter;
public class TestUtils {
private static final Logger LOG = LoggerFactory.getLogger(TestUtils.class);
public static String random(){
return RandomStringUtils.randomAlphanumeric(10);
}
public static void writeConfiguration(PropertiesConfiguration configuration, String fileName) throws Exception {
LOG.debug("Storing configuration in file {}", fileName);
File file = new File(fileName);
File parentFile = file.getParentFile();
if (!parentFile.exists() && !parentFile.mkdirs()) {
throw new Exception("Failed to create dir " + parentFile.getAbsolutePath());
}
file.createNewFile();
configuration.save(new FileWriter(file));
}
public static String getTempDirectory() {
return System.getProperty("projectBaseDir") + "/webapp/target/" + random();
}
}
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java → webapp/src/test/java/org/apache/atlas/web/security/BaseSSLAndKerberosTest.java
...@@ -14,15 +14,13 @@ ...@@ -14,15 +14,13 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.atlas.hive.hook; package org.apache.atlas.web.security;
import org.apache.atlas.web.security.BaseSecurityTest; import org.apache.atlas.security.SecurityProperties;
import org.apache.atlas.web.service.SecureEmbeddedServer; import org.apache.atlas.web.service.SecureEmbeddedServer;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.Server;
...@@ -30,10 +28,6 @@ import org.eclipse.jetty.server.Server; ...@@ -30,10 +28,6 @@ import org.eclipse.jetty.server.Server;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
/** /**
* *
*/ */
...@@ -57,7 +51,7 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest { ...@@ -57,7 +51,7 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest {
} }
@Override @Override
public PropertiesConfiguration getConfiguration() { public org.apache.commons.configuration.Configuration getConfiguration() {
return super.getConfiguration(); return super.getConfiguration();
} }
} }
...@@ -75,16 +69,16 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest { ...@@ -75,16 +69,16 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest {
try { try {
char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'}; char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(KEYSTORE_PASSWORD_KEY, storepass); provider.createCredentialEntry(SecurityProperties.KEYSTORE_PASSWORD_KEY, storepass);
char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'}; char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(TRUSTSTORE_PASSWORD_KEY, trustpass); provider.createCredentialEntry(SecurityProperties.TRUSTSTORE_PASSWORD_KEY, trustpass);
char[] trustpass2 = {'k', 'e', 'y', 'p', 'a', 's', 's'}; char[] trustpass2 = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry("ssl.client.truststore.password", trustpass2); provider.createCredentialEntry("ssl.client.truststore.password", trustpass2);
char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'}; char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(SERVER_CERT_PASSWORD_KEY, certpass); provider.createCredentialEntry(SecurityProperties.SERVER_CERT_PASSWORD_KEY, certpass);
// write out so that it can be found in checks // write out so that it can be found in checks
provider.flush(); provider.flush();
...@@ -115,13 +109,4 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest { ...@@ -115,13 +109,4 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest {
FileUtils.write(jaasFile, jaas.toString()); FileUtils.write(jaasFile, jaas.toString());
bindJVMtoJAASFile(jaasFile); bindJVMtoJAASFile(jaasFile);
} }
protected String getWarPath() {
return String.format("/../../webapp/target/atlas-webapp-%s",
System.getProperty("project.version"));
}
protected HiveConf getHiveConf() {
return HiveHookIT.createHiveConf(DGI_URL);
}
} }
webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java
...@@ -18,7 +18,10 @@ package org.apache.atlas.web.security; ...@@ -18,7 +18,10 @@ package org.apache.atlas.web.security;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.apache.zookeeper.Environment; import org.apache.zookeeper.Environment;
import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext; import org.eclipse.jetty.webapp.WebAppContext;
...@@ -32,6 +35,11 @@ import java.nio.file.Files; ...@@ -32,6 +35,11 @@ import java.nio.file.Files;
import java.util.Locale; import java.util.Locale;
import java.util.Properties; import java.util.Properties;
import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
/** /**
* *
*/ */
...@@ -42,11 +50,6 @@ public class BaseSecurityTest { ...@@ -42,11 +50,6 @@ public class BaseSecurityTest {
+ " useTicketCache=false\n" + " doNotPrompt=true\n" + " storeKey=true;\n" + "}; \n"; + " useTicketCache=false\n" + " doNotPrompt=true\n" + " storeKey=true;\n" + "}; \n";
protected MiniKdc kdc; protected MiniKdc kdc;
protected String getWarPath() {
return String.format("/target/atlas-webapp-%s.war",
System.getProperty("release.version"));
}
protected void generateTestProperties(Properties props) throws ConfigurationException, IOException { protected void generateTestProperties(Properties props) throws ConfigurationException, IOException {
PropertiesConfiguration config = PropertiesConfiguration config =
new PropertiesConfiguration(System.getProperty("user.dir") + "/../src/conf/application.properties"); new PropertiesConfiguration(System.getProperty("user.dir") + "/../src/conf/application.properties");
...@@ -62,7 +65,7 @@ public class BaseSecurityTest { ...@@ -62,7 +65,7 @@ public class BaseSecurityTest {
protected void startEmbeddedServer(Server server) throws Exception { protected void startEmbeddedServer(Server server) throws Exception {
WebAppContext webapp = new WebAppContext(); WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/"); webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath()); webapp.setWar(getWarPath());
server.setHandler(webapp); server.setHandler(webapp);
server.start(); server.start();
...@@ -105,4 +108,22 @@ public class BaseSecurityTest { ...@@ -105,4 +108,22 @@ public class BaseSecurityTest {
kdc.createPrincipal(keytab, principal, principal + "/localhost", principal + "/127.0.0.1"); kdc.createPrincipal(keytab, principal, principal + "/localhost", principal + "/127.0.0.1");
return keytab; return keytab;
} }
protected String getWarPath() {
return System.getProperty("projectBaseDir") + String.format("/webapp/target/atlas-webapp-%s",
System.getProperty("project.version"));
}
protected PropertiesConfiguration getSSLConfiguration(String providerUrl) {
String projectBaseDirectory = System.getProperty("projectBaseDir");
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(TLS_ENABLED, true);
configuration.setProperty(TRUSTSTORE_FILE_KEY, projectBaseDirectory + "/webapp/target/atlas.keystore");
configuration.setProperty(KEYSTORE_FILE_KEY, projectBaseDirectory + "/webapp/target/atlas.keystore");
configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY,
SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
return configuration;
}
} }
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java → webapp/src/test/java/org/apache/atlas/web/security/NegativeSSLAndKerberosIT.java
...@@ -16,90 +16,56 @@ ...@@ -16,90 +16,56 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.atlas.hive.hook; package org.apache.atlas.web.security;
import org.apache.atlas.security.SecurityProperties; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.web.TestUtils;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.net.URL; import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED; import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
/** /**
* Perform all the necessary setup steps for client and server comm over SSL/Kerberos, but then don't estalish a * Perform all the necessary setup steps for client and server comm over SSL/Kerberos, but then don't estalish a
* kerberos user for the invocation. Need a separate use case since the Jersey layer cached the URL connection handler, * kerberos user for the invocation. Need a separate use case since the Jersey layer cached the URL connection handler,
* which indirectly caches the kerberos delegation token. * which indirectly caches the kerberos delegation token.
*/ */
public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest { public class NegativeSSLAndKerberosIT extends BaseSSLAndKerberosTest {
private Driver driver;
private SessionState ss;
private TestSecureEmbeddedServer secureEmbeddedServer; private TestSecureEmbeddedServer secureEmbeddedServer;
private String originalConf; private String originalConf;
private AtlasClient dgiClient;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
//Set-up hive session
HiveConf conf = getHiveConf();
driver = new Driver(conf);
ss = new SessionState(conf, System.getProperty("user.name"));
ss = SessionState.start(ss);
SessionState.setCurrentSessionState(ss);
jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks"); jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
String persistDir = null; String persistDir = TestUtils.getTempDirectory();
URL resource = NegativeSSLAndKerberosHiveHookIT.class.getResource("/");
if (resource != null) {
persistDir = resource.toURI().getPath();
}
// delete prior ssl-client.xml file
resource = NegativeSSLAndKerberosHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
if (resource != null) {
File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
if (sslClientFile != null && sslClientFile.exists()) {
sslClientFile.delete();
}
}
setupKDCAndPrincipals(); setupKDCAndPrincipals();
setupCredentials(); setupCredentials();
// client will actually only leverage subset of these properties // client will actually only leverage subset of these properties
final PropertiesConfiguration configuration = new PropertiesConfiguration(); final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
configuration.setProperty(TLS_ENABLED, true);
configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
configuration.setProperty("atlas.http.authentication.type", "kerberos"); configuration.setProperty("atlas.http.authentication.type", "kerberos");
configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY,
SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
configuration.save(new FileWriter(persistDir + File.separator + "client.properties")); TestUtils.writeConfiguration(configuration, persistDir + File.separator + "client.properties");
String confLocation = System.getProperty("atlas.conf"); String confLocation = System.getProperty("atlas.conf");
URL url; URL url;
if (confLocation == null) { if (confLocation == null) {
url = NegativeSSLAndKerberosHiveHookIT.class.getResource("/application.properties"); url = NegativeSSLAndKerberosIT.class.getResource("/application.properties");
} else { } else {
url = new File(confLocation, "application.properties").toURI().toURL(); url = new File(confLocation, "application.properties").toURI().toURL();
} }
...@@ -112,24 +78,25 @@ public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest { ...@@ -112,24 +78,25 @@ public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
configuration.setProperty("atlas.http.authentication.kerberos.name.rules", configuration.setProperty("atlas.http.authentication.kerberos.name.rules",
"RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT"); "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
configuration.save(new FileWriter(persistDir + File.separator + "application.properties")); TestUtils.writeConfiguration(configuration, persistDir + File.separator + "application.properties");
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/apache-atlas") { dgiClient = new AtlasClient(DGI_URL) {
@Override @Override
public PropertiesConfiguration getConfiguration() { protected PropertiesConfiguration getClientProperties() throws AtlasException {
return configuration; return configuration;
} }
}; };
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.getServer().setHandler(webapp);
// save original setting // save original setting
originalConf = System.getProperty("atlas.conf"); originalConf = System.getProperty("atlas.conf");
System.setProperty("atlas.conf", persistDir); System.setProperty("atlas.conf", persistDir);
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {
@Override
public PropertiesConfiguration getConfiguration() {
return configuration;
}
};
secureEmbeddedServer.getServer().start(); secureEmbeddedServer.getServer().start();
} }
@AfterClass @AfterClass
...@@ -147,17 +114,14 @@ public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest { ...@@ -147,17 +114,14 @@ public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
} }
} }
private void runCommand(final String cmd) throws Exception {
ss.setCommandType(null);
driver.run(cmd);
Assert.assertNotNull(driver.getErrorMsg());
Assert.assertTrue(driver.getErrorMsg().contains("Mechanism level: Failed to find any Kerberos tgt"));
}
@Test @Test
public void testUnsecuredCreateDatabase() throws Exception { public void testUnsecuredClient() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); try {
runCommand("create database " + dbName); dgiClient.listTypes();
Assert.fail("Should have failed with GSSException");
} catch(Exception e) {
e.printStackTrace();
Assert.assertTrue(e.getMessage().contains("Mechanism level: Failed to find any Kerberos tgt"));
}
} }
} }
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java → webapp/src/test/java/org/apache/atlas/web/security/SSLAndKerberosIT.java
...@@ -16,25 +16,15 @@ ...@@ -16,25 +16,15 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.atlas.hive.hook; package org.apache.atlas.web.security;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.web.TestUtils;
import org.apache.atlas.security.SecurityProperties;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
...@@ -48,74 +38,43 @@ import javax.security.auth.callback.UnsupportedCallbackException; ...@@ -48,74 +38,43 @@ import javax.security.auth.callback.UnsupportedCallbackException;
import javax.security.auth.login.LoginContext; import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException; import javax.security.auth.login.LoginException;
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import java.security.PrivilegedExceptionAction; import java.security.PrivilegedExceptionAction;
import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED; import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest { public class SSLAndKerberosIT extends BaseSSLAndKerberosTest {
public static final String TEST_USER_JAAS_SECTION = "TestUser"; public static final String TEST_USER_JAAS_SECTION = "TestUser";
public static final String TESTUSER = "testuser"; public static final String TESTUSER = "testuser";
public static final String TESTPASS = "testpass"; public static final String TESTPASS = "testpass";
private static final String DGI_URL = "https://localhost:21443/"; private static final String DGI_URL = "https://localhost:21443/";
private Driver driver;
private AtlasClient dgiCLient; private AtlasClient dgiCLient;
private SessionState ss;
private TestSecureEmbeddedServer secureEmbeddedServer; private TestSecureEmbeddedServer secureEmbeddedServer;
private Subject subject; private Subject subject;
private String originalConf; private String originalConf;
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
//Set-up hive session
HiveConf conf = getHiveConf();
driver = new Driver(conf);
ss = new SessionState(conf, System.getProperty("user.name"));
ss = SessionState.start(ss);
SessionState.setCurrentSessionState(ss);
jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks"); jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
String persistDir = null; String persistDir = TestUtils.getTempDirectory();
URL resource = SSLAndKerberosHiveHookIT.class.getResource("/");
if (resource != null) {
persistDir = resource.toURI().getPath();
}
// delete prior ssl-client.xml file
resource = SSLAndKerberosHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
if (resource != null) {
File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
if (sslClientFile != null && sslClientFile.exists()) {
sslClientFile.delete();
}
}
setupKDCAndPrincipals(); setupKDCAndPrincipals();
setupCredentials(); setupCredentials();
// client will actually only leverage subset of these properties // client will actually only leverage subset of these properties
final PropertiesConfiguration configuration = new PropertiesConfiguration(); final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
configuration.setProperty(TLS_ENABLED, true);
configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
configuration.setProperty("atlas.http.authentication.type", "kerberos"); configuration.setProperty("atlas.http.authentication.type", "kerberos");
configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, TestUtils.writeConfiguration(configuration, persistDir + File.separator + "client.properties");
SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
String confLocation = System.getProperty("atlas.conf"); String confLocation = System.getProperty("atlas.conf");
URL url; URL url;
if (confLocation == null) { if (confLocation == null) {
url = SSLAndKerberosHiveHookIT.class.getResource("/application.properties"); url = SSLAndKerberosIT.class.getResource("/application.properties");
} else { } else {
url = new File(confLocation, "application.properties").toURI().toURL(); url = new File(confLocation, "application.properties").toURI().toURL();
} }
...@@ -127,7 +86,7 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest { ...@@ -127,7 +86,7 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
configuration.setProperty("atlas.http.authentication.kerberos.name.rules", configuration.setProperty("atlas.http.authentication.kerberos.name.rules",
"RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT"); "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
configuration.save(new FileWriter(persistDir + File.separator + "application.properties")); TestUtils.writeConfiguration(configuration, persistDir + File.separator + "application.properties");
subject = loginTestUser(); subject = loginTestUser();
UserGroupInformation.loginUserFromSubject(subject); UserGroupInformation.loginUserFromSubject(subject);
...@@ -147,22 +106,16 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest { ...@@ -147,22 +106,16 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
} }
}); });
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/apache-atlas") { // save original setting
originalConf = System.getProperty("atlas.conf");
System.setProperty("atlas.conf", persistDir);
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {
@Override @Override
public PropertiesConfiguration getConfiguration() { public PropertiesConfiguration getConfiguration() {
return configuration; return configuration;
} }
}; };
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.getServer().setHandler(webapp);
// save original setting
originalConf = System.getProperty("atlas.conf");
System.setProperty("atlas.conf", persistDir);
secureEmbeddedServer.getServer().start(); secureEmbeddedServer.getServer().start();
} }
@AfterClass @AfterClass
...@@ -202,48 +155,9 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest { ...@@ -202,48 +155,9 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
return lc.getSubject(); return lc.getSubject();
} }
private void runCommand(final String cmd) throws Exception {
ss.setCommandType(null);
UserGroupInformation.loginUserFromSubject(subject);
UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
"testUser",
UserGroupInformation.getLoginUser());
proxyUser.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
driver.run(cmd);
return null;
}
});
}
@Test @Test
public void testCreateDatabase() throws Exception { public void testService() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); dgiCLient.listTypes();
runCommand("create database " + dbName);
assertDatabaseIsRegistered(dbName);
}
private void assertDatabaseIsRegistered(String dbName) throws Exception {
assertInstanceIsRegistered(HiveDataTypes.HIVE_DB.getName(), "name", dbName);
} }
private void assertInstanceIsRegistered(final String typeName, final String colName, final String colValue)
throws Exception {
UserGroupInformation.loginUserFromSubject(subject);
UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
"testUser",
UserGroupInformation.getLoginUser());
proxyUser.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
JSONArray results = dgiCLient.rawSearch(typeName, colName, colValue);
Assert.assertEquals(results.length(), 1);
return null;
}
});
}
} }
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java → webapp/src/test/java/org/apache/atlas/web/security/SSLIT.java
...@@ -16,53 +16,33 @@ ...@@ -16,53 +16,33 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.atlas.hive.hook; package org.apache.atlas.web.security;
import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasException;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.web.TestUtils;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.security.SecurityProperties;
import org.apache.atlas.web.service.SecureEmbeddedServer; import org.apache.atlas.web.service.SecureEmbeddedServer;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray;
import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY; import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY; import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY; import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
public class SSLHiveHookIT { public class SSLIT extends BaseSSLAndKerberosTest {
private static final String DGI_URL = "https://localhost:21443/";
private Driver driver;
private AtlasClient dgiCLient; private AtlasClient dgiCLient;
private SessionState ss;
private Path jksPath; private Path jksPath;
private String providerUrl; private String providerUrl;
private TestSecureEmbeddedServer secureEmbeddedServer; private TestSecureEmbeddedServer secureEmbeddedServer;
...@@ -78,47 +58,22 @@ public class SSLHiveHookIT { ...@@ -78,47 +58,22 @@ public class SSLHiveHookIT {
} }
@Override @Override
public PropertiesConfiguration getConfiguration() { public org.apache.commons.configuration.Configuration getConfiguration() {
return super.getConfiguration(); return super.getConfiguration();
} }
} }
@BeforeClass @BeforeClass
public void setUp() throws Exception { public void setUp() throws Exception {
//Set-up hive session
HiveConf conf = getHiveConf();
driver = new Driver(conf);
ss = new SessionState(conf, System.getProperty("user.name"));
ss = SessionState.start(ss);
SessionState.setCurrentSessionState(ss);
jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks"); jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
String persistDir = null; String persistDir = TestUtils.getTempDirectory();
URL resource = SSLHiveHookIT.class.getResource("/");
if (resource != null) {
persistDir = resource.toURI().getPath();
}
// delete prior ssl-client.xml file
resource = SSLHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
if (resource != null) {
File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
if (sslClientFile != null && sslClientFile.exists()) {
sslClientFile.delete();
}
}
setupCredentials();
final PropertiesConfiguration configuration = new PropertiesConfiguration(); setupCredentials();
configuration.setProperty(TLS_ENABLED, true);
configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY,
SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
configuration.save(new FileWriter(persistDir + File.separator + "client.properties")); final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
TestUtils.writeConfiguration(configuration, persistDir + File.separator + "client.properties");
dgiCLient = new AtlasClient(DGI_URL) { dgiCLient = new AtlasClient(DGI_URL) {
@Override @Override
...@@ -127,19 +82,13 @@ public class SSLHiveHookIT { ...@@ -127,19 +82,13 @@ public class SSLHiveHookIT {
} }
}; };
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/apache-atlas") { secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {
@Override @Override
public PropertiesConfiguration getConfiguration() { public PropertiesConfiguration getConfiguration() {
return configuration; return configuration;
} }
}; };
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.getServer().setHandler(webapp);
secureEmbeddedServer.getServer().start(); secureEmbeddedServer.getServer().start();
} }
@AfterClass @AfterClass
...@@ -181,34 +130,8 @@ public class SSLHiveHookIT { ...@@ -181,34 +130,8 @@ public class SSLHiveHookIT {
} }
} }
protected String getWarPath() {
return String.format("/../../webapp/target/atlas-webapp-%s",
System.getProperty("project.version"));
}
private HiveConf getHiveConf() {
return HiveHookIT.createHiveConf(DGI_URL);
}
private void runCommand(String cmd) throws Exception {
ss.setCommandType(null);
driver.run(cmd);
}
@Test @Test
public void testCreateDatabase() throws Exception { public void testService() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase(); dgiCLient.listTypes();
runCommand("create database " + dbName);
assertDatabaseIsRegistered(dbName);
}
private void assertDatabaseIsRegistered(String dbName) throws Exception {
assertInstanceIsRegistered(HiveDataTypes.HIVE_DB.getName(), "name", dbName);
}
private void assertInstanceIsRegistered(String typeName, String colName, String colValue) throws Exception {
JSONArray results = dgiCLient.rawSearch(typeName, colName, colValue);
Assert.assertEquals(results.length(), 1);
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment