Skip to content

A
atlas
Commit b93fe4a4 by Shwetha GS
Options

ATLAS-86 Jenkins build failing as of build #41 (shwethags)

parent 9d303743
Showing with 187 additions and 429 deletions
addons/hive-bridge/pom.xml
......@@ -38,11 +38,6 @@
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
</dependency>
<!-- Logging -->
<dependency>
<groupId>org.slf4j</groupId>
......@@ -95,21 +90,20 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId>
<version>${version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
<scope>runtime</scope>
<type>test-jar</type>
</dependency>
<!-- to bring up atlas server for integration tests -->
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
<artifactId>atlas-webapp</artifactId>
<type>war</type>
<scope>test</scope>
</dependency>
<dependency>
......@@ -128,20 +122,6 @@
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<classifier>classes</classifier>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<type>test-jar</type>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<scope>test</scope>
......@@ -329,13 +309,6 @@
<skip>false</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<inherited>true</inherited>
<extensions>true</extensions>
</plugin>
</plugins>
</build>
</project>
client/pom.xml
......@@ -77,28 +77,4 @@
<artifactId>testng</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<inherited>true</inherited>
<extensions>true</extensions>
</plugin>
</plugins>
</build>
</project>
client/src/main/java/org/apache/atlas/ApplicationProperties.java
......@@ -17,8 +17,10 @@
package org.apache.atlas;
import org.apache.commons.configuration.AbstractConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.ConfigurationUtils;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -59,66 +61,26 @@ public class ApplicationProperties extends PropertiesConfiguration {
: new File(confLocation, fileName).toURI().toURL();
LOG.info("Loading {} from {}", fileName, url);
ApplicationProperties configuration = new ApplicationProperties(url);
Configuration configuration = new ApplicationProperties(url).interpolatedConfiguration();
logConfiguration(configuration);
return configuration;
} catch (Exception e) {
throw new AtlasException("Failed to load application properties", e);
}
}
private static void logConfiguration(Configuration configuration) {
if (LOG.isDebugEnabled()) {
Iterator<String> keys = configuration.getKeys();
LOG.debug("Configuration loaded:");
while(keys.hasNext()) {
while (keys.hasNext()) {
String key = keys.next();
LOG.debug("{} = {}", key, configuration.getProperty(key));
}
return configuration;
} catch (Exception e) {
throw new AtlasException("Failed to load application properties", e);
}
}
public static final Configuration getSubsetConfiguration(Configuration inConf, String prefix) {
return inConf.subset(prefix);
}
@Override
public Object getProperty(String key) {
Object value = super.getProperty(key);
if (value instanceof String) {
value = substituteVars((String) value);
}
return value;
}
private static final Pattern VAR_PATTERN = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
private static final int MAX_SUBST = 20;
private String substituteVars(String expr) {
if (expr == null) {
return null;
}
Matcher match = VAR_PATTERN.matcher("");
String eval = expr;
for(int s = 0; s < MAX_SUBST; s++) {
match.reset(eval);
if (!match.find()) {
return eval;
}
String var = match.group();
var = var.substring(2, var.length() - 1); // remove ${ .. }
String val = null;
try {
val = System.getProperty(var);
} catch(SecurityException se) {
LOG.warn("Unexpected SecurityException in Configuration", se);
}
if (val == null) {
val = getString(var);
}
if (val == null) {
return eval; // return literal ${var}: var is unbound
}
// substitute
eval = eval.substring(0, match.start()) + val + eval.substring(match.end());
}
throw new IllegalStateException("Variable substitution depth too large: " + MAX_SUBST + " " + expr);
}
}
client/src/main/java/org/apache/atlas/security/SecureClientUtils.java
......@@ -64,7 +64,7 @@ public class SecureClientUtils {
final UserGroupInformation ugi) {
config.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
Configuration conf = new Configuration();
conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, "ssl-client.xml"));
conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, SecurityProperties.SSL_CLIENT_PROPERTIES));
UserGroupInformation.setConfiguration(conf);
final ConnectionConfigurator connConfigurator = newConnConfigurator(conf);
String authType = "simple";
......
client/src/test/resources/application.properties
......@@ -17,7 +17,7 @@
#
#system property
atlas.data=/var/data/${user.name}/atlas
atlas.data=/var/data/${sys:user.name}/atlas
#re-use existing property
atlas.graph.data=${atlas.data}/graph
......
pom.xml
......@@ -353,6 +353,8 @@
<!-- skips checkstyle and find bugs -->
<skipCheck>false</skipCheck>
<skipTests>false</skipTests>
<projectBaseDir>${project.basedir}</projectBaseDir>
<titan.storage.backend>berkeleyje</titan.storage.backend>
<titan.index.backend>elasticsearch</titan.index.backend>
</properties>
......@@ -909,23 +911,15 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-webapp</artifactId>
<version>${project.version}</version>
<classifier>classes</classifier>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId>
<artifactId>atlas-webapp</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
<type>war</type>
</dependency>
<!--Scala dependencies-->
......@@ -1303,6 +1297,14 @@
</plugin>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<version>2.5.4</version>
<inherited>true</inherited>
<extensions>true</extensions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<inherited>false</inherited>
<configuration>
......@@ -1349,6 +1351,9 @@
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.18.1</version>
<configuration>
<systemPropertyVariables>
<projectBaseDir>${projectBaseDir}</projectBaseDir>
</systemPropertyVariables>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
-Dhadoop.tmp.dir=${project.build.directory}/tmp-hadoop-${user.name}
......
release-log.txt
......@@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES:
ATLAS-86 Jenkins build failing as of build #41 (shwethags)
ATLAS-80 Support for variables in application properties (shwethags)
ATLAS-37 atlas repository, webapp, hive-bridge tests fails with Hbase and Solr as Titan storage backend (suma.shivaprasad via shwethags)
ATLAS-56 atlas_config.py should give an informative error if jar or java binaries can't be found (dossett@gmail.com via shwethags)
......
src/conf/application.properties
......@@ -19,7 +19,7 @@
######### Graph Database Configs #########
# Graph Storage
atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.directory=${atlas.home}/data/berkley
atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
#Hbase as stoarge backend
#hbase
......@@ -40,7 +40,7 @@ atlas.graph.storage.directory=${atlas.home}/data/berkley
# Graph Search Index
atlas.graph.index.search.backend=elasticsearch
atlas.graph.index.search.directory=${atlas.home}/data/es
atlas.graph.index.search.directory=${sys:atlas.home}/data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
......
webapp/pom.xml
......@@ -34,6 +34,7 @@
<packaging>war</packaging>
<properties>
<projectBaseDir>${project.basedir}/..</projectBaseDir>
<debug.jetty.daemon>true</debug.jetty.daemon>
</properties>
......@@ -51,7 +52,6 @@
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId>
<type>test-jar</type>
</dependency>
<dependency>
......@@ -252,17 +252,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId>
<configuration>
<attachClasses>true</attachClasses>
......@@ -395,12 +384,6 @@
</executions>
</plugin>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<inherited>true</inherited>
<extensions>true</extensions>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
......
webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
......@@ -18,6 +18,8 @@
package org.apache.atlas.web.service;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
......@@ -58,7 +60,7 @@ public class SecureEmbeddedServer extends EmbeddedServer {
}
protected Connector getConnector(int port) throws IOException {
PropertiesConfiguration config = getConfiguration();
org.apache.commons.configuration.Configuration config = getConfiguration();
SslContextFactory sslContextFactory = new SslContextFactory();
sslContextFactory.setKeyStorePath(config.getString(KEYSTORE_FILE_KEY,
......@@ -102,7 +104,7 @@ public class SecureEmbeddedServer extends EmbeddedServer {
* @return the password.
* @throws IOException
*/
private String getPassword(PropertiesConfiguration config, String key) throws IOException {
private String getPassword(org.apache.commons.configuration.Configuration config, String key) throws IOException {
String password;
......@@ -131,10 +133,10 @@ public class SecureEmbeddedServer extends EmbeddedServer {
* Returns the application configuration.
* @return
*/
protected PropertiesConfiguration getConfiguration() {
protected org.apache.commons.configuration.Configuration getConfiguration() {
try {
return new PropertiesConfiguration("application.properties");
} catch (ConfigurationException e) {
return ApplicationProperties.get();
} catch (AtlasException e) {
throw new RuntimeException("Unable to load configuration: application.properties");
}
}
......
webapp/src/test/java/org/apache/atlas/web/TestUtils.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.web;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileWriter;
public class TestUtils {
private static final Logger LOG = LoggerFactory.getLogger(TestUtils.class);
public static String random(){
return RandomStringUtils.randomAlphanumeric(10);
}
public static void writeConfiguration(PropertiesConfiguration configuration, String fileName) throws Exception {
LOG.debug("Storing configuration in file {}", fileName);
File file = new File(fileName);
File parentFile = file.getParentFile();
if (!parentFile.exists() && !parentFile.mkdirs()) {
throw new Exception("Failed to create dir " + parentFile.getAbsolutePath());
}
file.createNewFile();
configuration.save(new FileWriter(file));
}
public static String getTempDirectory() {
return System.getProperty("projectBaseDir") + "/webapp/target/" + random();
}
}
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java → webapp/src/test/java/org/apache/atlas/web/security/BaseSSLAndKerberosTest.java
......@@ -14,15 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.hive.hook;
package org.apache.atlas.web.security;
import org.apache.atlas.web.security.BaseSecurityTest;
import org.apache.atlas.security.SecurityProperties;
import org.apache.atlas.web.service.SecureEmbeddedServer;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.eclipse.jetty.server.Server;
......@@ -30,10 +28,6 @@ import org.eclipse.jetty.server.Server;
import java.io.File;
import java.io.IOException;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
/**
*
*/
......@@ -57,7 +51,7 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest {
}
@Override
public PropertiesConfiguration getConfiguration() {
public org.apache.commons.configuration.Configuration getConfiguration() {
return super.getConfiguration();
}
}
......@@ -75,16 +69,16 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest {
try {
char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(KEYSTORE_PASSWORD_KEY, storepass);
provider.createCredentialEntry(SecurityProperties.KEYSTORE_PASSWORD_KEY, storepass);
char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(TRUSTSTORE_PASSWORD_KEY, trustpass);
provider.createCredentialEntry(SecurityProperties.TRUSTSTORE_PASSWORD_KEY, trustpass);
char[] trustpass2 = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry("ssl.client.truststore.password", trustpass2);
char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(SERVER_CERT_PASSWORD_KEY, certpass);
provider.createCredentialEntry(SecurityProperties.SERVER_CERT_PASSWORD_KEY, certpass);
// write out so that it can be found in checks
provider.flush();
......@@ -115,13 +109,4 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest {
FileUtils.write(jaasFile, jaas.toString());
bindJVMtoJAASFile(jaasFile);
}
protected String getWarPath() {
return String.format("/../../webapp/target/atlas-webapp-%s",
System.getProperty("project.version"));
}
protected HiveConf getHiveConf() {
return HiveHookIT.createHiveConf(DGI_URL);
}
}
webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java
......@@ -18,7 +18,10 @@ package org.apache.atlas.web.security;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.apache.zookeeper.Environment;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
......@@ -32,6 +35,11 @@ import java.nio.file.Files;
import java.util.Locale;
import java.util.Properties;
import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
/**
*
*/
......@@ -42,11 +50,6 @@ public class BaseSecurityTest {
+ " useTicketCache=false\n" + " doNotPrompt=true\n" + " storeKey=true;\n" + "}; \n";
protected MiniKdc kdc;
protected String getWarPath() {
return String.format("/target/atlas-webapp-%s.war",
System.getProperty("release.version"));
}
protected void generateTestProperties(Properties props) throws ConfigurationException, IOException {
PropertiesConfiguration config =
new PropertiesConfiguration(System.getProperty("user.dir") + "/../src/conf/application.properties");
......@@ -62,7 +65,7 @@ public class BaseSecurityTest {
protected void startEmbeddedServer(Server server) throws Exception {
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
webapp.setWar(getWarPath());
server.setHandler(webapp);
server.start();
......@@ -105,4 +108,22 @@ public class BaseSecurityTest {
kdc.createPrincipal(keytab, principal, principal + "/localhost", principal + "/127.0.0.1");
return keytab;
}
protected String getWarPath() {
return System.getProperty("projectBaseDir") + String.format("/webapp/target/atlas-webapp-%s",
System.getProperty("project.version"));
}
protected PropertiesConfiguration getSSLConfiguration(String providerUrl) {
String projectBaseDirectory = System.getProperty("projectBaseDir");
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(TLS_ENABLED, true);
configuration.setProperty(TRUSTSTORE_FILE_KEY, projectBaseDirectory + "/webapp/target/atlas.keystore");
configuration.setProperty(KEYSTORE_FILE_KEY, projectBaseDirectory + "/webapp/target/atlas.keystore");
configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY,
SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
return configuration;
}
}
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java → webapp/src/test/java/org/apache/atlas/web/security/NegativeSSLAndKerberosIT.java
......@@ -16,90 +16,56 @@
* limitations under the License.
*/
package org.apache.atlas.hive.hook;
package org.apache.atlas.web.security;
import org.apache.atlas.security.SecurityProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.web.TestUtils;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.File;
import java.io.FileWriter;
import java.net.URL;
import java.nio.file.Files;
import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
/**
* Perform all the necessary setup steps for client and server comm over SSL/Kerberos, but then don't estalish a
* kerberos user for the invocation. Need a separate use case since the Jersey layer cached the URL connection handler,
* which indirectly caches the kerberos delegation token.
*/
public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
public class NegativeSSLAndKerberosIT extends BaseSSLAndKerberosTest {
private Driver driver;
private SessionState ss;
private TestSecureEmbeddedServer secureEmbeddedServer;
private String originalConf;
private AtlasClient dgiClient;
@BeforeClass
public void setUp() throws Exception {
//Set-up hive session
HiveConf conf = getHiveConf();
driver = new Driver(conf);
ss = new SessionState(conf, System.getProperty("user.name"));
ss = SessionState.start(ss);
SessionState.setCurrentSessionState(ss);
jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
String persistDir = null;
URL resource = NegativeSSLAndKerberosHiveHookIT.class.getResource("/");
if (resource != null) {
persistDir = resource.toURI().getPath();
}
// delete prior ssl-client.xml file
resource = NegativeSSLAndKerberosHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
if (resource != null) {
File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
if (sslClientFile != null && sslClientFile.exists()) {
sslClientFile.delete();
}
}
String persistDir = TestUtils.getTempDirectory();
setupKDCAndPrincipals();
setupCredentials();
// client will actually only leverage subset of these properties
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(TLS_ENABLED, true);
configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
configuration.setProperty("atlas.http.authentication.type", "kerberos");
configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY,
SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
TestUtils.writeConfiguration(configuration, persistDir + File.separator + "client.properties");
String confLocation = System.getProperty("atlas.conf");
URL url;
if (confLocation == null) {
url = NegativeSSLAndKerberosHiveHookIT.class.getResource("/application.properties");
url = NegativeSSLAndKerberosIT.class.getResource("/application.properties");
} else {
url = new File(confLocation, "application.properties").toURI().toURL();
}
......@@ -112,24 +78,25 @@ public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
configuration.setProperty("atlas.http.authentication.kerberos.name.rules",
"RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
configuration.save(new FileWriter(persistDir + File.separator + "application.properties"));
TestUtils.writeConfiguration(configuration, persistDir + File.separator + "application.properties");
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/apache-atlas") {
dgiClient = new AtlasClient(DGI_URL) {
@Override
public PropertiesConfiguration getConfiguration() {
protected PropertiesConfiguration getClientProperties() throws AtlasException {
return configuration;
}
};
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.getServer().setHandler(webapp);
// save original setting
originalConf = System.getProperty("atlas.conf");
System.setProperty("atlas.conf", persistDir);
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {
@Override
public PropertiesConfiguration getConfiguration() {
return configuration;
}
};
secureEmbeddedServer.getServer().start();
}
@AfterClass
......@@ -147,17 +114,14 @@ public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
}
}
private void runCommand(final String cmd) throws Exception {
ss.setCommandType(null);
driver.run(cmd);
Assert.assertNotNull(driver.getErrorMsg());
Assert.assertTrue(driver.getErrorMsg().contains("Mechanism level: Failed to find any Kerberos tgt"));
}
@Test
public void testUnsecuredCreateDatabase() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase();
runCommand("create database " + dbName);
public void testUnsecuredClient() throws Exception {
try {
dgiClient.listTypes();
Assert.fail("Should have failed with GSSException");
} catch(Exception e) {
e.printStackTrace();
Assert.assertTrue(e.getMessage().contains("Mechanism level: Failed to find any Kerberos tgt"));
}
}
}
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java → webapp/src/test/java/org/apache/atlas/web/security/SSLAndKerberosIT.java
......@@ -16,25 +16,15 @@
* limitations under the License.
*/
package org.apache.atlas.hive.hook;
package org.apache.atlas.web.security;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.security.SecurityProperties;
import org.apache.atlas.web.TestUtils;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
......@@ -48,74 +38,43 @@ import javax.security.auth.callback.UnsupportedCallbackException;
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.security.PrivilegedExceptionAction;
import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
public class SSLAndKerberosIT extends BaseSSLAndKerberosTest {
public static final String TEST_USER_JAAS_SECTION = "TestUser";
public static final String TESTUSER = "testuser";
public static final String TESTPASS = "testpass";
private static final String DGI_URL = "https://localhost:21443/";
private Driver driver;
private AtlasClient dgiCLient;
private SessionState ss;
private TestSecureEmbeddedServer secureEmbeddedServer;
private Subject subject;
private String originalConf;
@BeforeClass
public void setUp() throws Exception {
//Set-up hive session
HiveConf conf = getHiveConf();
driver = new Driver(conf);
ss = new SessionState(conf, System.getProperty("user.name"));
ss = SessionState.start(ss);
SessionState.setCurrentSessionState(ss);
jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
String persistDir = null;
URL resource = SSLAndKerberosHiveHookIT.class.getResource("/");
if (resource != null) {
persistDir = resource.toURI().getPath();
}
// delete prior ssl-client.xml file
resource = SSLAndKerberosHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
if (resource != null) {
File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
if (sslClientFile != null && sslClientFile.exists()) {
sslClientFile.delete();
}
}
String persistDir = TestUtils.getTempDirectory();
setupKDCAndPrincipals();
setupCredentials();
// client will actually only leverage subset of these properties
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(TLS_ENABLED, true);
configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
configuration.setProperty("atlas.http.authentication.type", "kerberos");
configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY,
SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
TestUtils.writeConfiguration(configuration, persistDir + File.separator + "client.properties");
String confLocation = System.getProperty("atlas.conf");
URL url;
if (confLocation == null) {
url = SSLAndKerberosHiveHookIT.class.getResource("/application.properties");
url = SSLAndKerberosIT.class.getResource("/application.properties");
} else {
url = new File(confLocation, "application.properties").toURI().toURL();
}
......@@ -127,7 +86,7 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
configuration.setProperty("atlas.http.authentication.kerberos.name.rules",
"RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
configuration.save(new FileWriter(persistDir + File.separator + "application.properties"));
TestUtils.writeConfiguration(configuration, persistDir + File.separator + "application.properties");
subject = loginTestUser();
UserGroupInformation.loginUserFromSubject(subject);
......@@ -147,22 +106,16 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
}
});
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/apache-atlas") {
// save original setting
originalConf = System.getProperty("atlas.conf");
System.setProperty("atlas.conf", persistDir);
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {
@Override
public PropertiesConfiguration getConfiguration() {
return configuration;
}
};
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.getServer().setHandler(webapp);
// save original setting
originalConf = System.getProperty("atlas.conf");
System.setProperty("atlas.conf", persistDir);
secureEmbeddedServer.getServer().start();
}
@AfterClass
......@@ -202,48 +155,9 @@ public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
return lc.getSubject();
}
private void runCommand(final String cmd) throws Exception {
ss.setCommandType(null);
UserGroupInformation.loginUserFromSubject(subject);
UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
"testUser",
UserGroupInformation.getLoginUser());
proxyUser.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
driver.run(cmd);
return null;
}
});
}
@Test
public void testCreateDatabase() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase();
runCommand("create database " + dbName);
assertDatabaseIsRegistered(dbName);
}
private void assertDatabaseIsRegistered(String dbName) throws Exception {
assertInstanceIsRegistered(HiveDataTypes.HIVE_DB.getName(), "name", dbName);
public void testService() throws Exception {
dgiCLient.listTypes();
}
private void assertInstanceIsRegistered(final String typeName, final String colName, final String colValue)
throws Exception {
UserGroupInformation.loginUserFromSubject(subject);
UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
"testUser",
UserGroupInformation.getLoginUser());
proxyUser.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
JSONArray results = dgiCLient.rawSearch(typeName, colName, colValue);
Assert.assertEquals(results.length(), 1);
return null;
}
});
}
}
addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java → webapp/src/test/java/org/apache/atlas/web/security/SSLIT.java
......@@ -16,53 +16,33 @@
* limitations under the License.
*/
package org.apache.atlas.hive.hook;
package org.apache.atlas.web.security;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.security.SecurityProperties;
import org.apache.atlas.web.TestUtils;
import org.apache.atlas.web.service.SecureEmbeddedServer;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
import org.codehaus.jettison.json.JSONArray;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
public class SSLHiveHookIT {
private static final String DGI_URL = "https://localhost:21443/";
private Driver driver;
public class SSLIT extends BaseSSLAndKerberosTest {
private AtlasClient dgiCLient;
private SessionState ss;
private Path jksPath;
private String providerUrl;
private TestSecureEmbeddedServer secureEmbeddedServer;
......@@ -78,47 +58,22 @@ public class SSLHiveHookIT {
}
@Override
public PropertiesConfiguration getConfiguration() {
public org.apache.commons.configuration.Configuration getConfiguration() {
return super.getConfiguration();
}
}
@BeforeClass
public void setUp() throws Exception {
//Set-up hive session
HiveConf conf = getHiveConf();
driver = new Driver(conf);
ss = new SessionState(conf, System.getProperty("user.name"));
ss = SessionState.start(ss);
SessionState.setCurrentSessionState(ss);
jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
String persistDir = null;
URL resource = SSLHiveHookIT.class.getResource("/");
if (resource != null) {
persistDir = resource.toURI().getPath();
}
// delete prior ssl-client.xml file
resource = SSLHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
if (resource != null) {
File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
if (sslClientFile != null && sslClientFile.exists()) {
sslClientFile.delete();
}
}
setupCredentials();
String persistDir = TestUtils.getTempDirectory();
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(TLS_ENABLED, true);
configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY,
SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
setupCredentials();
configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
TestUtils.writeConfiguration(configuration, persistDir + File.separator + "client.properties");
dgiCLient = new AtlasClient(DGI_URL) {
@Override
......@@ -127,19 +82,13 @@ public class SSLHiveHookIT {
}
};
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/apache-atlas") {
secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {
@Override
public PropertiesConfiguration getConfiguration() {
return configuration;
}
};
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.getServer().setHandler(webapp);
secureEmbeddedServer.getServer().start();
}
@AfterClass
......@@ -181,34 +130,8 @@ public class SSLHiveHookIT {
}
}
protected String getWarPath() {
return String.format("/../../webapp/target/atlas-webapp-%s",
System.getProperty("project.version"));
}
private HiveConf getHiveConf() {
return HiveHookIT.createHiveConf(DGI_URL);
}
private void runCommand(String cmd) throws Exception {
ss.setCommandType(null);
driver.run(cmd);
}
@Test
public void testCreateDatabase() throws Exception {
String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase();
runCommand("create database " + dbName);
assertDatabaseIsRegistered(dbName);
}
private void assertDatabaseIsRegistered(String dbName) throws Exception {
assertInstanceIsRegistered(HiveDataTypes.HIVE_DB.getName(), "name", dbName);
}
private void assertInstanceIsRegistered(String typeName, String colName, String colValue) throws Exception {
JSONArray results = dgiCLient.rawSearch(typeName, colName, colValue);
Assert.assertEquals(results.length(), 1);
}
public void testService() throws Exception {
dgiCLient.listTypes();
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment