Skip to content

A
atlas
Commit cb5ceb13 by Jon Maron
Options

Merge branch 'BUG-32837'

merge of the fixes for enabling SSL and leveraging the Credential Providre Utility from hadoop-common
parents af99895b b91dcefd
Showing with 879 additions and 131 deletions
pom.xml
......@@ -79,7 +79,7 @@
<jersey.version>1.9</jersey.version>
<tinkerpop.version>2.5.0</tinkerpop.version>
<titan.version>0.5.3</titan.version>
<hadoop.version>2.5.0</hadoop.version>
<hadoop.version>2.6.0</hadoop.version>
<!-- scala versions -->
<scala.version>2.10.4</scala.version>
......@@ -915,7 +915,7 @@
<version>2.16</version>
<configuration>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.security.krb5.realm= -Djava.security.krb5.kdc=
<argLine>-Dproject.version=${project.version}
-Dhadoop.tmp.dir=${project.build.directory}/tmp-hadoop-${user.name}
</argLine>
<parallel>none</parallel>
......
src/bin/cputil.sh 0 → 100755
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
#
DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
source ${DIR}/metadata-config.sh
${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${METADATACPPATH} org.apache.hadoop.metadata.util.CredentialProviderUtility
src/bin/metadata-config.sh 0 → 100644
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
#
# resolve links - $0 may be a softlink
PRG="${0}"
while [ -h "${PRG}" ]; do
ls=`ls -ld "${PRG}"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "${PRG}"`/"$link"
fi
done
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`
if [ -z "$METADATA_CONF" ]; then
METADATA_CONF=${BASEDIR}/conf
fi
export METADATA_CONF
if [ -f "${METADATA_CONF}/metadata-env.sh" ]; then
. "${METADATA_CONF}/metadata-env.sh"
fi
if test -z ${JAVA_HOME}
then
JAVA_BIN=`which java`
JAR_BIN=`which jar`
else
JAVA_BIN=${JAVA_HOME}/bin/java
JAR_BIN=${JAVA_HOME}/bin/jar
fi
export JAVA_BIN
if [ ! -e $JAVA_BIN ] || [ ! -e $JAR_BIN ]; then
echo "$JAVA_BIN and/or $JAR_BIN not found on the system. Please make sure java and jar commands are available."
exit 1
fi
# default the heap size to 1GB
DEFAULT_JAVA_HEAP_MAX=-Xmx1024m
METADATA_OPTS="$DEFAULT_JAVA_HEAP_MAX $METADATA_OPTS"
METADATACPPATH="$METADATA_CONF"
METADATA_EXPANDED_WEBAPP_DIR=${METADATA_EXPANDED_WEBAPP_DIR:-${BASEDIR}/server/webapp}
export METADATA_EXPANDED_WEBAPP_DIR
# set the server classpath
if [ ! -d ${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF ]; then
mkdir -p ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
cd ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
$JAR_BIN -xf ${BASEDIR}/server/webapp/metadata.war
cd -
fi
METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/classes"
METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/lib/*:${BASEDIR}/libext/*"
# log and pid dirs for applications
METADATA_LOG_DIR="${METADATA_LOG_DIR:-$BASEDIR/logs}"
export METADATA_LOG_DIR
METADATA_PID_DIR="${METADATA_PID_DIR:-$BASEDIR/logs}"
# create the pid dir if its not there
[ -w "$METADATA_PID_DIR" ] || mkdir -p "$METADATA_PID_DIR"
export METADATA_PID_DIR
METADATA_PID_FILE=${METADATA_PID_DIR}/metadata.pid
export METADATA_PID_FILE
METADATA_DATA_DIR=${METADATA_DATA_DIR:-${BASEDIR}/data}
METADATA_HOME_DIR="${METADATA_HOME_DIR:-$BASEDIR}"
export METADATA_HOME_DIR
# make sure the process is not running
if [ -f $METADATA_PID_FILE ]; then
if kill -0 `cat $METADATA_PID_FILE` > /dev/null 2>&1; then
echo metadata running as process `cat $METADATA_PID_FILE`. Stop it first.
exit 1
fi
fi
mkdir -p $METADATA_LOG_DIR
pushd ${BASEDIR} > /dev/null
JAVA_PROPERTIES="$METADATA_OPTS $METADATA_PROPERTIES -Dmetadata.log.dir=$METADATA_LOG_DIR -Dmetadata.home=${METADATA_HOME_DIR}"
shift
while [[ ${1} =~ ^\-D ]]; do
JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
shift
done
TIME=`date +%Y%m%d%H%M%s`
src/bin/metadata-start.sh
......@@ -13,100 +13,8 @@
# limitations under the License. See accompanying LICENSE file.
#
# resolve links - $0 may be a softlink
PRG="${0}"
while [ -h "${PRG}" ]; do
ls=`ls -ld "${PRG}"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "${PRG}"`/"$link"
fi
done
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`
if [ -z "$METADATA_CONF" ]; then
METADATA_CONF=${BASEDIR}/conf
fi
export METADATA_CONF
if [ -f "${METADATA_CONF}/metadata-env.sh" ]; then
. "${METADATA_CONF}/metadata-env.sh"
fi
if test -z ${JAVA_HOME}
then
JAVA_BIN=`which java`
JAR_BIN=`which jar`
else
JAVA_BIN=${JAVA_HOME}/bin/java
JAR_BIN=${JAVA_HOME}/bin/jar
fi
export JAVA_BIN
if [ ! -e $JAVA_BIN ] || [ ! -e $JAR_BIN ]; then
echo "$JAVA_BIN and/or $JAR_BIN not found on the system. Please make sure java and jar commands are available."
exit 1
fi
# default the heap size to 1GB
DEFAULT_JAVA_HEAP_MAX=-Xmx1024m
METADATA_OPTS="$DEFAULT_JAVA_HEAP_MAX $METADATA_OPTS"
METADATACPPATH="$METADATA_CONF"
METADATA_EXPANDED_WEBAPP_DIR=${METADATA_EXPANDED_WEBAPP_DIR:-${BASEDIR}/server/webapp}
export METADATA_EXPANDED_WEBAPP_DIR
# set the server classpath
if [ ! -d ${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF ]; then
mkdir -p ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
cd ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
$JAR_BIN -xf ${BASEDIR}/server/webapp/metadata.war
cd -
fi
METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/classes"
METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/lib/*:${BASEDIR}/libext/*"
# log and pid dirs for applications
METADATA_LOG_DIR="${METADATA_LOG_DIR:-$BASEDIR/logs}"
export METADATA_LOG_DIR
METADATA_PID_DIR="${METADATA_PID_DIR:-$BASEDIR/logs}"
# create the pid dir if its not there
[ -w "$METADATA_PID_DIR" ] || mkdir -p "$METADATA_PID_DIR"
export METADATA_PID_DIR
METADATA_PID_FILE=${METADATA_PID_DIR}/metadata.pid
export METADATA_PID_FILE
METADATA_DATA_DIR=${METADATA_DATA_DIR:-${BASEDIR}/data}
METADATA_HOME_DIR="${METADATA_HOME_DIR:-$BASEDIR}"
export METADATA_HOME_DIR
# make sure the process is not running
if [ -f $METADATA_PID_FILE ]; then
if kill -0 `cat $METADATA_PID_FILE` > /dev/null 2>&1; then
echo metadata running as process `cat $METADATA_PID_FILE`. Stop it first.
exit 1
fi
fi
mkdir -p $METADATA_LOG_DIR
pushd ${BASEDIR} > /dev/null
JAVA_PROPERTIES="$METADATA_OPTS $METADATA_PROPERTIES -Dmetadata.log.dir=$METADATA_LOG_DIR -Dmetadata.home=${METADATA_HOME_DIR}"
shift
while [[ ${1} =~ ^\-D ]]; do
JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
shift
done
TIME=`date +%Y%m%d%H%M%s`
DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
source ${DIR}/metadata-config.sh
nohup ${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${METADATACPPATH} org.apache.hadoop.metadata.Main -app ${BASEDIR}/server/webapp/metadata $* > "${METADATA_LOG_DIR}/metadata-server.$TIME.out" 2>&1 &
echo $! > $METADATA_PID_FILE
......
webapp/pom.xml
......@@ -54,6 +54,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
......@@ -209,11 +215,12 @@
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/webapps</outputDirectory>
<outputDirectory>${basedir}/conf
</outputDirectory>
<resources>
<resource>
<directory>src/conf</directory>
<filtering>true</filtering>
<directory>${project.build.directory}/conf</directory>
<filtering>false</filtering>
</resource>
</resources>
</configuration>
......@@ -250,7 +257,7 @@
</goals>
</execution>
<execution>
<id>genkey</id>
<id>generateKeyPair</id>
<phase>generate-resources</phase>
<goals>
<goal>generateKeyPair</goal>
......@@ -260,8 +267,8 @@
<configuration>
<dname>cn=metadata.incubator.apache.org</dname>
<keystore>${project.build.directory}/metadata.keystore</keystore>
<keypass>metadata-passwd</keypass>
<storepass>metadata-passwd</storepass>
<keypass>keypass</keypass>
<storepass>keypass</storepass>
<alias>metadata</alias>
<keyalg>RSA</keyalg>
<validity>100000</validity>
......@@ -285,14 +292,12 @@
<password>metadata-passwd</password>
</connector>
-->
<connector
implementation="org.mortbay.jetty.nio.SelectChannelConnector">
<connector implementation="org.mortbay.jetty.nio.SelectChannelConnector">
<port>21000</port>
<maxIdleTime>60000</maxIdleTime>
</connector>
</connectors>
<webApp>${project.build.directory}/metadata-webapp-${project.version}
</webApp>
<webApp>${project.build.directory}/metadata-webapp-${project.version}</webApp>
<contextPath>/</contextPath>
<useTestClasspath>true</useTestClasspath>
<systemProperties>
......@@ -302,15 +307,12 @@
</systemProperty>
<systemProperty>
<name>keystore.file</name>
<value>
${project.build.directory}/../../webapp/target/metadata.keystore
<value>${project.build.directory}/../../webapp/target/metadata.keystore
</value>
</systemProperty>
<systemProperty>
<name>truststore.file</name>
<value>
${project.build.directory}/../../webapp/target/metadata.keystore
</value>
<value>${project.build.directory}/../../webapp/target/metadata.keystore</value>
</systemProperty>
</systemProperties>
<stopKey>metadata-stop</stopKey>
......
webapp/src/main/java/org/apache/hadoop/metadata/util/CredentialProviderUtility.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.util;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import java.io.*;
import java.util.Arrays;
/**
* A utility class for generating a credential provider containing the entries required for supporting the SSL implementation
* of the DGC server.
*/
public class CredentialProviderUtility {
private static final String[] KEYS = new String[] {SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY,
SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY, SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY};
public static abstract class TextDevice {
public abstract void printf(String fmt, Object... params);
public abstract String readLine(String fmt, Object ... args);
public abstract char[] readPassword(String fmt, Object ... args);
}
private static TextDevice DEFAULT_TEXT_DEVICE = new TextDevice() {
Console console = System.console();
@Override
public void printf(String fmt, Object... params) {
console.printf(fmt, params);
}
@Override
public String readLine(String fmt, Object ... args) {
return console.readLine(fmt, args);
}
@Override
public char[] readPassword(String fmt, Object ... args) {
return console.readPassword(fmt, args);
}
};
public static TextDevice textDevice = DEFAULT_TEXT_DEVICE;
public static void main(String[] args) throws IOException {
// prompt for the provider name
CredentialProvider provider = getCredentialProvider(textDevice);
char[] cred;
for (String key : KEYS) {
cred = getPassword(textDevice, key);
// create a credential entry and store it
boolean overwrite = true;
if (provider.getCredentialEntry(key) != null) {
String choice = textDevice.readLine("Entry for %s already exists. Overwrite? (y/n) [y]:", key);
overwrite = StringUtils.isEmpty(choice) || choice.equalsIgnoreCase("y");
if (overwrite) {
provider.deleteCredentialEntry(key);
provider.flush();
provider.createCredentialEntry(key, cred);
provider.flush();
textDevice.printf("Entry for %s was overwritten with the new value.\n", key);
} else {
textDevice.printf("Entry for %s was not overwritten.\n", key);
}
} else {
provider.createCredentialEntry(key, cred);
provider.flush();
}
}
}
/**
* Retrieves a password from the command line.
* @param textDevice the system console.
* @param key the password key/alias.
* @return the password.
*/
private static char[] getPassword(TextDevice textDevice, String key) {
boolean noMatch;
char[] cred = new char[0];
char[] passwd1;
char[] passwd2;
do {
passwd1 = textDevice.readPassword("Please enter the password value for %s:", key);
passwd2 = textDevice.readPassword("Please enter the password value for %s again:", key);
noMatch = !Arrays.equals(passwd1, passwd2);
if (noMatch) {
if (passwd1 != null) Arrays.fill(passwd1, ' ');
textDevice.printf("Password entries don't match. Please try again.\n");
} else {
if (passwd1.length == 0) {
textDevice.printf("An empty password is not valid. Please try again.\n");
noMatch = true;
} else {
cred = passwd1;
}
}
if (passwd2 != null) Arrays.fill(passwd2, ' ');
} while (noMatch);
return cred;
}
/**\
* Returns a credential provider for the entered JKS path.
* @param textDevice the system console.
* @return the Credential provider
* @throws IOException
*/
private static CredentialProvider getCredentialProvider(TextDevice textDevice) throws IOException {
String providerPath = textDevice.readLine("Please enter the full path to the credential provider:");
File file = new File(providerPath);
if (file.exists()) {
textDevice.printf("%s already exists. You will need to specify whether existing entries should be overwritten " +
"(default is 'yes')\n", providerPath);
}
String providerURI = JavaKeyStoreProvider.SCHEME_NAME + "://file" + providerPath;
Configuration conf = new Configuration(false);
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerURI);
return CredentialProviderFactory.getProviders(conf).get(0);
}
}
webapp/src/main/java/org/apache/hadoop/metadata/web/service/EmbeddedServer.java
......@@ -25,6 +25,8 @@ import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.webapp.WebAppContext;
import java.io.IOException;
/**
* This class embeds a Jetty server and a connector.
*/
......@@ -33,7 +35,7 @@ public class EmbeddedServer {
protected final Server server = new Server();
public EmbeddedServer(int port, String path) {
public EmbeddedServer(int port, String path) throws IOException {
Connector connector = getConnector(port);
server.addConnector(connector);
......@@ -41,7 +43,7 @@ public class EmbeddedServer {
server.setHandler(application);
}
public static EmbeddedServer newServer(int port, String path, boolean secure) {
public static EmbeddedServer newServer(int port, String path, boolean secure) throws IOException {
if (secure) {
return new SecureEmbeddedServer(port, path);
} else {
......@@ -49,7 +51,7 @@ public class EmbeddedServer {
}
}
protected Connector getConnector(int port) {
protected Connector getConnector(int port) throws IOException {
Connector connector = new SocketConnector();
connector.setPort(port);
connector.setHost("0.0.0.0");
......
webapp/src/main/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServer.java
......@@ -20,43 +20,97 @@ package org.apache.hadoop.metadata.web.service;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.security.SslSocketConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
* This is a jetty server which requires client auth via certificates.
*/
public class SecureEmbeddedServer extends EmbeddedServer {
public SecureEmbeddedServer(int port, String path) {
public static final String KEYSTORE_FILE_KEY = "keystore.file";
public static final String DEFAULT_KEYSTORE_FILE_LOCATION = "target/metadata.keystore";
public static final String KEYSTORE_PASSWORD_KEY = "keystore.password";
public static final String TRUSTSTORE_FILE_KEY = "truststore.file";
public static final String DEFATULT_TRUSTORE_FILE_LOCATION = "target/metadata.keystore";
public static final String TRUSTSTORE_PASSWORD_KEY = "truststore.password";
public static final String SERVER_CERT_PASSWORD_KEY = "password";
public static final String CLIENT_AUTH_KEY = "client.auth.enabled";
public static final String CERT_STORES_CREDENTIAL_PROVIDER_PATH = "cert.stores.credential.provider.path";
private static final Logger LOG = LoggerFactory.getLogger(SecureEmbeddedServer.class);
public SecureEmbeddedServer(int port, String path) throws IOException {
super(port, path);
}
protected Connector getConnector(int port) {
protected Connector getConnector(int port) throws IOException {
PropertiesConfiguration config = getConfiguration();
SslSocketConnector connector = new SslSocketConnector();
connector.setPort(port);
connector.setHost("0.0.0.0");
connector.setKeystore(config.getString("keystore.file",
System.getProperty("keystore.file", "conf/metadata.keystore")));
connector.setKeyPassword(config.getString("keystore.password",
System.getProperty("keystore.password", "metadata-passwd")));
connector.setTruststore(config.getString("truststore.file",
System.getProperty("truststore.file", "conf/metadata.keystore")));
connector.setTrustPassword(config.getString("truststore.password",
System.getProperty("truststore.password", "metadata-passwd")));
connector.setPassword(config.getString("password",
System.getProperty("password", "metadata-passwd")));
connector.setWantClientAuth(true);
connector.setKeystore(config.getString(KEYSTORE_FILE_KEY,
System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
connector.setKeyPassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY,
System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
connector.setPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
return connector;
}
private PropertiesConfiguration getConfiguration() {
/**
* Retrieves a password from a configured credential provider or prompts for the password and stores it in the
* configured credential provider.
* @param config application configuration
* @param key the key/alias for the password.
* @return the password.
* @throws IOException
*/
private String getPassword(PropertiesConfiguration config, String key) throws IOException {
String password = null;
String provider = config.getString(CERT_STORES_CREDENTIAL_PROVIDER_PATH);
if (provider != null) {
LOG.info("Attempting to retrieve password from configured credential provider path");
Configuration c = new Configuration();
c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
CredentialProvider credentialProvider =
CredentialProviderFactory.getProviders(c).get(0);
CredentialProvider.CredentialEntry entry = credentialProvider.getCredentialEntry(key);
if (entry == null) {
throw new IOException(String.format("No credential entry found for %s. " +
"Please create an entry in the configured credential provider", key));
} else {
password = String.valueOf(entry.getCredential());
}
} else {
throw new IOException("No credential provider path configured for storage of certificate store passwords");
}
return password;
}
/**
* Returns the application configuration.
* @return
*/
protected PropertiesConfiguration getConfiguration() {
try {
return new PropertiesConfiguration("application.properties");
} catch (ConfigurationException e) {
throw new RuntimeException("Unable to load configuration: application.properties");
}
}
}
\ No newline at end of file
}
webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metadata.util.CredentialProviderUtility;
import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.testng.annotations.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.*;
/**
*
*/
public class CredentialProviderUtilityIT {
private char[] defaultPass = new char[]{'k', 'e', 'y', 'p', 'a', 's', 's'};
@Test
public void testEnterValidValues() throws Exception {
Path testPath = null;
try {
testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
} catch (IOException e) {
e.printStackTrace();
}
new File(testPath.toUri().getPath()).delete();
final Path finalTestPath = testPath;
CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
@Override
public void printf(String fmt, Object... params) {
System.out.print(String.format(fmt, params));
}
public String readLine(String fmt, Object ... args) {
return finalTestPath.toString();
}
@Override
public char[] readPassword(String fmt, Object ... args) {
return defaultPass;
}
};
CredentialProviderUtility.main(new String[] {});
String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
Configuration conf = new Configuration(false);
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
CredentialProvider.CredentialEntry entry =
provider.getCredentialEntry(SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
entry = provider.getCredentialEntry(SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
entry = provider.getCredentialEntry(SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
}
@Test
public void testEnterEmptyValues() throws Exception {
Path testPath = null;
try {
testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
} catch (IOException e) {
e.printStackTrace();
}
new File(testPath.toUri().getPath()).delete();
final Path finalTestPath = testPath;
CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
private Random random = new Random();
@Override
public void printf(String fmt, Object... params) {
System.out.print(String.format(fmt, params));
}
public String readLine(String fmt, Object ... args) {
return finalTestPath.toString();
}
@Override
public char[] readPassword(String fmt, Object ... args) {
List<char[]> responses = new ArrayList<>();
responses.add(new char[0]);
responses.add(defaultPass);
int size = responses.size();
int item = random.nextInt(size);
return responses.get(item);
}
};
CredentialProviderUtility.main(new String[] {});
String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
Configuration conf = new Configuration(false);
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
CredentialProvider.CredentialEntry entry =
provider.getCredentialEntry(SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
entry = provider.getCredentialEntry(SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
entry = provider.getCredentialEntry(SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
}
@Test
public void testEnterMismatchedValues() throws Exception {
Path testPath = null;
try {
testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
} catch (IOException e) {
e.printStackTrace();
}
new File(testPath.toUri().getPath()).delete();
final Path finalTestPath = testPath;
CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
int i = 0;
@Override
public void printf(String fmt, Object... params) {
System.out.print(String.format(fmt, params));
}
public String readLine(String fmt, Object ... args) {
return finalTestPath.toString();
}
@Override
public char[] readPassword(String fmt, Object ... args) {
List<char[]> responses = new ArrayList<>();
responses.add(defaultPass);
responses.add(new char[] {'b', 'a', 'd', 'p', 'a', 's', 's'});
responses.add(defaultPass);
int item = i % 3;
i++;
return responses.get(item);
}
};
CredentialProviderUtility.main(new String[] {});
String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
Configuration conf = new Configuration(false);
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
CredentialProvider.CredentialEntry entry =
provider.getCredentialEntry(SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
entry = provider.getCredentialEntry(SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
entry = provider.getCredentialEntry(SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
}
@Test
public void testOverwriteValues() throws Exception {
Path testPath = null;
try {
testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
} catch (IOException e) {
e.printStackTrace();
}
new File(testPath.toUri().getPath()).delete();
final Path finalTestPath = testPath;
CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
@Override
public void printf(String fmt, Object... params) {
System.out.print(String.format(fmt, params));
}
public String readLine(String fmt, Object ... args) {
return finalTestPath.toString();
}
@Override
public char[] readPassword(String fmt, Object ... args) {
return defaultPass;
}
};
CredentialProviderUtility.main(new String[] {});
// now attempt to overwrite values
CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
int i = 0;
@Override
public void printf(String fmt, Object... params) {
System.out.print(String.format(fmt, params));
}
public String readLine(String fmt, Object ... args) {
return i++ == 0 ? finalTestPath.toString() : "y";
}
@Override
public char[] readPassword(String fmt, Object ... args) {
return new char[] {'n', 'e', 'w', 'p', 'a', 's', 's'};
}
};
CredentialProviderUtility.main(new String[] {});
String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
Configuration conf = new Configuration(false);
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
char[] newpass = "newpass".toCharArray();
CredentialProvider.CredentialEntry entry =
provider.getCredentialEntry(SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), newpass);
entry = provider.getCredentialEntry(SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), newpass);
entry = provider.getCredentialEntry(SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY);
assert entry != null && Arrays.equals(entry.getCredential(), newpass);
}
}
webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java
......@@ -29,6 +29,7 @@ import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
......@@ -43,9 +44,10 @@ public abstract class BaseResourceIT {
protected WebResource service;
protected MetadataServiceClient serviceClient;
public static String baseUrl = "http://localhost:21000/";;
@BeforeClass
public void setUp() throws Exception {
String baseUrl = "http://localhost:21000/";
DefaultClientConfig config = new DefaultClientConfig();
Client client = Client.create(config);
......
webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerIT.java 0 → 100644
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.web.service;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metadata.web.resources.*;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.mortbay.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.TestListenerAdapter;
import org.testng.TestNG;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import javax.ws.rs.core.UriBuilder;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.util.List;
/**
*
*/
public class SecureEmbeddedServerIT {
private SecureEmbeddedServer secureEmbeddedServer;
private String providerUrl;
private Path jksPath;
protected WebResource service;
static {
//for localhost testing only
javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
new javax.net.ssl.HostnameVerifier(){
public boolean verify(String hostname,
javax.net.ssl.SSLSession sslSession) {
if (hostname.equals("localhost")) {
return true;
}
return false;
}
});
System.setProperty("javax.net.ssl.trustStore", SecureEmbeddedServer.DEFAULT_KEYSTORE_FILE_LOCATION);
System.setProperty("javax.net.ssl.trustStorePassword", "keypass");
System.setProperty("javax.net.ssl.trustStoreType", "JKS");
}
@BeforeClass
public void setupServerURI () throws Exception {
BaseResourceIT.baseUrl = "https://localhost:21443";
}
@AfterClass
public void resetServerURI() throws Exception {
BaseResourceIT.baseUrl = "http://localhost:21000";
}
@BeforeMethod
public void setup() throws Exception {
jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
String baseUrl = "https://localhost:21443/";
DefaultClientConfig config = new DefaultClientConfig();
Client client = Client.create(config);
client.resource(UriBuilder.fromUri(baseUrl).build());
service = client.resource(UriBuilder.fromUri(baseUrl).build());
}
@Test
public void testNoConfiguredCredentialProvider() throws Exception {
try {
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance");
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.server.setHandler(webapp);
secureEmbeddedServer.server.start();
Assert.fail("Should have thrown an exception");
} catch (IOException e) {
assert e.getMessage().equals("No credential provider path configured for storage of certificate store passwords");
}
}
@Test
public void testServerConfiguredUsingCredentialProvider() throws Exception {
// setup the configuration
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
// setup the credential provider
setupCredentials();
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
@Override
protected PropertiesConfiguration getConfiguration() {
return configuration;
}
};
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.server.setHandler(webapp);
secureEmbeddedServer.server.start();
URL url = new URL("https://localhost:21443/");
HttpURLConnection connection = (HttpURLConnection)url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
// test to see whether server is up and root page can be served
assert connection.getResponseCode() == 200;
secureEmbeddedServer.server.stop();
}
@Test
public void testMissingEntriesInCredentialProvider() throws Exception {
// setup the configuration
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
try {
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
@Override
protected PropertiesConfiguration getConfiguration() {
return configuration;
}
};
Assert.fail("No entries should generate an exception");
} catch (IOException e) {
assert e.getMessage().startsWith("No credential entry found for");
}
}
/**
* Runs the existing webapp test cases, this time against the initiated secure server instance.
* @throws Exception
*/
@Test
public void runOtherSuitesAgainstSecureServer() throws Exception {
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
// setup the credential provider
setupCredentials();
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
@Override
protected PropertiesConfiguration getConfiguration() {
return configuration;
}
};
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.server.setHandler(webapp);
secureEmbeddedServer.server.start();
TestListenerAdapter tla = new TestListenerAdapter();
TestNG testng = new TestNG();
testng.setTestClasses(new Class[] { AdminJerseyResourceIT.class, EntityJerseyResourceIT.class,
MetadataDiscoveryJerseyResourceIT.class, RexsterGraphJerseyResourceIT.class, TypesJerseyResourceIT.class});
testng.addListener(tla);
testng.run();
secureEmbeddedServer.server.stop();
}
protected String getWarPath() {
return String.format("/target/metadata-webapp-%s.war",
System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
}
private void setupCredentials()
throws Exception {
Configuration conf = new Configuration(false);
File file = new File(jksPath.toUri().getPath());
file.delete();
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
// create new aliases
try {
char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY, storepass);
char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY, trustpass);
char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY, certpass);
// write out so that it can be found in checks
provider.flush();
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment