BUG-32837 enhance DGI ssl

pom.xml

@@ -79,7 +79,7 @@
         <jersey.version>1.9</jersey.version>
         <tinkerpop.version>2.5.0</tinkerpop.version>
         <titan.version>0.5.3</titan.version>
-        <hadoop.version>2.5.0</hadoop.version>
+        <hadoop.version>2.6.0</hadoop.version>
 
         <!-- scala versions -->
         <scala.version>2.10.4</scala.version>
@@ -915,7 +915,7 @@
                 <version>2.16</version>
                 <configuration>
                     <redirectTestOutputToFile>true</redirectTestOutputToFile>
-                    <argLine>-Djava.security.krb5.realm= -Djava.security.krb5.kdc=
+                    <argLine>-Dproject.version=${project.version}
                         -Dhadoop.tmp.dir=${project.build.directory}/tmp-hadoop-${user.name}
                     </argLine>
                     <parallel>none</parallel>

src/bin/cputil.sh

+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+#
+
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${DIR}/metadata-config.sh
+
+${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${METADATACPPATH} org.apache.hadoop.metadata.util.CredentialProviderUtility

src/bin/metadata-config.sh

+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+#
+
+# resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+  ls=`ls -ld "${PRG}"`
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    PRG="$link"
+  else
+    PRG=`dirname "${PRG}"`/"$link"
+  fi
+done
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR}/..;pwd`
+
+if [ -z "$METADATA_CONF" ]; then
+  METADATA_CONF=${BASEDIR}/conf
+fi
+export METADATA_CONF
+
+if [ -f "${METADATA_CONF}/metadata-env.sh" ]; then
+  . "${METADATA_CONF}/metadata-env.sh"
+fi
+
+if test -z ${JAVA_HOME}
+then
+    JAVA_BIN=`which java`
+    JAR_BIN=`which jar`
+else
+    JAVA_BIN=${JAVA_HOME}/bin/java
+    JAR_BIN=${JAVA_HOME}/bin/jar
+fi
+export JAVA_BIN
+
+if [ ! -e $JAVA_BIN ] || [ ! -e $JAR_BIN ]; then
+  echo "$JAVA_BIN and/or $JAR_BIN not found on the system. Please make sure java and jar commands are available."
+  exit 1
+fi
+
+# default the heap size to 1GB
+DEFAULT_JAVA_HEAP_MAX=-Xmx1024m
+METADATA_OPTS="$DEFAULT_JAVA_HEAP_MAX $METADATA_OPTS"
+
+METADATACPPATH="$METADATA_CONF"
+
+METADATA_EXPANDED_WEBAPP_DIR=${METADATA_EXPANDED_WEBAPP_DIR:-${BASEDIR}/server/webapp}
+export METADATA_EXPANDED_WEBAPP_DIR
+# set the server classpath
+if [ ! -d ${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF ]; then
+  mkdir -p ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
+  cd ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
+  $JAR_BIN -xf ${BASEDIR}/server/webapp/metadata.war
+  cd -
+fi
+
+METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/classes"
+METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/lib/*:${BASEDIR}/libext/*"
+
+# log and pid dirs for applications
+METADATA_LOG_DIR="${METADATA_LOG_DIR:-$BASEDIR/logs}"
+export METADATA_LOG_DIR
+
+METADATA_PID_DIR="${METADATA_PID_DIR:-$BASEDIR/logs}"
+# create the pid dir if its not there
+[ -w "$METADATA_PID_DIR" ] ||  mkdir -p "$METADATA_PID_DIR"
+export METADATA_PID_DIR
+METADATA_PID_FILE=${METADATA_PID_DIR}/metadata.pid
+export METADATA_PID_FILE
+
+METADATA_DATA_DIR=${METADATA_DATA_DIR:-${BASEDIR}/data}
+METADATA_HOME_DIR="${METADATA_HOME_DIR:-$BASEDIR}"
+export METADATA_HOME_DIR
+
+# make sure the process is not running
+if [ -f $METADATA_PID_FILE ]; then
+  if kill -0 `cat $METADATA_PID_FILE` > /dev/null 2>&1; then
+    echo metadata running as process `cat $METADATA_PID_FILE`.  Stop it first.
+    exit 1
+  fi
+fi
+
+mkdir -p $METADATA_LOG_DIR
+
+pushd ${BASEDIR} > /dev/null
+
+JAVA_PROPERTIES="$METADATA_OPTS $METADATA_PROPERTIES -Dmetadata.log.dir=$METADATA_LOG_DIR -Dmetadata.home=${METADATA_HOME_DIR}"
+shift
+
+while [[ ${1} =~ ^\-D ]]; do
+  JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
+  shift
+done
+TIME=`date +%Y%m%d%H%M%s`
+
+

src/bin/metadata-start.sh

@@ -13,100 +13,8 @@
 #  limitations under the License. See accompanying LICENSE file.
 #
 
-# resolve links - $0 may be a softlink
-PRG="${0}"
-
-while [ -h "${PRG}" ]; do
-  ls=`ls -ld "${PRG}"`
-  link=`expr "$ls" : '.*-> \(.*\)$'`
-  if expr "$link" : '/.*' > /dev/null; then
-    PRG="$link"
-  else
-    PRG=`dirname "${PRG}"`/"$link"
-  fi
-done
-
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR}/..;pwd`
-
-if [ -z "$METADATA_CONF" ]; then
-  METADATA_CONF=${BASEDIR}/conf
-fi
-export METADATA_CONF
-
-if [ -f "${METADATA_CONF}/metadata-env.sh" ]; then
-  . "${METADATA_CONF}/metadata-env.sh"
-fi
-
-if test -z ${JAVA_HOME}
-then
-    JAVA_BIN=`which java`
-    JAR_BIN=`which jar`
-else
-    JAVA_BIN=${JAVA_HOME}/bin/java
-    JAR_BIN=${JAVA_HOME}/bin/jar
-fi
-export JAVA_BIN
-
-if [ ! -e $JAVA_BIN ] || [ ! -e $JAR_BIN ]; then
-  echo "$JAVA_BIN and/or $JAR_BIN not found on the system. Please make sure java and jar commands are available."
-  exit 1
-fi
-
-# default the heap size to 1GB
-DEFAULT_JAVA_HEAP_MAX=-Xmx1024m
-METADATA_OPTS="$DEFAULT_JAVA_HEAP_MAX $METADATA_OPTS"
-
-METADATACPPATH="$METADATA_CONF"
-
-METADATA_EXPANDED_WEBAPP_DIR=${METADATA_EXPANDED_WEBAPP_DIR:-${BASEDIR}/server/webapp}
-export METADATA_EXPANDED_WEBAPP_DIR
-# set the server classpath
-if [ ! -d ${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF ]; then
-  mkdir -p ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
-  cd ${METADATA_EXPANDED_WEBAPP_DIR}/metadata
-  $JAR_BIN -xf ${BASEDIR}/server/webapp/metadata.war
-  cd -
-fi
-
-METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/classes"
-METADATACPPATH="${METADATACPPATH}:${METADATA_EXPANDED_WEBAPP_DIR}/metadata/WEB-INF/lib/*:${BASEDIR}/libext/*"
-
-# log and pid dirs for applications
-METADATA_LOG_DIR="${METADATA_LOG_DIR:-$BASEDIR/logs}"
-export METADATA_LOG_DIR
-
-METADATA_PID_DIR="${METADATA_PID_DIR:-$BASEDIR/logs}"
-# create the pid dir if its not there
-[ -w "$METADATA_PID_DIR" ] ||  mkdir -p "$METADATA_PID_DIR"
-export METADATA_PID_DIR
-METADATA_PID_FILE=${METADATA_PID_DIR}/metadata.pid
-export METADATA_PID_FILE
-
-METADATA_DATA_DIR=${METADATA_DATA_DIR:-${BASEDIR}/data}
-METADATA_HOME_DIR="${METADATA_HOME_DIR:-$BASEDIR}"
-export METADATA_HOME_DIR
-
-# make sure the process is not running
-if [ -f $METADATA_PID_FILE ]; then
-  if kill -0 `cat $METADATA_PID_FILE` > /dev/null 2>&1; then
-    echo metadata running as process `cat $METADATA_PID_FILE`.  Stop it first.
-    exit 1
-  fi
-fi
-
-mkdir -p $METADATA_LOG_DIR
-
-pushd ${BASEDIR} > /dev/null
-
-JAVA_PROPERTIES="$METADATA_OPTS $METADATA_PROPERTIES -Dmetadata.log.dir=$METADATA_LOG_DIR -Dmetadata.home=${METADATA_HOME_DIR}"
-shift
-
-while [[ ${1} =~ ^\-D ]]; do
-  JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
-  shift
-done
-TIME=`date +%Y%m%d%H%M%s`
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source ${DIR}/metadata-config.sh
 
 nohup ${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${METADATACPPATH} org.apache.hadoop.metadata.Main -app ${BASEDIR}/server/webapp/metadata $* > "${METADATA_LOG_DIR}/metadata-server.$TIME.out" 2>&1 &
 echo $! > $METADATA_PID_FILE

webapp/pom.xml

@@ -54,6 +54,12 @@
         </dependency>
 
         <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
             <groupId>joda-time</groupId>
             <artifactId>joda-time</artifactId>
         </dependency>
@@ -250,7 +256,7 @@
                         </goals>
                     </execution>
                     <execution>
-                        <id>genkey</id>
+                        <id>generateKeyPair</id>
                         <phase>generate-resources</phase>
                         <goals>
                             <goal>generateKeyPair</goal>
@@ -260,8 +266,8 @@
                 <configuration>
                     <dname>cn=metadata.incubator.apache.org</dname>
                     <keystore>${project.build.directory}/metadata.keystore</keystore>
-                    <keypass>metadata-passwd</keypass>
-                    <storepass>metadata-passwd</storepass>
+                    <keypass>keypass</keypass>
+                    <storepass>keypass</storepass>
                     <alias>metadata</alias>
                     <keyalg>RSA</keyalg>
                     <validity>100000</validity>

webapp/src/main/java/org/apache/hadoop/metadata/util/CredentialProviderUtility.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metadata.util;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+
+import java.io.*;
+import java.util.Arrays;
+
+/**
+ * A utility class for generating a credential provider containing the entries required for supporting the SSL implementation
+ * of the DGC server.
+ */
+public class CredentialProviderUtility {
+    private static final String[] KEYS = new String[] {SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY,
+            SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY, SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY};
+
+    public static abstract class TextDevice {
+        public abstract void printf(String fmt, Object... params);
+
+        public abstract String readLine(String fmt, Object ... args);
+
+        public abstract char[] readPassword(String fmt, Object ... args);
+
+    }
+
+    private static TextDevice DEFAULT_TEXT_DEVICE = new TextDevice() {
+        Console console = System.console();
+
+        @Override
+        public void printf(String fmt, Object... params) {
+            console.printf(fmt, params);
+        }
+
+        @Override
+        public String readLine(String fmt, Object ... args) {
+            return console.readLine(fmt, args);
+        }
+
+        @Override
+        public char[] readPassword(String fmt, Object ... args) {
+            return console.readPassword(fmt, args);
+        }
+    };
+
+    public static TextDevice textDevice = DEFAULT_TEXT_DEVICE;
+
+    public static void main(String[] args) throws IOException {
+        // prompt for the provider name
+        CredentialProvider provider = getCredentialProvider(textDevice);
+
+        char[] cred;
+        for (String key : KEYS) {
+            cred = getPassword(textDevice, key);
+            // create a credential entry and store it
+            boolean overwrite = true;
+            if (provider.getCredentialEntry(key) != null) {
+                String choice = textDevice.readLine("Entry for %s already exists.  Overwrite? (y/n) [y]:", key);
+                overwrite = StringUtils.isEmpty(choice) || choice.equalsIgnoreCase("y");
+                if (overwrite) {
+                    provider.deleteCredentialEntry(key);
+                    provider.flush();
+                    provider.createCredentialEntry(key, cred);
+                    provider.flush();
+                    textDevice.printf("Entry for %s was overwritten with the new value.\n", key);
+                } else {
+                    textDevice.printf("Entry for %s was not overwritten.\n", key);
+                }
+            } else {
+                provider.createCredentialEntry(key, cred);
+                provider.flush();
+            }
+        }
+    }
+
+    /**
+     * Retrieves a password from the command line.
+     * @param textDevice  the system console.
+     * @param key   the password key/alias.
+     * @return  the password.
+     */
+    private static char[] getPassword(TextDevice textDevice, String key) {
+        boolean noMatch;
+        char[] cred = new char[0];
+        char[] passwd1;
+        char[] passwd2;
+        do {
+            passwd1 = textDevice.readPassword("Please enter the password value for %s:", key);
+            passwd2 = textDevice.readPassword("Please enter the password value for %s again:", key);
+            noMatch = !Arrays.equals(passwd1, passwd2);
+            if (noMatch) {
+                if (passwd1 != null) Arrays.fill(passwd1, ' ');
+                textDevice.printf("Password entries don't match. Please try again.\n");
+            } else {
+                if (passwd1.length == 0) {
+                    textDevice.printf("An empty password is not valid.  Please try again.\n");
+                    noMatch = true;
+                } else {
+                    cred = passwd1;
+                }
+            }
+            if (passwd2 != null) Arrays.fill(passwd2, ' ');
+        } while (noMatch);
+        return cred;
+    }
+
+    /**\
+     * Returns a credential provider for the entered JKS path.
+     * @param textDevice the system console.
+     * @return  the Credential provider
+     * @throws IOException
+     */
+    private static CredentialProvider getCredentialProvider(TextDevice textDevice) throws IOException {
+        String providerPath = textDevice.readLine("Please enter the full path to the credential provider:");
+        File file = new File(providerPath);
+        if (file.exists()) {
+            textDevice.printf("%s already exists.  You will need to specify whether existing entries should be overwritten " +
+                    "(default is 'yes')\n", providerPath);
+        }
+        String providerURI = JavaKeyStoreProvider.SCHEME_NAME + "://file" + providerPath;
+        Configuration conf = new Configuration(false);
+        conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerURI);
+        return CredentialProviderFactory.getProviders(conf).get(0);
+    }
+}

webapp/src/main/java/org/apache/hadoop/metadata/web/service/EmbeddedServer.java

@@ -25,6 +25,8 @@ import org.mortbay.jetty.Server;
 import org.mortbay.jetty.bio.SocketConnector;
 import org.mortbay.jetty.webapp.WebAppContext;
 
+import java.io.IOException;
+
 /**
  * This class embeds a Jetty server and a connector.
  */
@@ -33,7 +35,7 @@ public class EmbeddedServer {
 
     protected final Server server = new Server();
 
-    public EmbeddedServer(int port, String path) {
+    public EmbeddedServer(int port, String path) throws IOException {
         Connector connector = getConnector(port);
         server.addConnector(connector);
 
@@ -41,7 +43,7 @@ public class EmbeddedServer {
         server.setHandler(application);
     }
 
-    public static EmbeddedServer newServer(int port, String path, boolean secure) {
+    public static EmbeddedServer newServer(int port, String path, boolean secure) throws IOException {
         if (secure) {
             return new SecureEmbeddedServer(port, path);
         } else {
@@ -49,7 +51,7 @@ public class EmbeddedServer {
         }
     }
 
-    protected Connector getConnector(int port) {
+    protected Connector getConnector(int port) throws IOException {
         Connector connector = new SocketConnector();
         connector.setPort(port);
         connector.setHost("0.0.0.0");

webapp/src/main/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServer.java

@@ -20,39 +20,93 @@ package org.apache.hadoop.metadata.web.service;
 
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.security.SslSocketConnector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
 
 /**
  * This is a jetty server which requires client auth via certificates.
  */
 public class SecureEmbeddedServer extends EmbeddedServer {
 
-    public SecureEmbeddedServer(int port, String path) {
+    public static final String KEYSTORE_FILE_KEY = "keystore.file";
+    public static final String DEFAULT_KEYSTORE_FILE_LOCATION = "target/metadata.keystore";
+    public static final String KEYSTORE_PASSWORD_KEY = "keystore.password";
+    public static final String TRUSTSTORE_FILE_KEY = "truststore.file";
+    public static final String DEFATULT_TRUSTORE_FILE_LOCATION = "target/metadata.keystore";
+    public static final String TRUSTSTORE_PASSWORD_KEY = "truststore.password";
+    public static final String SERVER_CERT_PASSWORD_KEY = "password";
+    public static final String CLIENT_AUTH_KEY = "client.auth.enabled";
+    public static final String CERT_STORES_CREDENTIAL_PROVIDER_PATH = "cert.stores.credential.provider.path";
+
+    private static final Logger LOG = LoggerFactory.getLogger(SecureEmbeddedServer.class);
+
+    public SecureEmbeddedServer(int port, String path) throws IOException {
         super(port, path);
     }
 
-    protected Connector getConnector(int port) {
+    protected Connector getConnector(int port) throws IOException {
         PropertiesConfiguration config = getConfiguration();
 
         SslSocketConnector connector = new SslSocketConnector();
         connector.setPort(port);
         connector.setHost("0.0.0.0");
-        connector.setKeystore(config.getString("keystore.file",
-                System.getProperty("keystore.file", "conf/metadata.keystore")));
-        connector.setKeyPassword(config.getString("keystore.password",
-                System.getProperty("keystore.password", "metadata-passwd")));
-        connector.setTruststore(config.getString("truststore.file",
-                System.getProperty("truststore.file", "conf/metadata.keystore")));
-        connector.setTrustPassword(config.getString("truststore.password",
-                System.getProperty("truststore.password", "metadata-passwd")));
-        connector.setPassword(config.getString("password",
-                System.getProperty("password", "metadata-passwd")));
-        connector.setWantClientAuth(true);
+        connector.setKeystore(config.getString(KEYSTORE_FILE_KEY,
+                System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
+        connector.setKeyPassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
+        connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY,
+                System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
+        connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
+        connector.setPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
+        connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
         return connector;
     }
 
-    private PropertiesConfiguration getConfiguration() {
+    /**
+     * Retrieves a password from a configured credential provider or prompts for the password and stores it in the
+     * configured credential provider.
+     * @param config application configuration
+     * @param key the key/alias for the password.
+     * @return  the password.
+     * @throws IOException
+     */
+    private String getPassword(PropertiesConfiguration config, String key) throws IOException {
+
+        String password = null;
+
+        String provider = config.getString(CERT_STORES_CREDENTIAL_PROVIDER_PATH);
+        if (provider != null) {
+            LOG.info("Attempting to retrieve password from configured credential provider path");
+            Configuration c = new Configuration();
+            c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
+            CredentialProvider credentialProvider =
+                    CredentialProviderFactory.getProviders(c).get(0);
+            CredentialProvider.CredentialEntry entry = credentialProvider.getCredentialEntry(key);
+            if (entry == null) {
+                throw new IOException(String.format("No credential entry found for %s. " +
+                        "Please create an entry in the configured credential provider", key));
+            } else {
+                password = String.valueOf(entry.getCredential());
+            }
+
+        } else {
+            throw new IOException("No credential provider path configured for storage of certificate store passwords");
+        }
+
+        return password;
+    }
+
+    /**
+     * Returns the application configuration.
+     * @return
+     */
+    protected PropertiesConfiguration getConfiguration() {
         try {
             return new PropertiesConfiguration("application.properties");
         } catch (ConfigurationException e) {

webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metadata;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metadata.util.CredentialProviderUtility;
+import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.*;
+
+/**
+ *
+ */
+public class CredentialProviderUtilityIT {
+
+    private char[] defaultPass = new char[]{'k', 'e', 'y', 'p', 'a', 's', 's'};
+
+    @Test
+    public void testEnterValidValues() throws Exception {
+        Path testPath = null;
+        try {
+            testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        new File(testPath.toUri().getPath()).delete();
+        final Path finalTestPath = testPath;
+        CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+            @Override
+            public void printf(String fmt, Object... params) {
+                System.out.print(String.format(fmt, params));
+            }
+
+            public String readLine(String fmt, Object ... args) {
+                return finalTestPath.toString();
+            }
+
+            @Override
+            public char[] readPassword(String fmt, Object ... args) {
+                return defaultPass;
+            }
+        };
+
+        CredentialProviderUtility.main(new String[] {});
+
+        String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
+        Configuration conf = new Configuration(false);
+
+        conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+        CredentialProvider provider =
+                CredentialProviderFactory.getProviders(conf).get(0);
+
+        CredentialProvider.CredentialEntry entry =
+                provider.getCredentialEntry(SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
+        entry = provider.getCredentialEntry(SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
+        entry = provider.getCredentialEntry(SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
+    }
+
+    @Test
+    public void testEnterEmptyValues() throws Exception {
+        Path testPath = null;
+        try {
+            testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        new File(testPath.toUri().getPath()).delete();
+        final Path finalTestPath = testPath;
+        CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+
+            private Random random = new Random();
+
+            @Override
+            public void printf(String fmt, Object... params) {
+                System.out.print(String.format(fmt, params));
+            }
+
+            public String readLine(String fmt, Object ... args) {
+                return finalTestPath.toString();
+            }
+
+            @Override
+            public char[] readPassword(String fmt, Object ... args) {
+                List<char[]> responses = new ArrayList<>();
+                responses.add(new char[0]);
+                responses.add(defaultPass);
+
+                int size = responses.size();
+                int item = random.nextInt(size);
+                return responses.get(item);
+            }
+        };
+
+        CredentialProviderUtility.main(new String[] {});
+
+        String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
+        Configuration conf = new Configuration(false);
+
+        conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+        CredentialProvider provider =
+                CredentialProviderFactory.getProviders(conf).get(0);
+
+        CredentialProvider.CredentialEntry entry =
+                provider.getCredentialEntry(SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
+        entry = provider.getCredentialEntry(SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
+        entry = provider.getCredentialEntry(SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
+    }
+
+    @Test
+    public void testEnterMismatchedValues() throws Exception {
+        Path testPath = null;
+        try {
+            testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        new File(testPath.toUri().getPath()).delete();
+        final Path finalTestPath = testPath;
+        CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+
+            int i = 0;
+            @Override
+            public void printf(String fmt, Object... params) {
+                System.out.print(String.format(fmt, params));
+            }
+
+            public String readLine(String fmt, Object ... args) {
+                return finalTestPath.toString();
+            }
+
+            @Override
+            public char[] readPassword(String fmt, Object ... args) {
+                List<char[]> responses = new ArrayList<>();
+                responses.add(defaultPass);
+                responses.add(new char[] {'b', 'a', 'd', 'p', 'a', 's', 's'});
+                responses.add(defaultPass);
+
+                int item = i % 3;
+                i++;
+                return responses.get(item);
+            }
+        };
+
+        CredentialProviderUtility.main(new String[] {});
+
+        String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
+        Configuration conf = new Configuration(false);
+
+        conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+        CredentialProvider provider =
+                CredentialProviderFactory.getProviders(conf).get(0);
+
+        CredentialProvider.CredentialEntry entry =
+                provider.getCredentialEntry(SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
+        entry = provider.getCredentialEntry(SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
+        entry = provider.getCredentialEntry(SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), defaultPass);
+    }
+
+    @Test
+    public void testOverwriteValues() throws Exception {
+        Path testPath = null;
+        try {
+            testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        new File(testPath.toUri().getPath()).delete();
+        final Path finalTestPath = testPath;
+        CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+            @Override
+            public void printf(String fmt, Object... params) {
+                System.out.print(String.format(fmt, params));
+            }
+
+            public String readLine(String fmt, Object ... args) {
+                return finalTestPath.toString();
+            }
+
+            @Override
+            public char[] readPassword(String fmt, Object ... args) {
+                return defaultPass;
+            }
+        };
+
+        CredentialProviderUtility.main(new String[] {});
+
+        // now attempt to overwrite values
+        CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+
+            int i = 0;
+
+            @Override
+            public void printf(String fmt, Object... params) {
+                System.out.print(String.format(fmt, params));
+            }
+
+            public String readLine(String fmt, Object ... args) {
+                return i++ == 0 ? finalTestPath.toString() : "y";
+            }
+
+            @Override
+            public char[] readPassword(String fmt, Object ... args) {
+                return new char[] {'n', 'e', 'w', 'p', 'a', 's', 's'};
+            }
+        };
+
+        CredentialProviderUtility.main(new String[] {});
+
+        String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
+        Configuration conf = new Configuration(false);
+
+        conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+        CredentialProvider provider =
+                CredentialProviderFactory.getProviders(conf).get(0);
+
+        char[] newpass = "newpass".toCharArray();
+        CredentialProvider.CredentialEntry entry =
+                provider.getCredentialEntry(SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), newpass);
+        entry = provider.getCredentialEntry(SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), newpass);
+        entry = provider.getCredentialEntry(SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY);
+        assert entry != null && Arrays.equals(entry.getCredential(), newpass);
+    }
+}

webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java

@@ -29,6 +29,7 @@ import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
 import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
 import org.codehaus.jettison.json.JSONObject;
 import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
 
 import javax.ws.rs.HttpMethod;
 import javax.ws.rs.core.MediaType;
@@ -43,9 +44,10 @@ public abstract class BaseResourceIT {
 
     protected WebResource service;
     protected MetadataServiceClient serviceClient;
+    public static String baseUrl = "http://localhost:21000/";;
 
+    @BeforeClass
     public void setUp() throws Exception {
-        String baseUrl = "http://localhost:21000/";
 
         DefaultClientConfig config = new DefaultClientConfig();
         Client client = Client.create(config);

webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerIT.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metadata.web.service;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metadata.web.resources.*;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.testng.Assert;
+import org.testng.TestListenerAdapter;
+import org.testng.TestNG;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.core.UriBuilder;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.Files;
+import java.util.List;
+
+/**
+ *
+ */
+public class SecureEmbeddedServerIT {
+
+
+    private SecureEmbeddedServer secureEmbeddedServer;
+    private String providerUrl;
+    private Path jksPath;
+    protected WebResource service;
+
+    static {
+        //for localhost testing only
+        javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
+                new javax.net.ssl.HostnameVerifier(){
+
+                    public boolean verify(String hostname,
+                                          javax.net.ssl.SSLSession sslSession) {
+                        if (hostname.equals("localhost")) {
+                            return true;
+                        }
+                        return false;
+                    }
+                });
+        System.setProperty("javax.net.ssl.trustStore", SecureEmbeddedServer.DEFAULT_KEYSTORE_FILE_LOCATION);
+        System.setProperty("javax.net.ssl.trustStorePassword", "keypass");
+        System.setProperty("javax.net.ssl.trustStoreType", "JKS");
+
+
+    }
+
+    @BeforeClass
+    public void setupServerURI () throws Exception {
+        BaseResourceIT.baseUrl = "https://localhost:21443";
+    }
+
+    @AfterClass
+    public void resetServerURI() throws Exception {
+        BaseResourceIT.baseUrl = "http://localhost:21000";
+    }
+
+    @BeforeMethod
+    public void setup() throws Exception {
+        jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+        providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+        String baseUrl = "https://localhost:21443/";
+
+        DefaultClientConfig config = new DefaultClientConfig();
+        Client client = Client.create(config);
+        client.resource(UriBuilder.fromUri(baseUrl).build());
+
+        service = client.resource(UriBuilder.fromUri(baseUrl).build());
+    }
+
+    @Test
+    public void testNoConfiguredCredentialProvider() throws Exception {
+
+        try {
+            secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance");
+            WebAppContext webapp = new WebAppContext();
+            webapp.setContextPath("/");
+            webapp.setWar(System.getProperty("user.dir") + getWarPath());
+            secureEmbeddedServer.server.setHandler(webapp);
+
+            secureEmbeddedServer.server.start();
+
+            Assert.fail("Should have thrown an exception");
+        } catch (IOException e) {
+            assert e.getMessage().equals("No credential provider path configured for storage of certificate store passwords");
+        }
+    }
+
+    @Test
+    public void testServerConfiguredUsingCredentialProvider() throws Exception {
+        // setup the configuration
+        final PropertiesConfiguration configuration = new PropertiesConfiguration();
+        configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
+        // setup the credential provider
+        setupCredentials();
+
+        secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
+            @Override
+            protected PropertiesConfiguration getConfiguration() {
+                return configuration;
+            }
+        };
+        WebAppContext webapp = new WebAppContext();
+        webapp.setContextPath("/");
+        webapp.setWar(System.getProperty("user.dir") + getWarPath());
+        secureEmbeddedServer.server.setHandler(webapp);
+
+        secureEmbeddedServer.server.start();
+
+        URL url = new URL("https://localhost:21443/");
+        HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+        connection.setRequestMethod("GET");
+        connection.connect();
+
+        // test to see whether server is up and root page can be served
+        assert connection.getResponseCode() == 200;
+        secureEmbeddedServer.server.stop();
+
+    }
+
+    @Test
+    public void testMissingEntriesInCredentialProvider() throws Exception {
+        // setup the configuration
+        final PropertiesConfiguration configuration = new PropertiesConfiguration();
+        configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+        try {
+            secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
+                @Override
+                protected PropertiesConfiguration getConfiguration() {
+                    return configuration;
+                }
+            };
+            Assert.fail("No entries should generate an exception");
+        } catch (IOException e) {
+            assert e.getMessage().startsWith("No credential entry found for");
+        }
+
+    }
+
+    /**
+     * Runs the existing webapp test cases, this time against the initiated secure server instance.
+     * @throws Exception
+     */
+    @Test
+    public void runOtherSuitesAgainstSecureServer() throws Exception {
+        final PropertiesConfiguration configuration = new PropertiesConfiguration();
+        configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
+        // setup the credential provider
+        setupCredentials();
+
+        secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
+            @Override
+            protected PropertiesConfiguration getConfiguration() {
+                return configuration;
+            }
+        };
+        WebAppContext webapp = new WebAppContext();
+        webapp.setContextPath("/");
+        webapp.setWar(System.getProperty("user.dir") + getWarPath());
+        secureEmbeddedServer.server.setHandler(webapp);
+
+        secureEmbeddedServer.server.start();
+
+        TestListenerAdapter tla = new TestListenerAdapter();
+        TestNG testng = new TestNG();
+        testng.setTestClasses(new Class[] { AdminJerseyResourceIT.class, EntityJerseyResourceIT.class,
+                MetadataDiscoveryJerseyResourceIT.class, RexsterGraphJerseyResourceIT.class, TypesJerseyResourceIT.class});
+        testng.addListener(tla);
+        testng.run();
+
+        secureEmbeddedServer.server.stop();
+
+    }
+
+    protected String getWarPath() {
+        return String.format("/target/metadata-webapp-%s.war",
+                System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
+    }
+
+    private void setupCredentials()
+            throws Exception {
+        Configuration conf = new Configuration(false);
+
+        File file = new File(jksPath.toUri().getPath());
+        file.delete();
+        conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+        CredentialProvider provider =
+                CredentialProviderFactory.getProviders(conf).get(0);
+
+        // create new aliases
+        try {
+
+            char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+            provider.createCredentialEntry(
+                    SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY, storepass);
+
+            char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+            provider.createCredentialEntry(
+                    SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY, trustpass);
+
+            char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+            provider.createCredentialEntry(
+                    SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY, certpass);
+
+            // write out so that it can be found in checks
+            provider.flush();
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw e;
+        }
+    }
+
+}