Skip to content

A
atlas
Commit 42578ccc by Shwetha GS
Options

test fixes

parent bb64966d
Showing with 280 additions and 207 deletions
addons/hive-bridge/pom.xml
...@@ -222,6 +222,10 @@ ...@@ -222,6 +222,10 @@
<name>metadata.log.dir</name> <name>metadata.log.dir</name>
<value>${project.build.directory}/logs</value> <value>${project.build.directory}/logs</value>
</systemProperty> </systemProperty>
<systemProperty>
<name>metadata.properties.location</name>
<value>addons/hive-bridge/src/test/resources</value>
</systemProperty>
</systemProperties> </systemProperties>
<stopKey>metadata-stop</stopKey> <stopKey>metadata-stop</stopKey>
<stopPort>41001</stopPort> <stopPort>41001</stopPort>
......
addons/hive-bridge/src/test/resources/application.properties 0 → 100644
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
######### Graph Database Configs #########
# Graph Storage
metadata.graph.storage.backend=inmemory
# Graph Search Index
metadata.graph.index.search.backend=lucene
metadata.graph.index.search.directory=target/data/lucene
######### Hive Lineage Configs #########
metadata.lineage.hive.table.type.name=hive_table
metadata.lineage.hive.column.type.name=hive_column
metadata.lineage.hive.table.column.name=columns
metadata.lineage.hive.process.type.name=hive_process
metadata.lineage.hive.process.inputs.name=inputTables
metadata.lineage.hive.process.outputs.name=outputTables
######### Security Properties #########
# SSL config
metadata.enableTLS=false
pom.xml
...@@ -807,11 +807,12 @@ ...@@ -807,11 +807,12 @@
<redirectTestOutputToFile>true</redirectTestOutputToFile> <redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.awt.headless=true -Dproject.version=${project.version} <argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
-Dhadoop.tmp.dir=${project.build.directory}/tmp-hadoop-${user.name} -Dhadoop.tmp.dir=${project.build.directory}/tmp-hadoop-${user.name}
-Xmx1024m -XX:MaxPermSize=512m
</argLine> </argLine>
<parallel>none</parallel> <parallel>none</parallel>
<reuseForks>false</reuseForks> <reuseForks>false</reuseForks>
<forkCount>1</forkCount> <forkCount>1</forkCount>
<threadCount>1</threadCount> <threadCount>5</threadCount>
</configuration> </configuration>
<executions> <executions>
<execution> <execution>
......
repository/src/main/java/org/apache/hadoop/metadata/PropertiesUtil.java
...@@ -21,10 +21,17 @@ package org.apache.hadoop.metadata; ...@@ -21,10 +21,17 @@ package org.apache.hadoop.metadata;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import java.io.File;
public class PropertiesUtil { public class PropertiesUtil {
private static final String APPLICATION_PROPERTIES = "/application.properties"; private static final String APPLICATION_PROPERTIES = "application.properties";
public static final PropertiesConfiguration getApplicationProperties() throws ConfigurationException { public static final PropertiesConfiguration getApplicationProperties() throws ConfigurationException {
return new PropertiesConfiguration(PropertiesUtil.class.getResource(APPLICATION_PROPERTIES)); String proprtiesLocation = System.getProperty("metadata.properties.location");
if (proprtiesLocation == null) {
return new PropertiesConfiguration(PropertiesUtil.class.getResource("/" + APPLICATION_PROPERTIES));
} else {
return new PropertiesConfiguration(new File(proprtiesLocation, APPLICATION_PROPERTIES));
}
} }
} }
webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerIT.java 100755 → 100644
...@@ -14,112 +14,17 @@ ...@@ -14,112 +14,17 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.metadata.web.service; package org.apache.hadoop.metadata.web.service;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metadata.web.resources.*;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.mortbay.jetty.webapp.WebAppContext;
import org.testng.Assert; import org.testng.Assert;
import org.testng.TestListenerAdapter;
import org.testng.TestNG;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import javax.ws.rs.core.UriBuilder;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.nio.file.Files;
import java.util.List;
/**
*
*/
public class SecureEmbeddedServerIT {
private SecureEmbeddedServer secureEmbeddedServer;
private String providerUrl;
private Path jksPath;
protected WebResource service;
static {
//for localhost testing only
javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
new javax.net.ssl.HostnameVerifier(){
public boolean verify(String hostname,
javax.net.ssl.SSLSession sslSession) {
if (hostname.equals("localhost")) {
return true;
}
return false;
}
});
System.setProperty("javax.net.ssl.trustStore", SecureEmbeddedServer.DEFAULT_KEYSTORE_FILE_LOCATION);
System.setProperty("javax.net.ssl.trustStorePassword", "keypass");
System.setProperty("javax.net.ssl.trustStoreType", "JKS");
}
@BeforeClass
public void setupServerURI () throws Exception {
BaseResourceIT.baseUrl = "https://localhost:21443";
}
@AfterClass
public void resetServerURI() throws Exception {
BaseResourceIT.baseUrl = "http://localhost:21000";
}
@BeforeMethod
public void setup() throws Exception {
jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
String baseUrl = "https://localhost:21443/";
DefaultClientConfig config = new DefaultClientConfig();
Client client = Client.create(config);
client.resource(UriBuilder.fromUri(baseUrl).build());
service = client.resource(UriBuilder.fromUri(baseUrl).build());
}
@Test
public void testNoConfiguredCredentialProvider() throws Exception {
try {
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance");
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.server.setHandler(webapp);
secureEmbeddedServer.server.start();
Assert.fail("Should have thrown an exception");
} catch (IOException e) {
assert e.getMessage().equals("No credential provider path configured for storage of certificate store passwords");
} finally {
secureEmbeddedServer.server.stop();
}
}
public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase{
@Test @Test
public void testServerConfiguredUsingCredentialProvider() throws Exception { public void testServerConfiguredUsingCredentialProvider() throws Exception {
// setup the configuration // setup the configuration
...@@ -128,18 +33,15 @@ public class SecureEmbeddedServerIT { ...@@ -128,18 +33,15 @@ public class SecureEmbeddedServerIT {
// setup the credential provider // setup the credential provider
setupCredentials(); setupCredentials();
SecureEmbeddedServer secureEmbeddedServer = null;
try { try {
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") { String appPath = System.getProperty("user.dir") + getWarPath();
secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) {
@Override @Override
protected PropertiesConfiguration getConfiguration() { protected PropertiesConfiguration getConfiguration() {
return configuration; return configuration;
} }
}; };
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.server.setHandler(webapp);
secureEmbeddedServer.server.start(); secureEmbeddedServer.server.start();
URL url = new URL("https://localhost:21443/"); URL url = new URL("https://localhost:21443/");
...@@ -148,110 +50,10 @@ public class SecureEmbeddedServerIT { ...@@ -148,110 +50,10 @@ public class SecureEmbeddedServerIT {
connection.connect(); connection.connect();
// test to see whether server is up and root page can be served // test to see whether server is up and root page can be served
assert connection.getResponseCode() == 200; Assert.assertEquals(connection.getResponseCode(), 200);
} finally {
secureEmbeddedServer.server.stop();
}
}
@Test
public void testMissingEntriesInCredentialProvider() throws Exception {
// setup the configuration
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
try {
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
@Override
protected PropertiesConfiguration getConfiguration() {
return configuration;
}
};
Assert.fail("No entries should generate an exception");
} catch (IOException e) {
assert e.getMessage().startsWith("No credential entry found for");
} finally {
secureEmbeddedServer.server.stop();
}
}
/**
* Runs the existing webapp test cases, this time against the initiated secure server instance.
* @throws Exception
*/
@Test
public void runOtherSuitesAgainstSecureServer() throws Exception {
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
// setup the credential provider
setupCredentials();
try {
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
@Override
protected PropertiesConfiguration getConfiguration() {
return configuration;
}
};
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.server.setHandler(webapp);
secureEmbeddedServer.server.start();
TestListenerAdapter tla = new TestListenerAdapter();
TestNG testng = new TestNG();
testng.setTestClasses(new Class[] { AdminJerseyResourceIT.class, EntityJerseyResourceIT.class,
MetadataDiscoveryJerseyResourceIT.class, RexsterGraphJerseyResourceIT.class, TypesJerseyResourceIT.class});
testng.addListener(tla);
testng.run();
} finally { } finally {
secureEmbeddedServer.server.stop(); secureEmbeddedServer.server.stop();
} }
} }
protected String getWarPath() {
return String.format("/target/metadata-webapp-%s.war",
System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
}
private void setupCredentials()
throws Exception {
Configuration conf = new Configuration(false);
File file = new File(jksPath.toUri().getPath());
file.delete();
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
// create new aliases
try {
char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY, storepass);
char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY, trustpass);
char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY, certpass);
// write out so that it can be found in checks
provider.flush();
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
} }
webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerITBase.java 0 → 100755
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.metadata.web.service;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metadata.web.resources.*;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.mortbay.jetty.webapp.WebAppContext;
import org.testng.Assert;
import org.testng.TestListenerAdapter;
import org.testng.TestNG;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import javax.ws.rs.core.UriBuilder;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.util.List;
/**
*
*/
public class SecureEmbeddedServerITBase {
private SecureEmbeddedServer secureEmbeddedServer;
protected String providerUrl;
private Path jksPath;
protected WebResource service;
static {
//for localhost testing only
javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
new javax.net.ssl.HostnameVerifier(){
public boolean verify(String hostname,
javax.net.ssl.SSLSession sslSession) {
if (hostname.equals("localhost")) {
return true;
}
return false;
}
});
System.setProperty("javax.net.ssl.trustStore", SecureEmbeddedServer.DEFAULT_KEYSTORE_FILE_LOCATION);
System.setProperty("javax.net.ssl.trustStorePassword", "keypass");
System.setProperty("javax.net.ssl.trustStoreType", "JKS");
}
@BeforeClass
public void setupServerURI () throws Exception {
BaseResourceIT.baseUrl = "https://localhost:21443";
}
@AfterClass
public void resetServerURI() throws Exception {
BaseResourceIT.baseUrl = "http://localhost:21000";
}
@BeforeMethod
public void setup() throws Exception {
jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
String baseUrl = "https://localhost:21443/";
DefaultClientConfig config = new DefaultClientConfig();
Client client = Client.create(config);
client.resource(UriBuilder.fromUri(baseUrl).build());
service = client.resource(UriBuilder.fromUri(baseUrl).build());
}
@Test
public void testNoConfiguredCredentialProvider() throws Exception {
try {
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance");
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.server.setHandler(webapp);
secureEmbeddedServer.server.start();
Assert.fail("Should have thrown an exception");
} catch (IOException e) {
assert e.getMessage().equals("No credential provider path configured for storage of certificate store passwords");
} finally {
secureEmbeddedServer.server.stop();
}
}
@Test
public void testMissingEntriesInCredentialProvider() throws Exception {
// setup the configuration
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
try {
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
@Override
protected PropertiesConfiguration getConfiguration() {
return configuration;
}
};
Assert.fail("No entries should generate an exception");
} catch (IOException e) {
assert e.getMessage().startsWith("No credential entry found for");
} finally {
secureEmbeddedServer.server.stop();
}
}
/**
* Runs the existing webapp test cases, this time against the initiated secure server instance.
* @throws Exception
*/
@Test
public void runOtherSuitesAgainstSecureServer() throws Exception {
final PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.setProperty(SecureEmbeddedServer.CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
// setup the credential provider
setupCredentials();
try {
secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
@Override
protected PropertiesConfiguration getConfiguration() {
return configuration;
}
};
WebAppContext webapp = new WebAppContext();
webapp.setContextPath("/");
webapp.setWar(System.getProperty("user.dir") + getWarPath());
secureEmbeddedServer.server.setHandler(webapp);
secureEmbeddedServer.server.start();
TestListenerAdapter tla = new TestListenerAdapter();
TestNG testng = new TestNG();
testng.setTestClasses(new Class[] { AdminJerseyResourceIT.class, EntityJerseyResourceIT.class,
MetadataDiscoveryJerseyResourceIT.class, RexsterGraphJerseyResourceIT.class, TypesJerseyResourceIT.class});
testng.addListener(tla);
testng.run();
} finally {
secureEmbeddedServer.server.stop();
}
}
protected String getWarPath() {
return String.format("/target/metadata-webapp-%s",
System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
}
protected void setupCredentials() throws Exception {
Configuration conf = new Configuration(false);
File file = new File(jksPath.toUri().getPath());
file.delete();
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
// create new aliases
try {
char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
SecureEmbeddedServer.KEYSTORE_PASSWORD_KEY, storepass);
char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
SecureEmbeddedServer.TRUSTSTORE_PASSWORD_KEY, trustpass);
char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
SecureEmbeddedServer.SERVER_CERT_PASSWORD_KEY, certpass);
// write out so that it can be found in checks
provider.flush();
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment