ATLAS-762 - Assertion in NegativeSSLAndKerberosTest.testUnsecuredClient needs to be hardened

client/src/main/java/org/apache/atlas/AtlasClient.java

@@ -237,8 +237,12 @@ public class AtlasClient {
 
         URLConnectionClientHandler handler = null;
 
-        if ((!AuthenticationUtil.isKerberosAuthenticationEnabled()) && basicAuthUser!=null && basicAuthPassword!=null) {
-            handler = new URLConnectionClientHandler();
+        if ((!AuthenticationUtil.isKerberosAuthenticationEnabled()) && basicAuthUser != null && basicAuthPassword != null) {
+            if (clientConfig.getBoolean(TLS_ENABLED, false)) {
+                handler = SecureClientUtils.getUrlConnectionClientHandler();
+            } else {
+                handler = new URLConnectionClientHandler();
+            }
         } else {
             handler =
                     SecureClientUtils.getClientConnectionHandler(config, clientConfig, doAsUser, ugi);

client/src/main/java/org/apache/atlas/security/SecureClientUtils.java

@@ -211,4 +211,39 @@ public class SecureClientUtils {
         }
     }
 
+    public static URLConnectionClientHandler getUrlConnectionClientHandler() {
+        return new URLConnectionClientHandler(new HttpURLConnectionFactory() {
+            @Override
+            public HttpURLConnection getHttpURLConnection(URL url)
+                    throws IOException {
+                HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+
+                if (connection instanceof HttpsURLConnection) {
+                    LOG.debug("Attempting to configure HTTPS connection using client "
+                            + "configuration");
+                    final SSLFactory factory;
+                    final SSLSocketFactory sf;
+                    final HostnameVerifier hv;
+
+                    try {
+                        Configuration conf = new Configuration();
+                        conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, SecurityProperties.SSL_CLIENT_PROPERTIES));
+                        UserGroupInformation.setConfiguration(conf);
+
+                        HttpsURLConnection c = (HttpsURLConnection) connection;
+                        factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+                        factory.init();
+                        sf = factory.createSSLSocketFactory();
+                        hv = factory.getHostnameVerifier();
+                        c.setSSLSocketFactory(sf);
+                        c.setHostnameVerifier(hv);
+                    } catch (Exception e) {
+                        LOG.info("Unable to configure HTTPS connection from "
+                                + "configuration.  Leveraging JDK properties.");
+                    }
+                }
+                return connection;
+            }
+        });
+    }
 }

release-log.txt

@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
 
 
 ALL CHANGES:
+ATLAS-762 Assertion in NegativeSSLAndKerberosTest.testUnsecuredClient needs to be hardened (nixonrodrigues via sumasai)
 ATLAS-1071 Regression - UI - Details Button under Audits Tab is not working.(kevalbhatt18 via sumasai)
 ATLAS-965 Old lineage still exists after dropping tables and re-creating tables with same name. (shwethags via sumasai)
 ATLAS-1048 TestMetadata.py test in distro project fails on Windows (jnhagelb via shwethags)

typesystem/src/test/resources/atlas-application.properties

@@ -119,3 +119,7 @@ atlas.server.ha.enabled=false
 #########POLICY FILE PATH #########
 atlas.auth.policy.file=${sys:user.dir}/distro/src/conf/policy-store.txt
 
+atlas.authentication.method.file=true
+atlas.authentication.method.ldap.type=none
+atlas.authentication.method.file.filename=${sys:user.dir}/distro/src/conf/users-credentials.properties
+atlas.authentication.method.kerberos=false
\ No newline at end of file

webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java

@@ -77,19 +77,15 @@ public abstract class BaseResourceIT {
     @BeforeClass
     public void setUp() throws Exception {
 
-        DefaultClientConfig config = new DefaultClientConfig();
-        Client client = Client.create(config);
         Configuration configuration = ApplicationProperties.get();
         baseUrl = configuration.getString(ATLAS_REST_ADDRESS, "http://localhost:21000/");
-        client.resource(UriBuilder.fromUri(baseUrl).build());
-
-        service = client.resource(UriBuilder.fromUri(baseUrl).build());
 
         if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
             serviceClient = new AtlasClient(new String[]{baseUrl}, new String[]{"admin", "admin"});
         } else {
             serviceClient = new AtlasClient(baseUrl);
         }
+        service = serviceClient.getResource();
     }
 
     protected void createType(TypesDef typesDef) throws Exception {

webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java

-/**
+    /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -73,6 +73,7 @@ import java.util.UUID;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.fail;
+import org.apache.atlas.utils.AuthenticationUtil;
 
 
 /**
@@ -127,9 +128,13 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
         entity.set("name", randomString());
         entity.set("description", randomString());
 
-        String user = "testuser";
-        UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
-        AtlasClient localClient = new AtlasClient(ugi, null, baseUrl);
+        String user = "admin";
+        AtlasClient localClient = null;
+        if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
+            localClient = new AtlasClient(new String[]{baseUrl}, new String[]{"admin", "admin"});
+        } else {
+            localClient = new AtlasClient(baseUrl);
+        }
         String entityId = localClient.createEntity(entity).get(0);
 
         List<EntityAuditEvent> events = serviceClient.getEntityAuditEvents(entityId, (short) 10);

webapp/src/test/java/org/apache/atlas/web/security/BaseSSLAndKerberosTest.java

@@ -40,7 +40,7 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest {
     protected Path jksPath;
     protected String providerUrl;
     protected File httpKeytabFile;
-    private File userKeytabFile;
+    protected File userKeytabFile;
 
     class TestSecureEmbeddedServer extends SecureEmbeddedServer {
 
@@ -98,7 +98,7 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest {
         File kdcWorkDir = startKDC();
 
         userKeytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab");
-        createKeytab(kdc, kdcWorkDir, "zookeeper", "dgi.keytab");
+        //createKeytab(kdc, kdcWorkDir, "zookeeper", "dgi.keytab");
         httpKeytabFile = createKeytab(kdc, kdcWorkDir, "HTTP", "spnego.service.keytab");
 
         // create a test user principal

webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java

@@ -41,6 +41,13 @@ import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIA
 import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
 import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
 import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
+import static org.apache.atlas.security.SecurityProperties.SSL_CLIENT_PROPERTIES;
+import static org.apache.atlas.security.SecurityProperties.CLIENT_AUTH_KEY;
+import static org.apache.atlas.security.SecurityProperties.SSL_CLIENT_PROPERTIES;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.atlas.AtlasException;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
 
 /**
  *
@@ -153,9 +160,79 @@ public class BaseSecurityTest {
         configuredProperties.copy(configuration);
 
         String persistDir = TestUtils.getTempDirectory();
+        configuredProperties.setProperty("atlas.authentication.method.file", "true");
+        configuredProperties.setProperty("atlas.authentication.method.file.filename", persistDir
+                + "/users-credentials");
+        configuredProperties.setProperty("atlas.auth.policy.file",persistDir
+                + "/policy-store.txt" );
         TestUtils.writeConfiguration(configuredProperties, persistDir + File.separator +
                 ApplicationProperties.APPLICATION_PROPERTIES);
+        setupUserCredential(persistDir);
+        setUpPolicyStore(persistDir);
         ApplicationProperties.forceReload();
         return persistDir;
     }
+
+    public static void setupUserCredential(String tmpDir) throws Exception {
+
+        StringBuilder credentialFileStr = new StringBuilder(1024);
+        credentialFileStr.append("admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918\n");
+        credentialFileStr.append("michael=DATA_SCIENTIST::95bfb24de17d285d734b9eaa9109bfe922adc85f20d2e5e66a78bddb4a4ebddb\n");
+        credentialFileStr.append("paul=DATA_STEWARD::e7c0dcf5f8a93e93791e9bac1ae454a691c1d2a902fc4256d489e96c1b9ac68c\n");
+        credentialFileStr.append("testuser=DATA_STEWARD::e7c0dcf5f8a93e93791e9bac1ae454a691c1d2a902fc4256d489e96c1b9ac68c\n");
+        File credentialFile = new File(tmpDir, "users-credentials");
+        FileUtils.write(credentialFile, credentialFileStr.toString());
+    }
+
+    public static void setUpPolicyStore(String tmpDir) throws Exception {
+        StringBuilder policyStr = new StringBuilder(1024);
+        policyStr.append("adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:*\n");
+        policyStr.append("dataStewardPolicy;;testuser:rwud;;DATA_STEWARD:rwu;;type:*,entity:*,taxonomy:*,term:*\n");
+
+        File policyFile = new File(tmpDir, "policy-store.txt");
+        FileUtils.write(policyFile, policyStr.toString());
+    }
+
+    public static void persistSSLClientConfiguration(org.apache.commons.configuration.Configuration clientConfig)
+            throws AtlasException, IOException {
+        //trust settings
+        Configuration configuration = new Configuration(false);
+        File sslClientFile = getSSLClientFile();
+        if (!sslClientFile.exists()) {
+            configuration.set("ssl.client.truststore.type", "jks");
+            configuration.set("ssl.client.truststore.location", clientConfig.getString(TRUSTSTORE_FILE_KEY));
+            if (clientConfig.getBoolean(CLIENT_AUTH_KEY, false)) {
+                // need to get client key properties
+                configuration.set("ssl.client.keystore.location", clientConfig.getString(KEYSTORE_FILE_KEY));
+                configuration.set("ssl.client.keystore.type", "jks");
+            }
+            // add the configured credential provider
+            configuration.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+                    clientConfig.getString(CERT_STORES_CREDENTIAL_PROVIDER_PATH));
+            String hostnameVerifier = clientConfig.getString(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY);
+            if (hostnameVerifier != null) {
+                configuration.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, hostnameVerifier);
+            }
+
+            configuration.writeXml(new FileWriter(sslClientFile));
+        }
+    }
+
+    private static File getSSLClientFile() throws AtlasException {
+        File sslDir;
+        try {
+            String persistDir = null;
+            URL resource = BaseSecurityTest.class.getResource("/");
+            if (resource != null) {
+                persistDir = resource.toURI().getPath();
+            }
+            assert persistDir != null;
+            sslDir = new File(persistDir);
+
+            // LOG.info("ssl-client.xml will be created in {}", sslDir);
+        } catch (Exception e) {
+            throw new AtlasException("Failed to find client configuration directory", e);
+        }
+        return new File(sslDir, SSL_CLIENT_PROPERTIES);
+    }
 }

webapp/src/test/java/org/apache/atlas/web/security/NegativeSSLAndKerberosTest.java

@@ -61,6 +61,8 @@ public class NegativeSSLAndKerberosTest extends BaseSSLAndKerberosTest {
         // client will actually only leverage subset of these properties
         final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
 
+        persistSSLClientConfiguration((org.apache.commons.configuration.Configuration) configuration);
+
         TestUtils.writeConfiguration(configuration, persistDir + File.separator +
             ApplicationProperties.APPLICATION_PROPERTIES);
 
@@ -74,11 +76,15 @@ public class NegativeSSLAndKerberosTest extends BaseSSLAndKerberosTest {
         configuration.load(url);
 
         configuration.setProperty(TLS_ENABLED, true);
-        configuration.setProperty("atlas.http.authentication.enabled", "true");
-        configuration.setProperty("atlas.http.authentication.type", "kerberos");
-        configuration.setProperty("atlas.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
-        configuration.setProperty("atlas.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath());
-        configuration.setProperty("atlas.http.authentication.kerberos.name.rules",
+        configuration.setProperty("atlas.authentication.method.kerberos", "true");
+        configuration.setProperty("atlas.authentication.keytab",userKeytabFile.getAbsolutePath());
+        configuration.setProperty("atlas.authentication.principal","dgi/localhost@"+kdc.getRealm());
+
+        configuration.setProperty("atlas.authentication.method.file", "false");
+        configuration.setProperty("atlas.authentication.method.kerberos", "true");
+        configuration.setProperty("atlas.authentication.method.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
+        configuration.setProperty("atlas.authentication.method.kerberos.keytab", httpKeytabFile.getAbsolutePath());
+        configuration.setProperty("atlas.authentication.method.kerberos.name.rules",
                 "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
 
         TestUtils.writeConfiguration(configuration, persistDir + File.separator +
@@ -127,6 +133,7 @@ public class NegativeSSLAndKerberosTest extends BaseSSLAndKerberosTest {
             Assert.fail("Should have failed with GSSException");
         } catch(Exception e) {
             e.printStackTrace();
+            Assert.assertTrue(e.getMessage().contains("Mechanism level: Failed to find any Kerberos tgt"));
         }
     }
 }

webapp/src/test/java/org/apache/atlas/web/security/SSLAndKerberosTest.java

@@ -70,6 +70,8 @@ public class SSLAndKerberosTest extends BaseSSLAndKerberosTest {
         // client will actually only leverage subset of these properties
         final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
 
+        persistSSLClientConfiguration((org.apache.commons.configuration.Configuration) configuration);
+
         TestUtils.writeConfiguration(configuration, persistDir + File.separator +
             ApplicationProperties.APPLICATION_PROPERTIES);
 
@@ -82,22 +84,39 @@ public class SSLAndKerberosTest extends BaseSSLAndKerberosTest {
         }
         configuration.load(url);
         configuration.setProperty(TLS_ENABLED, true);
-        configuration.setProperty("atlas.http.authentication.enabled", "true");
-        configuration.setProperty("atlas.http.authentication.type", "kerberos");
-        configuration.setProperty("atlas.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
-        configuration.setProperty("atlas.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath());
-        configuration.setProperty("atlas.http.authentication.kerberos.name.rules",
+        configuration.setProperty("atlas.authentication.method.kerberos", "true");
+        configuration.setProperty("atlas.authentication.keytab",userKeytabFile.getAbsolutePath());
+        configuration.setProperty("atlas.authentication.principal","dgi/localhost@"+kdc.getRealm());
+
+        configuration.setProperty("atlas.authentication.method.file", "false");
+        configuration.setProperty("atlas.authentication.method.kerberos", "true");
+        configuration.setProperty("atlas.authentication.method.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
+        configuration.setProperty("atlas.authentication.method.kerberos.keytab", httpKeytabFile.getAbsolutePath());
+        configuration.setProperty("atlas.authentication.method.kerberos.name.rules",
                 "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
 
+        configuration.setProperty("atlas.authentication.method.file", "true");
+        configuration.setProperty("atlas.authentication.method.file.filename", persistDir
+                + "/users-credentials");
+        configuration.setProperty("atlas.auth.policy.file",persistDir
+                + "/policy-store.txt" );
+
         TestUtils.writeConfiguration(configuration, persistDir + File.separator +
           "atlas-application.properties");
 
+        setupUserCredential(persistDir);
+        setUpPolicyStore(persistDir);
+
         subject = loginTestUser();
         UserGroupInformation.loginUserFromSubject(subject);
         UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
             "testUser",
             UserGroupInformation.getLoginUser());
 
+        // save original setting
+        originalConf = System.getProperty("atlas.conf");
+        System.setProperty("atlas.conf", persistDir);
+
         dgiCLient = proxyUser.doAs(new PrivilegedExceptionAction<AtlasClient>() {
             @Override
             public AtlasClient run() throws Exception {
@@ -110,9 +129,7 @@ public class SSLAndKerberosTest extends BaseSSLAndKerberosTest {
             }
         });
 
-        // save original setting
-        originalConf = System.getProperty("atlas.conf");
-        System.setProperty("atlas.conf", persistDir);
+
         secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {
             @Override
             public PropertiesConfiguration getConfiguration() {

webapp/src/test/java/org/apache/atlas/web/security/SSLTest.java

@@ -74,16 +74,18 @@ public class SSLTest extends BaseSSLAndKerberosTest {
         setupCredentials();
         final PropertiesConfiguration configuration = getSSLConfiguration(providerUrl);
         String persistDir = writeConfiguration(configuration);
+        persistSSLClientConfiguration((org.apache.commons.configuration.Configuration) configuration);
 
-        atlasClient = new AtlasClient(DGI_URL) {
+        originalConf = System.getProperty("atlas.conf");
+        System.setProperty("atlas.conf", persistDir);
+
+        atlasClient = new AtlasClient(new String[]{DGI_URL},new String[]{"admin","admin"}) {
             @Override
             protected PropertiesConfiguration getClientProperties() {
                 return configuration;
             }
         };
 
-        originalConf = System.getProperty("atlas.conf");
-        System.setProperty("atlas.conf", persistDir);
         secureEmbeddedServer = new TestSecureEmbeddedServer(21443, getWarPath()) {
             @Override
             public PropertiesConfiguration getConfiguration() {

webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerTest.java

@@ -67,7 +67,7 @@ public class SecureEmbeddedServerTest extends SecureEmbeddedServerTestBase {
             };
             secureEmbeddedServer.server.start();
 
-            URL url = new URL("https://localhost:21443/api/atlas/admin/version");
+            URL url = new URL("https://localhost:21443/api/atlas/admin/status");
             HttpURLConnection connection = (HttpURLConnection) url.openConnection();
             connection.setRequestMethod("GET");
             connection.connect();

webapp/src/test/webapp/WEB-INF/web.xml

@@ -38,6 +38,16 @@
      -->
 
     <filter>
+        <filter-name>springSecurityFilterChain</filter-name>
+        <filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>
+    </filter>
+
+    <filter-mapping>
+        <filter-name>springSecurityFilterChain</filter-name>
+        <url-pattern>/*</url-pattern>
+    </filter-mapping>
+
+    <filter>
         <filter-name>guiceFilter</filter-name>
         <filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
     </filter>
@@ -50,4 +60,17 @@
     <listener>
         <listener-class>org.apache.atlas.web.listeners.TestGuiceServletConfig</listener-class>
     </listener>
+
+    <listener>
+        <listener-class>org.springframework.web.util.Log4jConfigListener</listener-class>
+    </listener>
+
+    <listener>
+        <listener-class>org.springframework.web.context.request.RequestContextListener</listener-class>
+    </listener>
+
+    <listener>
+        <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
+    </listener>
+
 </web-app>