changes to make sure secure configs will work

regression/pom.xml

@@ -31,7 +31,7 @@
     <properties>
         <atlas.url>http://localhost:21000</atlas.url>
         <surefire.version>2.18.1</surefire.version>
-        <hadoop.version>2.6.0</hadoop.version>
+        <hadoop.version>2.7.0</hadoop.version>
         <testng.groups>admin</testng.groups>
         <testng.exclude.groups></testng.exclude.groups>
     </properties>
@@ -74,6 +74,11 @@
             <version>${hadoop.version}</version>
         </dependency>
         <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpcore</artifactId>
             <version>4.2.5</version>
@@ -99,6 +104,15 @@
             </properties>
           </configuration>
         </plugin>
+        <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.3</version>
+        <configuration>
+          <source>1.7</source>
+          <target>1.7</target>
+        </configuration>
+      </plugin>
       </plugins>
     </pluginManagement>
   </build>

regression/src/test/java/org/apache/atlas/regression/request/BaseRequest.java

@@ -17,13 +17,16 @@
  */
 package org.apache.atlas.regression.request;
 
+import org.apache.atlas.regression.security.AtlasAuthenticationToken;
 import org.apache.commons.net.util.TrustManagerUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
 import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
 import org.apache.http.Header;
 import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
 import org.apache.http.auth.AuthenticationException;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpDelete;
@@ -160,28 +163,25 @@ public class BaseRequest {
                     header.getValue()));
         }
         HttpResponse response = client.execute(target, request);
-        // TODO: Determine appropriate call for when running secure atlas.
-//        /*incase the cookie is expired and we get a negotiate error back, generate the token again
-//        and send the request*/
-//        if ((response.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED)) {
-//            Header[] wwwAuthHeaders = response.getHeaders(RequestKeys.WWW_AUTHENTICATE);
-//            if (wwwAuthHeaders != null && wwwAuthHeaders.length != 0
-//                    && wwwAuthHeaders[0].getValue().trim().startsWith(RequestKeys.NEGOTIATE)) {
-//                if (AbstractEntityHelper.AUTHENTICATE) {
-//                    token = FalconAuthorizationToken.getToken(user, uri.getScheme(),
-//                            uri.getHost(), uri.getPort(), true);
-//                    request.removeHeaders(RequestKeys.COOKIE);
-//                    request.addHeader(RequestKeys.COOKIE, RequestKeys.AUTH_COOKIE_EQ + token);
-//                }
-//                LOGGER.info("Request Url: " + request.getRequestLine().getUri());
-//                LOGGER.info("Request Method: " + request.getRequestLine().getMethod());
-//                for (Header header : request.getAllHeaders()) {
-//                    LOGGER.info(String.format("Request Header: Name=%s Value=%s", header.getName(),
-//                            header.getValue()));
-//                }
-//                response = client.execute(target, request);
-//            }
-//        }
+        // add token to the request in case we get a 401 back with negotiate.
+        if ((response.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED)) {
+            Header[] wwwAuthHeaders = response.getHeaders(RequestKeys.WWW_AUTHENTICATE);
+            if (wwwAuthHeaders != null && wwwAuthHeaders.length != 0
+                    && wwwAuthHeaders[0].getValue().trim().startsWith(RequestKeys.NEGOTIATE)) {
+                AuthenticatedURL.Token token = AtlasAuthenticationToken.getToken(user, uri
+                                .getScheme(),
+                        uri.getHost(), uri.getPort(), true);
+                request.removeHeaders(RequestKeys.COOKIE);
+                request.addHeader(RequestKeys.COOKIE, RequestKeys.AUTH_COOKIE_EQ + token);
+                LOGGER.info("Request Url: " + request.getRequestLine().getUri());
+                LOGGER.info("Request Method: " + request.getRequestLine().getMethod());
+                for (Header header : request.getAllHeaders()) {
+                    LOGGER.info(String.format("Request Header: Name=%s Value=%s", header.getName(),
+                            header.getValue()));
+                }
+                response = client.execute(target, request);
+            }
+        }
         LOGGER.info("Response Status: " + response.getStatusLine());
         for (Header header : response.getAllHeaders()) {
             LOGGER.info(String.format("Response Header: Name=%s Value=%s", header.getName(),

regression/src/test/java/org/apache/atlas/regression/security/AtlasAuthenticationToken.java

+package org.apache.atlas.regression.security;
+
+import org.apache.atlas.regression.request.BaseRequest;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
+import org.apache.http.auth.AuthenticationException;
+import org.apache.log4j.Logger;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSession;
+import java.io.IOException;
+import java.net.URL;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class AtlasAuthenticationToken {
+    private static final String AUTH_URL = "api/options";
+    private static final KerberosAuthenticator AUTHENTICATOR = new KerberosAuthenticator();
+    private static final AtlasAuthenticationToken INSTANCE = new AtlasAuthenticationToken();
+    private static final Logger LOGGER = Logger.getLogger(AtlasAuthenticationToken.class);
+
+    // Use a hashmap so that we can cache the tokens.
+    private final ConcurrentHashMap<String, AuthenticatedURL.Token> tokens =
+            new ConcurrentHashMap<>();
+
+    private AtlasAuthenticationToken() {
+    }
+
+    public static final HostnameVerifier ALL_TRUSTING_HOSTNAME_VERIFIER = new HostnameVerifier() {
+        @Override
+        public boolean verify(String hostname, SSLSession sslSession) {
+            return true;
+        }
+    };
+
+    private static void authenticate(String user, String protocol, String host, int port)
+            throws IOException, AuthenticationException, InterruptedException {
+        final URL url = new URL(String.format("%s://%s:%d/%s", protocol, host, port,
+                AUTH_URL + "?" + PseudoAuthenticator.USER_NAME + "=" + user));
+        LOGGER.info("Authorize using url: " + url.toString());
+
+        final AuthenticatedURL.Token currentToken = new AuthenticatedURL.Token();
+
+        /*using KerberosAuthenticator which falls back to PsuedoAuthenticator
+        instead of passing authentication type from the command line - bad factory*/
+        try {
+            HttpsURLConnection.setDefaultSSLSocketFactory(BaseRequest.getSslContext()
+                    .getSocketFactory());
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        HttpsURLConnection.setDefaultHostnameVerifier(ALL_TRUSTING_HOSTNAME_VERIFIER);
+        UserGroupInformation callerUGI = KerberosHelper.getUGI(user);
+        callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
+            @Override
+            public Void run() throws Exception {
+                new AuthenticatedURL(AUTHENTICATOR).openConnection(url, currentToken);
+                return null;
+            }
+        });
+        String key = getKey(user, protocol, host, port);
+
+        // initialize a hash map if its null.
+        LOGGER.info("Authorization Token: " + currentToken.toString());
+        INSTANCE.tokens.put(key, currentToken);
+    }
+
+    public static AuthenticatedURL.Token getToken(String user, String protocol, String host,
+                                                  int port, boolean overWrite)
+            throws IOException, AuthenticationException, InterruptedException {
+        String key = getKey(user, protocol, host, port);
+
+        /*if the tokens are null or if token is not found then we will go ahead and authenticate
+        or if we are asked to overwrite*/
+        if (!INSTANCE.tokens.containsKey(key) || overWrite) {
+            authenticate(user, protocol, host, port);
+        }
+        return INSTANCE.tokens.get(key);
+    }
+
+    public static AuthenticatedURL.Token getToken(String user, String protocol, String host,
+                                                  int port)
+            throws IOException, AuthenticationException, InterruptedException {
+        return getToken(user, protocol, host, port, false);
+    }
+
+    /*spnego token will be unique to the user and uri its being requested for.
+    Hence the key of the hash map is the combination of user, protocol, host and port.*/
+    private static String getKey(String user, String protocol, String host, int port) {
+        return String.format("%s-%s-%s-%d", user, protocol, host, port);
+    }
+}

regression/src/test/java/org/apache/atlas/regression/security/KerberosHelper.java

+package org.apache.atlas.regression.security;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.testng.Assert;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+public class KerberosHelper {
+
+    private KerberosHelper() {
+        throw new AssertionError("Instantiating utility class...");
+    }
+
+    // determine if running on a secure cluster if secure=true is sent
+    public static final boolean IS_SECURE =  Boolean.parseBoolean(System.getProperty("secure",
+            "false"));
+
+    /** keytab of current user. */
+    private static final String CURRENT_USER_KEYTAB = System.getProperty("current.user.keytab",
+            null);
+
+    // determine the user realm to use
+    private static final String USER_REALM = System.getProperty("user.realm", "");
+
+    private static HashMap<String, String> keyTabMap;
+
+    /* initialize keyTabMap */
+    static {
+        keyTabMap = new HashMap<>();
+        keyTabMap.put(System.getProperty("user.name"), CURRENT_USER_KEYTAB);
+    }
+
+    public static UserGroupInformation getUGI(String user) throws IOException {
+        // if unsecure cluster create a remote user object
+        if (!IS_SECURE) {
+            return UserGroupInformation.createRemoteUser(user);
+        }
+        // if secure create a ugi object from keytab
+        return UserGroupInformation.loginUserFromKeytabAndReturnUGI(getPrincipal(user),
+                getKeytabForUser(user));
+    }
+
+    private static String getPrincipal(String user) {
+        return USER_REALM.isEmpty() ? user : user + '@' + USER_REALM;
+    }
+
+    private static String getKeytabForUser(String user) {
+        Assert.assertTrue(keyTabMap.containsKey(user), "Unknown user: " + user);
+        return keyTabMap.get(user);
+    }
+}
\ No newline at end of file

regression/src/test/java/org/apache/atlas/regression/tests/AdminResourceTest.java

@@ -26,13 +26,11 @@ import org.apache.http.HttpResponse;
 import org.apache.http.util.EntityUtils;
 import org.apache.log4j.Logger;
 import org.testng.annotations.Test;
-import org.testng.asserts.SoftAssert;
 
 @Test(groups = "admin")
 public class AdminResourceTest extends BaseTest {
     private static final Logger logger = Logger.getLogger(AdminResourceTest.class);
     private static String baseReqUrl = ATLAS_URL + "/api/metadata/admin";
-    public SoftAssert softassert = new SoftAssert();
 
     @Test
     public void testVersion()
@@ -44,13 +42,13 @@ public class AdminResourceTest extends BaseTest {
         String version = JsonPath.read(document, "$.Version");
         String name = JsonPath.read(document, "$.Name");
         String description = JsonPath.read(document, "$.Description");
-        softassert.assertTrue(null != version && !version.isEmpty(), "Version is empty");
-        softassert.assertEquals(name, "metadata-governance", "Name does not match");
-        softassert.assertEquals(description,
+        SOFT_ASSERT.assertTrue(null != version && !version.isEmpty(), "Version is empty");
+        SOFT_ASSERT.assertEquals(name, "metadata-governance", "Name does not match");
+        SOFT_ASSERT.assertEquals(description,
                 "Metadata Management and Data Governance Platform over " +
                         "Hadoop", "Description does not match");
-        TestUtils.assert200(softassert, RequestKeys.JSON_CONTENT_TYPE, response);
-        softassert.assertAll();
+        TestUtils.assert200(SOFT_ASSERT, RequestKeys.JSON_CONTENT_TYPE, response);
+        SOFT_ASSERT.assertAll();
     }
 
     @Test
@@ -58,8 +56,8 @@ public class AdminResourceTest extends BaseTest {
             throws Exception {
         BaseRequest req = new BaseRequest(baseReqUrl + "/stack");
         HttpResponse response = req.run();
-        TestUtils.assert200(softassert, RequestKeys.TEXT_CONTENT_TYPE, response);
-        softassert.assertNotNull(EntityUtils.toString(response.getEntity()), "Content is not null");
-        softassert.assertAll();
+        TestUtils.assert200(SOFT_ASSERT, RequestKeys.TEXT_CONTENT_TYPE, response);
+        SOFT_ASSERT.assertNotNull(EntityUtils.toString(response.getEntity()), "Content is not null");
+        SOFT_ASSERT.assertAll();
     }
 }
\ No newline at end of file

regression/src/test/java/org/apache/atlas/regression/tests/BaseTest.java

@@ -17,7 +17,16 @@
  */
 package org.apache.atlas.regression.tests;
 
+import org.testng.annotations.BeforeClass;
+import org.testng.asserts.SoftAssert;
+
 public class BaseTest {
     public static final String ATLAS_URL = System.getProperty("atlas.url",
             "http://localhost:21000") ;
+    public SoftAssert SOFT_ASSERT;
+
+    @BeforeClass
+    public void beforeClass() {
+        SOFT_ASSERT = new SoftAssert();
+    }
 }
\ No newline at end of file