ATLAS-809 JAAS configuration needed for Kafka interaction via Atlas config file…

common/pom.xml

@@ -52,6 +52,16 @@
         </dependency>
 
         <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-collections</groupId>
+            <artifactId>commons-collections</artifactId>
+        </dependency>
+
+        <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-all</artifactId>
         </dependency>

common/src/main/java/org/apache/atlas/ApplicationProperties.java

@@ -17,6 +17,7 @@
  */
 package org.apache.atlas;
 
+import org.apache.atlas.security.InMemoryJAASConfiguration;
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
@@ -56,6 +57,7 @@ public final class ApplicationProperties extends PropertiesConfiguration {
             synchronized (ApplicationProperties.class) {
                 if (instance == null) {
                     instance = get(APPLICATION_PROPERTIES);
+                    InMemoryJAASConfiguration.init(instance);
                 }
             }
         }

common/src/test/java/org/apache/atlas/security/InMemoryJAASConfigurationTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.security;
+
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+import org.apache.hadoop.util.StringUtils;
+
+public class InMemoryJAASConfigurationTest extends TestCase {
+
+    private static final String ATLAS_JAAS_PROP_FILE = "atlas-jaas.properties";
+
+    protected void setUp() throws Exception {
+        super.setUp();
+        try {
+            InMemoryJAASConfiguration.init(ATLAS_JAAS_PROP_FILE);
+        } catch(Throwable t) {
+            fail("InMemoryJAASConfiguration.init() is not expected to throw Exception:" +  t);
+        }
+    }
+
+    protected void tearDown() throws Exception {
+        super.tearDown();
+    }
+
+    public void testGetAppConfigurationEntryStringForKafkaClient() {
+        AppConfigurationEntry[] entries =
+                Configuration.getConfiguration().getAppConfigurationEntry("KafkaClient");
+        Assert.assertNotNull(entries);
+        Assert.assertEquals(1, entries.length);
+        String principal = (String) entries[0].getOptions().get("principal");
+        Assert.assertNotNull(principal);
+        String[] components = principal.split("[/@]");
+        Assert.assertEquals(3, components.length);
+        Assert.assertEquals(false, StringUtils.equalsIgnoreCase(components[1], "_HOST"));
+
+    }
+
+    public void testGetAppConfigurationEntryStringForMyClient() {
+        AppConfigurationEntry[] entries =
+                Configuration.getConfiguration().getAppConfigurationEntry("myClient");
+        Assert.assertNotNull(entries);
+        Assert.assertEquals(2, entries.length);
+        String principal = (String) entries[0].getOptions().get("principal");
+        Assert.assertNotNull(principal);
+        String[] components = principal.split("[/@]");
+        Assert.assertEquals(3, components.length);
+        Assert.assertEquals(true, StringUtils.equalsIgnoreCase(components[1], "abcd"));
+
+        principal = (String) entries[1].getOptions().get("principal");
+        Assert.assertNotNull(principal);
+        components = principal.split("[/@]");
+        Assert.assertEquals(2, components.length);
+    }
+
+    public void testGetAppConfigurationEntryStringForUnknownClient() {
+        AppConfigurationEntry[] entries =
+                Configuration.getConfiguration().getAppConfigurationEntry("UnknownClient");
+        Assert.assertNull(entries);
+    }
+
+}
+

common/src/test/resources/atlas-jaas.properties

+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#########  Notification Configs  #########
+atlas.notification.embedded=true
+
+atlas.kafka.zookeeper.connect=localhost:19026
+atlas.kafka.bootstrap.servers=localhost:19027
+atlas.kafka.data=${sys:atlas.data}/kafka
+atlas.kafka.zookeeper.session.timeout.ms=4000
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.kafka.consumer.timeout.ms=100
+atlas.kafka.auto.commit.interval.ms=100
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.entities.group.id=atlas_entities
+atlas.kafka.auto.commit.enable=false
+
+######## JAAS configs ##################
+
+atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.KafkaClient.loginModuleControlFlag = required
+atlas.jaas.KafkaClient.option.useKeyTab = true
+atlas.jaas.KafkaClient.option.storeKey = true
+atlas.jaas.KafkaClient.option.serviceName = kafka
+atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/kafka_client.keytab
+atlas.jaas.KafkaClient.option.principal = kafka-client-1/_HOST@EXAMPLE.COM
+
+atlas.jaas.myClient.0.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.myClient.0.loginModuleControlFlag = required
+atlas.jaas.myClient.0.option.useKeyTab = true
+atlas.jaas.myClient.0.option.storeKey = true
+atlas.jaas.myClient.0.option.serviceName = kafka
+atlas.jaas.myClient.0.option.keyTab = /etc/security/keytabs/kafka_client.keytab
+atlas.jaas.myClient.0.option.principal = kafka-client-1/abcd@EXAMPLE.COM
+
+atlas.jaas.myClient.1.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.myClient.1.loginModuleControlFlag = optional
+atlas.jaas.myClient.1.option.useKeyTab = true
+atlas.jaas.myClient.1.option.storeKey = true
+atlas.jaas.myClient.1.option.serviceName = kafka
+atlas.jaas.myClient.1.option.keyTab = /etc/security/keytabs/kafka_client.keytab
+atlas.jaas.myClient.1.option.principal = kafka-client-1@EXAMPLE.COM
\ No newline at end of file

distro/src/conf/atlas-application.properties

@@ -89,6 +89,16 @@ atlas.http.authentication.enabled=false
 # type:  simple or kerberos
 atlas.http.authentication.type=simple
 
+#########  JAAS Configuration ########
+
+#atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+#atlas.jaas.KafkaClient.loginModuleControlFlag = required
+#atlas.jaas.KafkaClient.option.useKeyTab = true
+#atlas.jaas.KafkaClient.option.storeKey = true
+#atlas.jaas.KafkaClient.option.serviceName = kafka
+#atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab
+#atlas.jaas.KafkaClient.option.principal = atlas/_HOST@EXAMPLE.COM
+
 #########  Server Properties  #########
 atlas.rest.address=http://localhost:21000
 # If enabled and set to true, this will run setup steps when the server starts

docs/src/site/twiki/security.twiki

@@ -47,24 +47,76 @@ Note that when Atlas is configured with HBase as the storage backend in a secure
 
 ---+++ JAAS configuration
 
-When Atlas is configured with HBase as the storage backend in a secure cluster, a JAAS configuration file should be created and specified so that the HBase client can attempt to SASL-authenticate.
+In a secure cluster, some of the components (such as Kafka) that Atlas interacts with, require Atlas to authenticate itself to them using JAAS. The following properties are used to set up appropriate JAAS Configuration.
 
-   * Create Atlas JAAS configuration file (e.g. /etc/atlas/conf/atlas-jaas.conf).
+   * <code>atlas.jaas.<code>client-id<code>.loginModuleName<code> - the authentication method used by the component (for example, com.sun.security.auth.module.Krb5LoginModule)
+   * <code>atlas.jaas.<code>client-id<code>.loginModuleControlFlag<code> (required|requisite|sufficient|optional) [default: required]
+   * <code>atlas.jaas.<code>client-id<code>.option.useKeyTab<code> (true|false)
+   * <code>atlas.jaas.<code>client-id<code>.option.storeKey<code> (true | false)
+   * <code>atlas.jaas.<code>client-id<code>.option.serviceName<code> - service name of server component
+   * <code>atlas.jaas.<code>client-id<code>.option.keyTab<code> = <atlas keytab>
+   * <code>atlas.jaas.<code>client-id<code>.option.principal<code> = <atlas principal>
+
+For example, the following property settings in jaas-application.properties file
 
 <verbatim>
-	Client {
+
+atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.KafkaClient.loginModuleControlFlag = required
+atlas.jaas.KafkaClient.option.useKeyTab = true
+atlas.jaas.KafkaClient.option.storeKey = true
+atlas.jaas.KafkaClient.option.serviceName = kafka
+atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/kafka_client.keytab
+atlas.jaas.KafkaClient.option.principal = kafka-client-1@EXAMPLE.COM
+
+atlas.jaas.MyClient.0.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.MyClient.0.loginModuleControlFlag = required
+atlas.jaas.MyClient.0.option.useKeyTab = true
+atlas.jaas.MyClient.0.option.storeKey = true
+atlas.jaas.MyClient.0.option.serviceName = kafka
+atlas.jaas.MyClient.0.option.keyTab = /etc/security/keytabs/kafka_client.keytab
+atlas.jaas.MyClient.0.option.principal = kafka-client-1@EXAMPLE.COM
+
+atlas.jaas.MyClient.1.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.MyClient.1.loginModuleControlFlag = optional
+atlas.jaas.MyClient.1.option.useKeyTab = true
+atlas.jaas.MyClient.1.option.storeKey = true
+atlas.jaas.MyClient.1.option.serviceName = kafka
+atlas.jaas.MyClient.1.option.keyTab = /etc/security/keytabs/kafka_client.keytab
+atlas.jaas.MyClient.1.option.principal = kafka-client-1@EXAMPLE.COM
+
+</verbatim>
+
+will set the JAAS configuration that is equivalent to the following jaas.conf file entries.
+
+<verbatim>
+
+KafkaClient {
 	com.sun.security.auth.module.Krb5LoginModule required
 	useKeyTab=true
-	   useTicketCache=false
 	storeKey=true
-	   doNotPrompt=false
-	   keyTab="<atlas keytab>"
-	   principal="<atlas principal>";
-	};
+	serviceName=kafka
+	keyTab="/etc/security/keytabs/kafka_client.keytab"
+	principal="kafka-client-1@EXAMPLE.COM";
+};
+MyClient {
+	com.sun.security.auth.module.Krb5LoginModule required
+	useKeyTab=true
+	storeKey=true
+	serviceName=kafka keyTab="/etc/security/keytabs/kafka_client.keytab"
+	principal="kafka-client-1@EXAMPLE.COM";
+};
+MyClient {
+	com.sun.security.auth.module.Krb5LoginModule optional
+	useKeyTab=true
+	storeKey=true
+	serviceName=kafka
+	keyTab="/etc/security/keytabs/kafka_client.keytab"
+	principal="kafka-client-1@EXAMPLE.COM";
+};
+
 </verbatim>
 
-   * Update Atlas ATLAS_OPTS to include ‘java.security.auth.login.config’ set to the above Atlas JAAS configuration file.
-      * For example, <code>-Djava.security.auth.login.config=/etc/atlas/conf/atlas-jaas.conf</code>
 
 ---+++ SPNEGO-based HTTP Authentication
 

pom.xml

@@ -390,6 +390,7 @@
         <!-- Needed for hooks -->
         <aopalliance.version>1.0</aopalliance.version>
         <commons-conf.version>1.10</commons-conf.version>
+        <commons-collections.version>3.2.2</commons-collections.version>
         <commons-logging.version>1.1.3</commons-logging.version>
         <javax-inject.version>1</javax-inject.version>
         <jettison.version>1.3.7</jettison.version>
@@ -705,6 +706,12 @@
                 <version>2.4</version>
             </dependency>
 
+            <dependency>
+                <groupId>commons-collections</groupId>
+                <artifactId>commons-collections</artifactId>
+                <version>${commons-collections.version}</version>
+            </dependency>
+
             <!-- utilities -->
             <dependency>
                 <groupId>com.google.inject</groupId>

release-log.txt

@@ -21,6 +21,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-809 JAAS configuration needed for Kafka interaction via Atlas config file (abhayk via shwethags)
 ATLAS-817 Asset details page -- generate schema dynamically based on attributeDefinitions (kevalbhatt18 via yhemanth)
 ATLAS-495 Atlas Ranger Authorization Plugin (nixonrodrigues via shwethags)
 ATLAS-805 Quickstart is failing if run after queries to the business taxonomy API (jspeidel via shwethags)

typesystem/src/main/resources/atlas-application.properties

@@ -83,6 +83,17 @@ atlas.server.https.port=31443
 hbase.security.authentication=simple
 
 atlas.hook.falcon.synchronous=true
+
+#########  JAAS Configuration ########
+
+atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.KafkaClient.loginModuleControlFlag = required
+atlas.jaas.KafkaClient.option.useKeyTab = true
+atlas.jaas.KafkaClient.option.storeKey = true
+atlas.jaas.KafkaClient.option.serviceName = kafka
+atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab
+atlas.jaas.KafkaClient.option.principal = atlas/_HOST@EXAMPLE.COM
+
 #########  High Availability Configuration ########
 atlas.server.ha.enabled=false
 #atlas.server.ids=id1