ATLAS-2586: added import-kafka bridge to import Kafka topic into Atlas

addons/kafka-bridge/src/bin/import-kafka.sh

+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+#
+
+# resolve links - $0 may be a softlink
+PRG="${0}"
+
+[[ `uname -s` == *"CYGWIN"* ]] && CYGWIN=true
+
+while [ -h "${PRG}" ]; do
+  ls=`ls -ld "${PRG}"`
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    PRG="$link"
+  else
+    PRG=`dirname "${PRG}"`/"$link"
+  fi
+done
+
+echo ">>>>> $PRG"
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR}/..;pwd`
+
+echo ">>>>> $BASEDIR"
+
+allargs=$@
+
+if test -z "${JAVA_HOME}"
+then
+    JAVA_BIN=`which java`
+    JAR_BIN=`which jar`
+else
+    JAVA_BIN="${JAVA_HOME}/bin/java"
+    JAR_BIN="${JAVA_HOME}/bin/jar"
+fi
+export JAVA_BIN
+
+if [ ! -e "${JAVA_BIN}" ] || [ ! -e "${JAR_BIN}" ]; then
+  echo "$JAVA_BIN and/or $JAR_BIN not found on the system. Please make sure java and jar commands are available."
+  exit 1
+fi
+
+# Construct Atlas classpath using jars from hook/kafka/atlas-kafka-plugin-impl/ directory.
+for i in "${BASEDIR}/hook/kafka/atlas-kafka-plugin-impl/"*.jar; do
+  ATLASCPPATH="${ATLASCPPATH}:$i"
+done
+
+ATLAS_CONF_DIR=/etc/atlas/conf
+ATLASCPPATH=${ATLASCPPATH}:${ATLAS_CONF_DIR}
+
+# log dir for applications
+ATLAS_LOG_DIR="${ATLAS_LOG_DIR:-$BASEDIR/logs}"
+export ATLAS_LOG_DIR
+LOGFILE="$ATLAS_LOG_DIR/import-kafka.log"
+
+TIME=`date +%Y%m%d%H%M%s`
+
+#Add Kafka conf in classpath
+if [ ! -z "$KAFKA_CONF_DIR" ]; then
+    KAFKA_CONF=$KAFKA_CONF_DIR
+elif [ ! -z "$KAFKA_HOME" ]; then
+    KAFKA_CONF="$KAFKA_HOME/conf"
+elif [ -e /etc/kafka/conf ]; then
+    KAFKA_CONF="/etc/kafka/conf"
+else
+    echo "Could not find a valid KAFKA configuration"
+    exit 1
+fi
+
+echo Using Kafka configuration directory "[$KAFKA_CONF]"
+
+
+if [ -f "${KAFKA_CONF}/kafka-env.sh" ]; then
+  . "${KAFKA_CONF}/kafka-env.sh"
+fi
+
+if [ -z "$KAFKA_HOME" ]; then
+    if [ -d "${BASEDIR}/../kafka" ]; then
+        KAFKA_HOME=${BASEDIR}/../kafka
+    else
+        echo "Please set KAFKA_HOME to the root of Kafka installation"
+        exit 1
+    fi
+fi
+
+KAFKA_CP="${KAFKA_CONF}"
+
+for i in "${KAFKA_HOME}/libs/"*.jar; do
+    KAFKA_CP="${KAFKA_CP}:$i"
+done
+
+
+#Add hadoop conf in classpath
+if [ ! -z "$HADOOP_CLASSPATH" ]; then
+    HADOOP_CP=$HADOOP_CLASSPATH
+elif [ ! -z "$HADOOP_HOME" ]; then
+    HADOOP_CP=`$HADOOP_HOME/bin/hadoop classpath`
+elif [ $(command -v hadoop) ]; then
+    HADOOP_CP=`hadoop classpath`
+   #echo $HADOOP_CP
+else
+    echo "Environment variable HADOOP_CLASSPATH or HADOOP_HOME need to be set"
+    exit 1
+fi
+
+CP="${KAFKA_CP}:${ATLASCPPATH}:${HADOOP_CP}"
+
+# If running in cygwin, convert pathnames and classpath to Windows format.
+if [ "${CYGWIN}" == "true" ]
+then
+   ATLAS_LOG_DIR=`cygpath -w ${ATLAS_LOG_DIR}`
+   LOGFILE=`cygpath -w ${LOGFILE}`
+   KAFKA_CP=`cygpath -w ${KAFKA_CP}`
+   HADOOP_CP=`cygpath -w ${HADOOP_CP}`
+   CP=`cygpath -w -p ${CP}`
+fi
+
+JAVA_PROPERTIES="$ATLAS_OPTS -Datlas.log.dir=$ATLAS_LOG_DIR -Datlas.log.file=import-kafka.log
+-Dlog4j.configuration=atlas-kafka-import-log4j.xml"
+shift
+
+while [[ ${1} =~ ^\-D ]]; do
+  JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
+  shift
+done
+
+echo "Log file for import is $LOGFILE"
+
+"${JAVA_BIN}" ${JAVA_PROPERTIES} -cp "${CP}" org.apache.atlas.kafka.bridge.KafkaBridge $allargs
+
+RETVAL=$?
+[ $RETVAL -eq 0 ] && echo Kafka Data Model imported successfully!!!
+[ $RETVAL -ne 0 ] && echo Failed to import Kafka Data Model!!!
\ No newline at end of file

addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/model/KafkaDataTypes.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.kafka.model;
+
+/**
+ * HBASE Data Types for model and bridge.
+ */
+public enum KafkaDataTypes {
+    // Classes
+    KAFKA_TOPIC;
+
+    public String getName() {
+        return name().toLowerCase();
+    }
+}

addons/kafka-bridge/src/main/resources/atlas-kafka-import-log4j.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+    <appender name="console" class="org.apache.log4j.ConsoleAppender">
+        <param name="Target" value="System.out"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
+        </layout>
+    </appender>
+
+    <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="${atlas.log.dir}/${atlas.log.file}"/>
+        <param name="Append" value="true"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
+        </layout>
+    </appender>
+
+
+    <logger name="org.apache.atlas" additivity="false">
+        <level value="info"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+    <!-- to avoid logs - The configuration log.flush.interval.messages = 1 was supplied but isn't a known config -->
+    <logger name="org.apache.kafka.common.config.AbstractConfig" additivity="false">
+        <level value="error"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+    <root>
+        <priority value="info"/>
+        <appender-ref ref="FILE"/>
+    </root>
+
+</log4j:configuration>

addons/models/1000-Hadoop/patches/006-kafka_topic_add_attribute.json

+{
+  "patches": [
+    {
+      "action": "ADD_ATTRIBUTE",
+      "typeName": "kafka_topic",
+      "applyToVersion": "1.0",
+      "updateToVersion": "1.1",
+      "params": null,
+      "attributeDefs": [
+        {
+          "name": "partitionCount",
+          "typeName": "int",
+          "cardinality": "SINGLE",
+          "isIndexable": true,
+          "isOptional": true,
+          "isUnique": false
+        }
+      ]
+    }
+  ]
+}

distro/pom.xml

@@ -116,6 +116,7 @@ atlas.graph.index.search.solr.wait-searcher=true
                                         <descriptor>src/main/assemblies/atlas-falcon-hook-package.xml</descriptor>
                                         <descriptor>src/main/assemblies/atlas-sqoop-hook-package.xml</descriptor>
                                         <descriptor>src/main/assemblies/atlas-storm-hook-package.xml</descriptor>
+                                        <descriptor>src/main/assemblies/atlas-kafka-hook-package.xml</descriptor>
                                         <descriptor>src/main/assemblies/standalone-package.xml</descriptor>
                                         <descriptor>src/main/assemblies/src-package.xml</descriptor>
                                         <descriptor>src/main/assemblies/migration-exporter.xml</descriptor>

distro/src/main/assemblies/atlas-kafka-hook-package.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<assembly xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <formats>
+        <format>tar.gz</format>
+        <format>dir</format>
+    </formats>
+    <id>kafka-hook</id>
+    <baseDirectory>apache-atlas-kafka-hook-${project.version}</baseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>target/bin</directory>
+            <outputDirectory>hook-bin</outputDirectory>
+            <includes>
+                <include>import-kafka.sh</include>
+            </includes>
+            <fileMode>0755</fileMode>
+            <directoryMode>0755</directoryMode>
+        </fileSet>
+
+        <!-- addons/kafka -->
+        <fileSet>
+            <directory>../addons/kafka-bridge/src/bin</directory>
+            <outputDirectory>hook-bin</outputDirectory>
+            <fileMode>0755</fileMode>
+            <directoryMode>0755</directoryMode>
+        </fileSet>
+
+        <!-- addons/hbase -->
+        <fileSet>
+            <directory>../addons/kafka-bridge/target/dependency/bridge</directory>
+            <outputDirectory>bridge</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>../addons/kafka-bridge/target/dependency/hook</directory>
+            <outputDirectory>hook</outputDirectory>
+        </fileSet>
+    </fileSets>
+</assembly>

distro/src/main/assemblies/standalone-package.xml

@@ -176,7 +176,18 @@
 
         <!-- for kafka topic setup -->
         <fileSet>
-            <directory>../notification/target/dependency/hook</directory>
+            <directory>../addons/kafka-bridge/src/bin</directory>
+            <outputDirectory>hook-bin</outputDirectory>
+            <fileMode>0755</fileMode>
+            <directoryMode>0755</directoryMode>
+        </fileSet>
+
+        <fileSet>
+            <directory>../addons/kakfa-bridge/target/dependency/bridge</directory>
+            <outputDirectory>bridge</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>../addons/kakfa-bridge/target/dependency/hook</directory>
             <outputDirectory>hook</outputDirectory>
         </fileSet>
     </fileSets>

pom.xml

@@ -767,6 +767,7 @@
         <module>addons/storm-bridge</module>
         <module>addons/hbase-bridge-shim</module>
         <module>addons/hbase-bridge</module>
+        <module>addons/kafka-bridge</module>
         <module>tools/atlas-migration-exporter</module>
 	
         <module>distro</module>
@@ -1492,6 +1493,11 @@
                 <version>${project.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.apache.atlas</groupId>
+                <artifactId>kafka-bridge</artifactId>
+                <version>${project.version}</version>
+            </dependency>
 
             <!-- API documentation  -->
             <dependency>