ATLAS-2051: Provide a utility to import HBase entities into Atlas

.gitignore

@@ -37,7 +37,7 @@ pom.xml.releaseBackup
 maven-eclipse.xml
 
 #binary files
-**/bin/**
+#**/bin/**
 !distro/src/bin/**
 
 #log files

addons/hbase-bridge/pom.xml

@@ -61,6 +61,11 @@
 
         <dependency>
             <groupId>org.apache.atlas</groupId>
+            <artifactId>atlas-client-v2</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.atlas</groupId>
             <artifactId>atlas-notification</artifactId>
         </dependency>
 
@@ -247,6 +252,11 @@
                                         </artifactItem>
                                         <artifactItem>
                                             <groupId>${project.groupId}</groupId>
+                                            <artifactId>atlas-client-v2</artifactId>
+                                            <version>${project.version}</version>
+                                        </artifactItem>
+                                        <artifactItem>
+                                            <groupId>${project.groupId}</groupId>
                                             <artifactId>atlas-intg</artifactId>
                                             <version>${project.version}</version>
                                         </artifactItem>
@@ -275,6 +285,41 @@
                                             <artifactId>kafka-clients</artifactId>
                                             <version>${kafka.version}</version>
                                         </artifactItem>
+                                        <artifactItem>
+                                            <groupId>com.sun.jersey.contribs</groupId>
+                                            <artifactId>jersey-multipart</artifactId>
+                                            <version>${jersey.version}</version>
+                                        </artifactItem>
+                                        <artifactItem>
+                                            <groupId>org.scala-lang</groupId>
+                                            <artifactId>scala-library</artifactId>
+                                            <version>${scala.version}</version>
+                                        </artifactItem>
+                                        <artifactItem>
+                                            <groupId>com.fasterxml.jackson.core</groupId>
+                                            <artifactId>jackson-databind</artifactId>
+                                            <version>${jackson.version}</version>
+                                        </artifactItem>
+                                        <artifactItem>
+                                            <groupId>com.fasterxml.jackson.core</groupId>
+                                            <artifactId>jackson-core</artifactId>
+                                            <version>${jackson.version}</version>
+                                        </artifactItem>
+                                        <artifactItem>
+                                            <groupId>com.fasterxml.jackson.core</groupId>
+                                            <artifactId>jackson-annotations</artifactId>
+                                            <version>${jackson.version}</version>
+                                        </artifactItem>
+                                        <artifactItem>
+                                            <groupId>commons-configuration</groupId>
+                                            <artifactId>commons-configuration</artifactId>
+                                            <version>${commons-conf.version}</version>
+                                        </artifactItem>
+                                        <artifactItem>
+                                            <groupId>org.apache.hbase</groupId>
+                                            <artifactId>hbase-common</artifactId>
+                                            <version>${hbase.version}</version>
+                                        </artifactItem>
                                     </artifactItems>
                                 </configuration>
                             </execution>

addons/hbase-bridge/src/bin/import-hbase.sh

+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+#
+
+# resolve links - $0 may be a softlink
+PRG="${0}"
+
+[[ `uname -s` == *"CYGWIN"* ]] && CYGWIN=true
+
+while [ -h "${PRG}" ]; do
+  ls=`ls -ld "${PRG}"`
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    PRG="$link"
+  else
+    PRG=`dirname "${PRG}"`/"$link"
+  fi
+done
+
+echo ">>>>> $PRG"
+
+BASEDIR=`dirname ${PRG}`
+BASEDIR=`cd ${BASEDIR}/..;pwd`
+
+echo ">>>>> $BASEDIR"
+
+allargs=$@
+
+if test -z "${JAVA_HOME}"
+then
+    JAVA_BIN=`which java`
+    JAR_BIN=`which jar`
+else
+    JAVA_BIN="${JAVA_HOME}/bin/java"
+    JAR_BIN="${JAVA_HOME}/bin/jar"
+fi
+export JAVA_BIN
+
+if [ ! -e "${JAVA_BIN}" ] || [ ! -e "${JAR_BIN}" ]; then
+  echo "$JAVA_BIN and/or $JAR_BIN not found on the system. Please make sure java and jar commands are available."
+  exit 1
+fi
+
+# Construct Atlas classpath using jars from hook/hbase/atlas-hbase-plugin-impl/ directory.
+for i in "${BASEDIR}/hook/hbase/atlas-hbase-plugin-impl/"*.jar; do
+  ATLASCPPATH="${ATLASCPPATH}:$i"
+done
+
+# log dir for applications
+ATLAS_LOG_DIR="${ATLAS_LOG_DIR:-$BASEDIR/logs}"
+export ATLAS_LOG_DIR
+LOGFILE="$ATLAS_LOG_DIR/import-hbase.log"
+
+TIME=`date +%Y%m%d%H%M%s`
+
+#Add HBase conf in classpath
+if [ ! -z "$HBASE_CONF_DIR" ]; then
+    HBASE_CONF=$HBASE_CONF_DIR
+elif [ ! -z "$HBASE_HOME" ]; then
+    HBASE_CONF="$HBASE_HOME/conf"
+elif [ -e /etc/hbase/conf ]; then
+    HBASE_CONF="/etc/hbase/conf"
+else
+    echo "Could not find a valid HBASE configuration"
+    exit 1
+fi
+
+echo Using HBase configuration directory "[$HBASE_CONF]"
+
+
+if [ -f "${HBASE_CONF}/hbase-env.sh" ]; then
+  . "${HBASE_CONF}/hbase-env.sh"
+fi
+
+if [ -z "$HBASE_HOME" ]; then
+    if [ -d "${BASEDIR}/../hbase" ]; then
+        HBASE_HOME=${BASEDIR}/../hbase
+    else
+        echo "Please set HBASE_HOME to the root of HBase installation"
+        exit 1
+    fi
+fi
+
+HBASE_CP="${HBASE_CONF}"
+
+for i in "${HBASE_HOME}/lib/"*.jar; do
+    HBASE_CP="${HBASE_CP}:$i"
+done
+
+#Add hadoop conf in classpath
+if [ ! -z "$HADOOP_CLASSPATH" ]; then
+    HADOOP_CP=$HADOOP_CLASSPATH
+elif [ ! -z "$HADOOP_HOME" ]; then
+    HADOOP_CP=`$HADOOP_HOME/bin/hadoop classpath`
+elif [ $(command -v hadoop) ]; then
+    HADOOP_CP=`hadoop classpath`
+    echo $HADOOP_CP
+else
+    echo "Environment variable HADOOP_CLASSPATH or HADOOP_HOME need to be set"
+    exit 1
+fi
+
+CP="${ATLASCPPATH}:${HBASE_CP}:${HADOOP_CP}"
+
+# If running in cygwin, convert pathnames and classpath to Windows format.
+if [ "${CYGWIN}" == "true" ]
+then
+   ATLAS_LOG_DIR=`cygpath -w ${ATLAS_LOG_DIR}`
+   LOGFILE=`cygpath -w ${LOGFILE}`
+   HBASE_CP=`cygpath -w ${HBASE_CP}`
+   HADOOP_CP=`cygpath -w ${HADOOP_CP}`
+   CP=`cygpath -w -p ${CP}`
+fi
+
+JAVA_PROPERTIES="$ATLAS_OPTS -Datlas.log.dir=$ATLAS_LOG_DIR -Datlas.log.file=import-hbase.log
+-Dlog4j.configuration=atlas-hbase-import-log4j.xml"
+shift
+
+while [[ ${1} =~ ^\-D ]]; do
+  JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}"
+  shift
+done
+
+echo "Log file for import is $LOGFILE"
+
+"${JAVA_BIN}" ${JAVA_PROPERTIES} -cp "${CP}" org.apache.atlas.hbase.util.ImportHBaseEntities $allargs
+
+RETVAL=$?
+[ $RETVAL -eq 0 ] && echo HBase Data Model imported successfully!!!
+[ $RETVAL -ne 0 ] && echo Failed to import HBase Data Model!!!

addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/util/ImportHBaseEntities.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hbase.util;
+
+
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.hook.AtlasHookException;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class ImportHBaseEntities extends ImportHBaseEntitiesBase {
+    private static final Logger LOG = LoggerFactory.getLogger(ImportHBaseEntities.class);
+
+    public static void main(String[] args) throws AtlasHookException {
+        try {
+            ImportHBaseEntities importHBaseEntities = new ImportHBaseEntities(args);
+
+            importHBaseEntities.execute();
+        } catch(Exception e) {
+            throw new AtlasHookException("ImportHBaseEntities failed.", e);
+        }
+    }
+
+    public ImportHBaseEntities(String[] args) throws Exception {
+        super(args);
+    }
+
+    public boolean execute() throws Exception {
+        boolean ret = false;
+
+        if (hbaseAdmin != null) {
+            if (StringUtils.isEmpty(namespaceToImport) && StringUtils.isEmpty(tableToImport)) {
+                NamespaceDescriptor[] namespaceDescriptors = hbaseAdmin.listNamespaceDescriptors();
+                if (!ArrayUtils.isEmpty(namespaceDescriptors)) {
+                    for (NamespaceDescriptor namespaceDescriptor : namespaceDescriptors) {
+                        String namespace = namespaceDescriptor.getName();
+                        importNameSpace(namespace);
+                    }
+                }
+                HTableDescriptor[] htds = hbaseAdmin.listTables();
+                if (!ArrayUtils.isEmpty(htds)) {
+                    for (HTableDescriptor htd : htds) {
+                        String tableName = htd.getNameAsString();
+                        importTable(tableName);
+                    }
+                }
+                ret = true;
+            } else if (StringUtils.isNotEmpty(namespaceToImport)) {
+                importNameSpace(namespaceToImport);
+                ret = true;
+            } else if (StringUtils.isNotEmpty(tableToImport)) {
+                importTable(tableToImport);
+                ret = true;
+            }
+        }
+
+        return ret;
+    }
+
+    public String importNameSpace(final String nameSpace) throws Exception {
+        NamespaceDescriptor namespaceDescriptor = hbaseAdmin.getNamespaceDescriptor(nameSpace);
+
+        createOrUpdateNameSpace(namespaceDescriptor);
+
+        return namespaceDescriptor.getName();
+    }
+
+    public String importTable(final String tableName) throws Exception {
+        byte[]              tblName      = tableName.getBytes();
+        HTableDescriptor    htd          = hbaseAdmin.getTableDescriptor(tblName);
+        String              nsName       = htd.getTableName().getNamespaceAsString();
+        NamespaceDescriptor nsDescriptor = hbaseAdmin.getNamespaceDescriptor(nsName);
+        AtlasEntity         nsEntity     = createOrUpdateNameSpace(nsDescriptor);
+        HColumnDescriptor[] hcdts        = htd.getColumnFamilies();
+        createOrUpdateTable(nsName, tableName, nsEntity, htd, hcdts);
+
+        return htd.getTableName().getNameAsString();
+    }
+}

distro/src/main/assemblies/standalone-package.xml

@@ -132,6 +132,14 @@
 
         <!-- addons/hbase -->
         <fileSet>
+            <directory>../addons/hbase-bridge/src/bin</directory>
+            <outputDirectory>hook-bin</outputDirectory>
+            <fileMode>0755</fileMode>
+            <directoryMode>0755</directoryMode>
+        </fileSet>
+
+        <!-- addons/hbase -->
+        <fileSet>
             <directory>../addons/hbase-bridge/target/dependency/bridge</directory>
             <outputDirectory>bridge</outputDirectory>
         </fileSet>