ATLAS-2525: updated HBase, Hive hooks to enable import namespaces/databases/tables listed in a file

addons/hbase-bridge/src/bin/import-hbase.sh

@@ -134,7 +134,7 @@ done
 
 echo "Log file for import is $LOGFILE"
 
-"${JAVA_BIN}" ${JAVA_PROPERTIES} -cp "${CP}" org.apache.atlas.hbase.util.ImportHBaseEntities $allargs
+"${JAVA_BIN}" ${JAVA_PROPERTIES} -cp "${CP}" org.apache.atlas.hbase.bridge.HBaseBridge $allargs
 
 RETVAL=$?
 [ $RETVAL -eq 0 ] && echo HBase Data Model imported successfully!!!

addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/util/ImportHBaseEntities.java

-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.hbase.util;
-
-
-import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.hook.AtlasHookException;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class ImportHBaseEntities extends ImportHBaseEntitiesBase {
-    private static final Logger LOG = LoggerFactory.getLogger(ImportHBaseEntities.class);
-
-    public static void main(String[] args) throws AtlasHookException {
-        try {
-            ImportHBaseEntities importHBaseEntities = new ImportHBaseEntities(args);
-
-            importHBaseEntities.execute();
-        } catch(Exception e) {
-            throw new AtlasHookException("ImportHBaseEntities failed.", e);
-        }
-    }
-
-    public ImportHBaseEntities(String[] args) throws Exception {
-        super(args);
-    }
-
-    public boolean execute() throws Exception {
-        boolean ret = false;
-
-        if (hbaseAdmin != null) {
-            if (StringUtils.isEmpty(namespaceToImport) && StringUtils.isEmpty(tableToImport)) {
-                NamespaceDescriptor[] namespaceDescriptors = hbaseAdmin.listNamespaceDescriptors();
-                if (!ArrayUtils.isEmpty(namespaceDescriptors)) {
-                    for (NamespaceDescriptor namespaceDescriptor : namespaceDescriptors) {
-                        String namespace = namespaceDescriptor.getName();
-                        importNameSpace(namespace);
-                    }
-                }
-                HTableDescriptor[] htds = hbaseAdmin.listTables();
-                if (!ArrayUtils.isEmpty(htds)) {
-                    for (HTableDescriptor htd : htds) {
-                        String tableName = htd.getNameAsString();
-                        importTable(tableName);
-                    }
-                }
-                ret = true;
-            } else if (StringUtils.isNotEmpty(namespaceToImport)) {
-                importNameSpace(namespaceToImport);
-                ret = true;
-            } else if (StringUtils.isNotEmpty(tableToImport)) {
-                importTable(tableToImport);
-                ret = true;
-            }
-        }
-
-        return ret;
-    }
-
-    public String importNameSpace(final String nameSpace) throws Exception {
-        NamespaceDescriptor namespaceDescriptor = hbaseAdmin.getNamespaceDescriptor(nameSpace);
-
-        createOrUpdateNameSpace(namespaceDescriptor);
-
-        return namespaceDescriptor.getName();
-    }
-
-    public String importTable(final String tableName) throws Exception {
-        byte[]              tblName      = tableName.getBytes();
-        HTableDescriptor    htd          = hbaseAdmin.getTableDescriptor(tblName);
-        String              nsName       = htd.getTableName().getNameWithNamespaceInclAsString();
-        NamespaceDescriptor nsDescriptor = hbaseAdmin.getNamespaceDescriptor(nsName);
-        AtlasEntity         nsEntity     = createOrUpdateNameSpace(nsDescriptor);
-        HColumnDescriptor[] hcdts        = htd.getColumnFamilies();
-        createOrUpdateTable(nsName, tableName, nsEntity, htd, hcdts);
-
-        return htd.getTableName().getNameAsString();
-    }
-}

addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java

@@ -44,6 +44,7 @@ import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Options;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.RandomStringUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.Path;
@@ -62,6 +63,9 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -102,14 +106,16 @@ public class HiveMetaStoreBridge {
 
         try {
             Options options = new Options();
-            options.addOption("d", "database", true, "Databbase name");
+            options.addOption("d", "database", true, "Database name");
             options.addOption("t", "table", true, "Table name");
+            options.addOption("f", "filename", true, "Filename");
             options.addOption("failOnError", false, "failOnError");
 
             CommandLine   cmd              = new BasicParser().parse(options, args);
             boolean       failOnError      = cmd.hasOption("failOnError");
             String        databaseToImport = cmd.getOptionValue("d");
             String        tableToImport    = cmd.getOptionValue("t");
+            String        fileToImport     = cmd.getOptionValue("f");
             Configuration atlasConf        = ApplicationProperties.get();
             String[]      atlasEndpoint    = atlasConf.getStringArray(ATLAS_ENDPOINT);
 
@@ -131,12 +137,40 @@ public class HiveMetaStoreBridge {
 
             HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClientV2);
 
+            if (StringUtils.isNotEmpty(fileToImport)) {
+                File f = new File(fileToImport);
+
+                if (f.exists() && f.canRead()) {
+                    BufferedReader br   = new BufferedReader(new FileReader(f));
+                    String         line = null;
+
+                    while((line = br.readLine()) != null) {
+                        String val[] = line.split(":");
+
+                        if (ArrayUtils.isNotEmpty(val)) {
+                            databaseToImport = val[0];
+
+                            if (val.length > 1) {
+                                tableToImport = val[1];
+                            } else {
+                                tableToImport = "";
+                            }
+
                             hiveMetaStoreBridge.importHiveMetadata(databaseToImport, tableToImport, failOnError);
+                        }
+                    }
+
+                    exitCode = EXIT_CODE_SUCCESS;
+                } else {
+                    LOG.error("Failed to read the input file: " + fileToImport);
+                }
+            } else {
+                hiveMetaStoreBridge.importHiveMetadata(databaseToImport, tableToImport, failOnError);
+            }
 
             exitCode = EXIT_CODE_SUCCESS;
         } catch(ParseException e) {
             LOG.error("Failed to parse arguments. Error: ", e.getMessage());
-
             printUsage();
         } catch(Exception e) {
             LOG.error("Import failed", e);
@@ -157,6 +191,12 @@ public class HiveMetaStoreBridge {
         System.out.println("Usage 3: import-hive.sh");
         System.out.println("    Imports all databases and tables...");
         System.out.println();
+        System.out.println("Usage 4: import-hive.sh -f <filename>");
+        System.out.println("  Imports all databases and tables in the file...");
+        System.out.println("    Format:");
+        System.out.println("    database1:tbl1");
+        System.out.println("    database1:tbl2");
+        System.out.println("    database2:tbl2");
         System.out.println();
     }