Skip to content

A
atlas
Commit 7f5a665e by rmani Committed by Madhan Neethiraj
Options

ATLAS-2525: updated HBase, Hive hooks to enable import namespaces/databases/tables listed in a file

Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent f42c1d9f
Showing with 44 additions and 105 deletions
addons/hbase-bridge/src/bin/import-hbase.sh
......@@ -134,7 +134,7 @@ done
echo "Log file for import is $LOGFILE"
"${JAVA_BIN}" ${JAVA_PROPERTIES} -cp "${CP}" org.apache.atlas.hbase.util.ImportHBaseEntities $allargs
"${JAVA_BIN}" ${JAVA_PROPERTIES} -cp "${CP}" org.apache.atlas.hbase.bridge.HBaseBridge $allargs
RETVAL=$?
[ $RETVAL -eq 0 ] && echo HBase Data Model imported successfully!!!
......
addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/util/ImportHBaseEntities.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.hbase.util;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.hook.AtlasHookException;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ImportHBaseEntities extends ImportHBaseEntitiesBase {
private static final Logger LOG = LoggerFactory.getLogger(ImportHBaseEntities.class);
public static void main(String[] args) throws AtlasHookException {
try {
ImportHBaseEntities importHBaseEntities = new ImportHBaseEntities(args);
importHBaseEntities.execute();
} catch(Exception e) {
throw new AtlasHookException("ImportHBaseEntities failed.", e);
}
}
public ImportHBaseEntities(String[] args) throws Exception {
super(args);
}
public boolean execute() throws Exception {
boolean ret = false;
if (hbaseAdmin != null) {
if (StringUtils.isEmpty(namespaceToImport) && StringUtils.isEmpty(tableToImport)) {
NamespaceDescriptor[] namespaceDescriptors = hbaseAdmin.listNamespaceDescriptors();
if (!ArrayUtils.isEmpty(namespaceDescriptors)) {
for (NamespaceDescriptor namespaceDescriptor : namespaceDescriptors) {
String namespace = namespaceDescriptor.getName();
importNameSpace(namespace);
}
}
HTableDescriptor[] htds = hbaseAdmin.listTables();
if (!ArrayUtils.isEmpty(htds)) {
for (HTableDescriptor htd : htds) {
String tableName = htd.getNameAsString();
importTable(tableName);
}
}
ret = true;
} else if (StringUtils.isNotEmpty(namespaceToImport)) {
importNameSpace(namespaceToImport);
ret = true;
} else if (StringUtils.isNotEmpty(tableToImport)) {
importTable(tableToImport);
ret = true;
}
}
return ret;
}
public String importNameSpace(final String nameSpace) throws Exception {
NamespaceDescriptor namespaceDescriptor = hbaseAdmin.getNamespaceDescriptor(nameSpace);
createOrUpdateNameSpace(namespaceDescriptor);
return namespaceDescriptor.getName();
}
public String importTable(final String tableName) throws Exception {
byte[] tblName = tableName.getBytes();
HTableDescriptor htd = hbaseAdmin.getTableDescriptor(tblName);
String nsName = htd.getTableName().getNameWithNamespaceInclAsString();
NamespaceDescriptor nsDescriptor = hbaseAdmin.getNamespaceDescriptor(nsName);
AtlasEntity nsEntity = createOrUpdateNameSpace(nsDescriptor);
HColumnDescriptor[] hcdts = htd.getColumnFamilies();
createOrUpdateTable(nsName, tableName, nsEntity, htd, hcdts);
return htd.getTableName().getNameAsString();
}
}
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
......@@ -44,6 +44,7 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;
......@@ -62,6 +63,9 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
......@@ -102,14 +106,16 @@ public class HiveMetaStoreBridge {
try {
Options options = new Options();
options.addOption("d", "database", true, "Databbase name");
options.addOption("d", "database", true, "Database name");
options.addOption("t", "table", true, "Table name");
options.addOption("f", "filename", true, "Filename");
options.addOption("failOnError", false, "failOnError");
CommandLine cmd = new BasicParser().parse(options, args);
boolean failOnError = cmd.hasOption("failOnError");
String databaseToImport = cmd.getOptionValue("d");
String tableToImport = cmd.getOptionValue("t");
String fileToImport = cmd.getOptionValue("f");
Configuration atlasConf = ApplicationProperties.get();
String[] atlasEndpoint = atlasConf.getStringArray(ATLAS_ENDPOINT);
......@@ -131,12 +137,40 @@ public class HiveMetaStoreBridge {
HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClientV2);
hiveMetaStoreBridge.importHiveMetadata(databaseToImport, tableToImport, failOnError);
if (StringUtils.isNotEmpty(fileToImport)) {
File f = new File(fileToImport);
if (f.exists() && f.canRead()) {
BufferedReader br = new BufferedReader(new FileReader(f));
String line = null;
while((line = br.readLine()) != null) {
String val[] = line.split(":");
if (ArrayUtils.isNotEmpty(val)) {
databaseToImport = val[0];
if (val.length > 1) {
tableToImport = val[1];
} else {
tableToImport = "";
}
hiveMetaStoreBridge.importHiveMetadata(databaseToImport, tableToImport, failOnError);
}
}
exitCode = EXIT_CODE_SUCCESS;
} else {
LOG.error("Failed to read the input file: " + fileToImport);
}
} else {
hiveMetaStoreBridge.importHiveMetadata(databaseToImport, tableToImport, failOnError);
}
exitCode = EXIT_CODE_SUCCESS;
} catch(ParseException e) {
LOG.error("Failed to parse arguments. Error: ", e.getMessage());
printUsage();
} catch(Exception e) {
LOG.error("Import failed", e);
......@@ -157,6 +191,12 @@ public class HiveMetaStoreBridge {
System.out.println("Usage 3: import-hive.sh");
System.out.println(" Imports all databases and tables...");
System.out.println();
System.out.println("Usage 4: import-hive.sh -f <filename>");
System.out.println(" Imports all databases and tables in the file...");
System.out.println(" Format:");
System.out.println(" database1:tbl1");
System.out.println(" database1:tbl2");
System.out.println(" database2:tbl2");
System.out.println();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment