import-hive.sh 4.22 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License. See accompanying LICENSE file.
#
# resolve links - $0 may be a softlink
PRG="${0}"

18 19
[[ `uname -s` == *"CYGWIN"* ]] && CYGWIN=true

20 21 22 23 24 25 26 27 28 29 30 31 32
while [ -h "${PRG}" ]; do
  ls=`ls -ld "${PRG}"`
  link=`expr "$ls" : '.*-> \(.*\)$'`
  if expr "$link" : '/.*' > /dev/null; then
    PRG="$link"
  else
    PRG=`dirname "${PRG}"`/"$link"
  fi
done

BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/..;pwd`

33
if test -z "${JAVA_HOME}"
34 35 36 37
then
    JAVA_BIN=`which java`
    JAR_BIN=`which jar`
else
38 39
    JAVA_BIN="${JAVA_HOME}/bin/java"
    JAR_BIN="${JAVA_HOME}/bin/jar"
40 41 42
fi
export JAVA_BIN

43
if [ ! -e "${JAVA_BIN}" ] || [ ! -e "${JAR_BIN}" ]; then
44 45 46 47
  echo "$JAVA_BIN and/or $JAR_BIN not found on the system. Please make sure java and jar commands are available."
  exit 1
fi

48 49
# Construct Atlas classpath using jars from hook/hive/atlas-hive-plugin-impl/ directory.
for i in "${BASEDIR}/hook/hive/atlas-hive-plugin-impl/"*.jar; do
50
  ATLASCPPATH="${ATLASCPPATH}:$i"
51 52
done

53 54 55 56 57
if [ -z "${ATLAS_CONF_DIR}" ] && [ -e /etc/atlas/conf ];then
    ATLAS_CONF_DIR=/etc/atlas/conf
fi
ATLASCPPATH=${ATLASCPPATH}:${ATLAS_CONF_DIR}

58
# log dir for applications
59
ATLAS_LOG_DIR="${ATLAS_LOG_DIR:-/var/log/atlas}"
60 61
export ATLAS_LOG_DIR
LOGFILE="$ATLAS_LOG_DIR/import-hive.log"
62 63 64

TIME=`date +%Y%m%d%H%M%s`

65
#Add hive conf in classpath
66
if [ ! -z "$HIVE_CONF_DIR" ]; then
67
    HIVE_CONF=$HIVE_CONF_DIR
68
elif [ ! -z "$HIVE_HOME" ]; then
69
    HIVE_CONF="$HIVE_HOME/conf"
70
elif [ -e /etc/hive/conf ]; then
71
    HIVE_CONF="/etc/hive/conf"
72 73 74 75 76
else
    echo "Could not find a valid HIVE configuration"
    exit 1
fi

77 78 79 80 81 82
echo Using Hive configuration directory ["$HIVE_CONF"]


if [ -f "${HIVE_CONF}/hive-env.sh" ]; then
  . "${HIVE_CONF}/hive-env.sh"
fi
83 84

if [ -z "$HIVE_HOME" ]; then
85 86 87 88 89 90
    if [ -d "${BASEDIR}/../hive" ]; then
        HIVE_HOME=${BASEDIR}/../hive
    else
        echo "Please set HIVE_HOME to the root of Hive installation"
        exit 1
    fi
91 92
fi

93 94
HIVE_CP="${HIVE_CONF}"

95 96 97 98
for i in "${HIVE_HOME}/lib/"*.jar; do
    HIVE_CP="${HIVE_CP}:$i"
done

99 100 101 102 103
#Add hadoop conf in classpath
if [ ! -z "$HADOOP_CLASSPATH" ]; then
    HADOOP_CP=$HADOOP_CLASSPATH
elif [ ! -z "$HADOOP_HOME" ]; then
    HADOOP_CP=`$HADOOP_HOME/bin/hadoop classpath`
104 105 106
elif [ $(command -v hadoop) ]; then
    HADOOP_CP=`hadoop classpath`
    echo $HADOOP_CP
107 108 109 110 111
else
    echo "Environment variable HADOOP_CLASSPATH or HADOOP_HOME need to be set"
    exit 1
fi

112
CP="${HIVE_CP}:${HADOOP_CP}:${ATLASCPPATH}"
113 114 115 116

# If running in cygwin, convert pathnames and classpath to Windows format.
if [ "${CYGWIN}" == "true" ]
then
117
   ATLAS_LOG_DIR=`cygpath -w ${ATLAS_LOG_DIR}`
118 119
   LOGFILE=`cygpath -w ${LOGFILE}`
   HIVE_CP=`cygpath -w ${HIVE_CP}`
120
   HADOOP_CP=`cygpath -w ${HADOOP_CP}`
121 122 123
   CP=`cygpath -w -p ${CP}`
fi

124
JAVA_PROPERTIES="$ATLAS_OPTS -Datlas.log.dir=$ATLAS_LOG_DIR -Datlas.log.file=import-hive.log
125
-Dlog4j.configuration=atlas-hive-import-log4j.xml"
126

127 128 129 130 131 132
IMPORT_ARGS=
JVM_ARGS=

while true
do
  option=$1
133
  shift
134 135 136 137 138 139 140 141

  case "$option" in
    -d) IMPORT_ARGS="$IMPORT_ARGS -d $1"; shift;;
    -t) IMPORT_ARGS="$IMPORT_ARGS -t $1"; shift;;
    -f) IMPORT_ARGS="$IMPORT_ARGS -f $1"; shift;;
    --database) IMPORT_ARGS="$IMPORT_ARGS --database $1"; shift;;
    --table) IMPORT_ARGS="$IMPORT_ARGS --table $1"; shift;;
    --filename) IMPORT_ARGS="$IMPORT_ARGS --filename $1"; shift;;
142
    -deleteNonExisting) IMPORT_ARGS="$IMPORT_ARGS -deleteNonExisting";;
143 144 145
    "") break;;
    *) JVM_ARGS="$JVM_ARGS $option"
  esac
146 147
done

148 149
JAVA_PROPERTIES="${JAVA_PROPERTIES} ${JVM_ARGS}"

150 151
echo "Log file for import is $LOGFILE"

152
"${JAVA_BIN}" ${JAVA_PROPERTIES} -cp "${CP}" org.apache.atlas.hive.bridge.HiveMetaStoreBridge $IMPORT_ARGS
153 154

RETVAL=$?
155 156
[ $RETVAL -eq 0 ] && echo Hive Meta Data imported successfully!!!
[ $RETVAL -ne 0 ] && echo Failed to import Hive Meta Data!!!
157 158 159

exit $RETVAL