#! /bin/bash
# # # # # # # # # # # # # # # # # # # # # #
# @file :extract_device.sh
# @author :liushuai
# @revision:2017-01-18 17:01
# # # # # # # # # # # # # # # # # # # # # #
source ../dmp_env.sh
LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime")
date_path=$(date +"%Y/%m/%d" -d "-1 day ${ScheduleTime}")
before_date_path=$(date +"%Y/%m/%d" -d "-4 day ${ScheduleTime}")
INPUT_PATH="${GENDER_CALC_DEVICE}/${date_path}/"
OUTPUT_PATH="${GENDER_VALIDATE}/${date_path}/"
hadoop fs -rm -r ${OUTPUT_PATH}
hadoop jar ../${JAR} mobvista.dmp.datasource.gender.ValidateMRV2 \
-Dmapreduce.job.inputformat.class=com.mobvista.ai.mrdemo.mapreduce.CombineOrcFileInputFormat \
-libjars /data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-orcinputwriter-1.0-SNAPSHOT.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-core-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-mapreduce-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/aircompressor-0.10.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/hive-storage-api-2.6.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-shims-1.5.0.jar \
-Dmapreduce.fileoutputcommitter.algorithm.version=2 \
"${INPUT_PATH}" "${OUTPUT_PATH}"
if [[ $? -ne 0 ]]; then
exit 255
fi
# 删除临时结果
hadoop fs -rm -r ${GENDER_GET_DSP_PATH}/${before_date_path}
hadoop fs -rm -r ${GENDER_GET_GA_PATH}/${before_date_path}
hadoop fs -rm -r ${GENDER_MERGE_INSTALL}/${before_date_path}
hadoop fs -rm -r ${GENDER_CALC_PACKAGE_DICT}/${before_date_path}