#! /bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file :extract_device.sh # @author :liushuai # @revision:2017-01-18 17:01 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime") date_path=$(date +"%Y/%m/%d" -d "-1 day ${ScheduleTime}") before_date_path=$(date +"%Y/%m/%d" -d "-4 day ${ScheduleTime}") INPUT_PATH="${GENDER_CALC_DEVICE}/${date_path}/" OUTPUT_PATH="${GENDER_VALIDATE}/${date_path}/" hadoop fs -rm -r ${OUTPUT_PATH} hadoop jar ../${JAR} mobvista.dmp.datasource.gender.ValidateMRV2 \ -Dmapreduce.job.inputformat.class=com.mobvista.ai.mrdemo.mapreduce.CombineOrcFileInputFormat \ -libjars /data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-orcinputwriter-1.0-SNAPSHOT.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-core-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-mapreduce-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/aircompressor-0.10.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/hive-storage-api-2.6.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-shims-1.5.0.jar \ -Dmapreduce.fileoutputcommitter.algorithm.version=2 \ "${INPUT_PATH}" "${OUTPUT_PATH}" if [[ $? -ne 0 ]]; then exit 255 fi # 删除临时结果 hadoop fs -rm -r ${GENDER_GET_DSP_PATH}/${before_date_path} hadoop fs -rm -r ${GENDER_GET_GA_PATH}/${before_date_path} hadoop fs -rm -r ${GENDER_MERGE_INSTALL}/${before_date_path} hadoop fs -rm -r ${GENDER_CALC_PACKAGE_DICT}/${before_date_path}