#! /bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file :extract_device.sh # @author :liushuai # @revision:2017-01-18 16:12 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime") date_path=$(date +'%Y/%m/%d' -d "-1 day $ScheduleTime") old_path=$(date +'%Y/%m/%d' -d "-2 day $ScheduleTime") INPUT_PATH_DSP="${DMP_ADN_DSP_PROFILE_TOTAL}/$date_path" OUTPUT_PATH="${GENDER_GET_DSP_PATH}/${date_path}/" OLD_PATH_DSP="${DMP_ADN_DSP_PROFILE_TOTAL}/$old_path" check_await $INPUT_PATH_DSP/_SUCCESS hadoop fs -rm -r ${OUTPUT_PATH} REDUCE_NUM=$(calculate_reduce_num ${OLD_PATH_DSP}) hadoop jar ../${JAR} mobvista.dmp.datasource.gender.GetDspGenderMR \ -Dmapreduce.fileoutputcommitter.algorithm.version=2 \ -Dmapreduce.job.reduces=0 \ -Dmapreduce.map.memory.mb=1433 \ ${INPUT_PATH_DSP} ${OUTPUT_PATH} || exit 1 if [ $? -ne 0 ]; then exit 255 fi