#!/bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file : ga_device_total.sh # @author: houying # @date : 16-10-10 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh LOG_TIME=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d") year=${LOG_TIME:0:4} month=${LOG_TIME:4:2} day=${LOG_TIME:6:2} YESTERDAY=$(date -d "$ScheduleTime 2 days ago" "+%Y%m%d") old_year=${YESTERDAY:0:4} old_month=${YESTERDAY:4:2} old_day=${YESTERDAY:6:2} INPUT_PATH_DAILY="$GA_DAILY_PATH/${year}/${month}/${day}" INPUT_PATH_TOTAL=$(get_recently_dir "$GA_TOTAL_PATH" "$YESTERDAY" "") OUTPUT_PATH="$GA_TOTAL_PATH/${year}/${month}/${day}" echo " INPUT_PATH_DAILY=$INPUT_PATH_DAILY INPUT_PATH_TOTAL=$INPUT_PATH_TOTAL OUTPUT_PATH=$OUTPUT_PATH " hadoop fs -rm ${OUTPUT_PATH}/* REDUCE_NUM=$(calculate_reduce_num "$INPUT_PATH_TOTAL") hadoop jar ../${JAR} mobvista.dmp.datasource.ga.mapreduce.GaDeviceTotalMR \ -Dtask.date=${LOG_TIME} \ -Dmapreduce.job.reduces=${REDUCE_NUM} \ -Dmapreduce.fileoutputcommitter.algorithm.version=2 \ "${INPUT_PATH_DAILY}" "${INPUT_PATH_TOTAL}" \ "${OUTPUT_PATH}" || exit 1 if [ $? -ne 0 ]; then exit 255 fi mount_partition "ods_ga_device_total" "year='${year}',month='${month}',day='${day}'" "$OUTPUT_PATH" echo "[GA device total End!]"