#!/bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file : nginx_setting.sh # @author: houying # @date : 17-2-10 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh LOG_TIME=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d") year=${LOG_TIME:0:4} month=${LOG_TIME:4:2} day=${LOG_TIME:6:2} NEW_INPUT_PATH="${NEW_NGINX_LOG}/$year/$month/$day/*/*" INPUT_PATH="${NGINX_LOG}/$year/$month/$day/*/*" OUTPUT_PATH="$NGINX_SETTING_DAILY/$year/$month/$day" REDUCE_NUM=$(calculate_reduce_num ${INPUT_PATH}) REDUCE_NUM=100 hadoop fs -rm -r ${OUTPUT_PATH}/* hadoop jar ../${JAR} mobvista.dmp.datasource.setting.NginxSettingMR \ -Dmapreduce.job.reduces=${REDUCE_NUM} \ -Dmapreduce.fileoutputcommitter.algorithm.version=2 \ "$INPUT_PATH" "$NEW_INPUT_PATH" "$OUTPUT_PATH" || exit 1 mount_partition "ods_adn_ngix_setting_v2" "yyyy='$year', mm='$month', dd='$day'" "$OUTPUT_PATH" echo "[Nginx Setting Job End!]"