#!/bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file : dsp_etl_daily.sh # @author: houying # @date : 17-2-17 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh ScheduleTime=${ScheduleTime:-$1} LOG_TIME=$(date -d "$ScheduleTime 1 days ago" "+%Y-%m-%d") dt=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d") date_path=$(date -d "$ScheduleTime 1 days ago" "+%Y/%m/%d") old_path=$(date -d "$ScheduleTime 2 days ago" "+%Y/%m/%d") rm_dt=$(date -d "$ScheduleTime 180 days ago" "+%Y%m%d") rm_dt_path=$(date -d "$ScheduleTime 180 days ago" "+%Y/%m/%d") ## INPUT_DSP_PATH="$ADN_DSP_PATH/$date_path" INPUT_DSP_PATH="${ETL_DSP_ORG_REQ_HOURS}/$date_path" TMP_OUTPUT_PATH="${TMP_DSP_REQUEST_DAILY_PATH}/$date_path" MDS_REQUEST_OUTPUT_PATH="${MDS_DSP_REQ_DAILY}/$date_path" RM_MDS_REQUEST_OUTPUT_PATH="${MDS_DSP_REQ_DAILY}/$rm_dt_path" check_await "${INPUT_DSP_PATH}/23/_SUCCESS" hadoop fs -rm -r $TMP_OUTPUT_PATH hadoop fs -rm -r $MDS_REQUEST_OUTPUT_PATH ## hadoop jar ../${JAR} mobvista.dmp.datasource.dsp.mapreduce.DspDailyEtlMR \ ## -Dmapreduce.job.reduces=400 -DdetailOutPath=${MDS_REQUEST_OUTPUT_PATH} -DoutPath=$TMP_OUTPUT_PATH \ ## "$INPUT_DSP_PATH" "$TMP_OUTPUT_PATH" || exit 1 : ' hadoop jar ../${JAR} mobvista.dmp.datasource.dsp.mapreduce.DspDailyEtlMR3 \ -D mapreduce.job.inputformat.class=com.mobvista.ai.mrdemo.mapreduce.CombineOrcFileInputFormat \ -Dmapreduce.map.memory.mb=3072 \ -D mapreduce.map.java.opts=-Xmx2458m \ -Dmapreduce.reduce.memory.mb=3072 \ -Dmapreduce.reduce.java.opts=-Xmx2458m \ -Dmapreduce.job.reduces=3200 -DdetailOutPath=${MDS_REQUEST_OUTPUT_PATH} -DoutPath=$TMP_OUTPUT_PATH \ -libjars /data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-orcinputwriter-1.0-SNAPSHOT.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-core-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-mapreduce-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/aircompressor-0.10.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/hive-storage-api-2.6.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-shims-1.5.0.jar \ "$INPUT_DSP_PATH" "$TMP_OUTPUT_PATH" || exit 1 ' hadoop jar ../${JAR} mobvista.dmp.datasource.dsp.mapreduce.DspDailyEtlMR3Demo \ -Dmapreduce.job.inputformat.class=com.mobvista.ai.mrdemo.mapreduce.CombineOrcFileInputFormat \ -Dmapreduce.map.memory.mb=3072 \ -Dmapreduce.map.java.opts=-Xmx2458m \ -Dmapreduce.reduce.memory.mb=3072 \ -Dmapreduce.reduce.java.opts=-Xmx2458m \ -Dmapreduce.fileoutputcommitter.algorithm.version=2 \ -Dmapreduce.job.reduces=1600 -DdetailOutPath=${MDS_REQUEST_OUTPUT_PATH} -DoutPath=$TMP_OUTPUT_PATH \ -libjars /data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-orcinputwriter-1.0-SNAPSHOT.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-core-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-mapreduce-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/aircompressor-0.10.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/hive-storage-api-2.6.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-shims-1.5.0.jar \ "$INPUT_DSP_PATH" "$TMP_OUTPUT_PATH" || exit 1 mount_partition "mds_dsp_request_daily" "\`date\`='$dt'" "$MDS_REQUEST_OUTPUT_PATH" unmount_partition "mds_dsp_request_daily" "\`date\`='$rm_dt'" "$RM_MDS_REQUEST_OUTPUT_PATH" hadoop fs -touchz $MDS_REQUEST_OUTPUT_PATH/_SUCCESS