#! /bin/bash # # # # # # # # # # # # # # # # # # # # # # # @author : 冯亮 # @date : 2017-08-07 # @desc : 将天数据合并到安装列表 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh date=$(date +"%Y-%m-%d" -d "-1 day $ScheduleTime") date_path=$(date +"%Y/%m/%d" -d "-1 day $ScheduleTime") dt=$(date +"%Y%m%d" -d "-1 day $ScheduleTime") old_path=$(date +"%Y/%m/%d" -d "-2 day $ScheduleTime") INPUT_PATH="${MP_REQUEST_DAILY_PATH}/$date_path" OLD_INPUT_PATH="${DM_INSTALL_LIST}/$old_path/mp" OUTPUT_PATH="${DM_INSTALL_LIST}/$date_path/mp" check_await "${OLD_INPUT_PATH}/_SUCCESS" hadoop fs -rm -r ${OUTPUT_PATH} spark-submit --class mobvista.dmp.datasource.mpsdk.MPSDKInstallList \ --conf spark.sql.shuffle.partitions=800 \ --conf spark.yarn.executor.memoryOverhead=4096 \ --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 150 \ ../${JAR} -input $INPUT_PATH -output ${OUTPUT_PATH} -date ${date} -oldInput ${OLD_INPUT_PATH} -parallelism 800 -coalesce 800 if [ $? -ne 0 ];then exit 255 fi mount_partition "dm_install_list" "year='${dt:0:4}', month='${dt:4:2}', day='${dt:6:2}', business='mp'" "$OUTPUT_PATH" if [ $? -ne 0 ];then exit fi