#!/bin/bash source ../dmp_env.sh ScheduleTime=${ScheduleTime:-$1} output_date_path=$(date -d "1 hours ago $ScheduleTime" "+%Y/%m/%d/%H") input_date=$(date -d "1 hours ago $ScheduleTime" "+%Y%m%d%H") dt=$(date -d "1 hours ago $ScheduleTime" "+%Y%m%d") input_date_path=$(date -d "1 hours ago $ScheduleTime" "+%Y/%m/%d") yearpath=$(date -d "1 hours ago $ScheduleTime" "+%Y") mothpath=$(date -d "1 hours ago $ScheduleTime" "+%m") daypath=$(date -d "1 hours ago $ScheduleTime" "+%d") hhpath=$(date -d "1 hours ago $ScheduleTime" "+%H") expire_date_path=$(date -d "2161 hours ago $ScheduleTime" "+%Y/%m/%d/%H") expire_date=$(date -d "2161 hours ago $ScheduleTime" "+%Y%m%d") expire_hhpath=$(date -d "2161 hours ago $ScheduleTime" "+%H") for area in cn tokyo virginia; do check_await "$ADN_DSP_PATH/$input_date_path/$area/$hhpath/_SUCCESS" common_mount_partition "adn_dsp" "log_adn_dsp_request_orc_hour" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='${area}',hh='${hhpath}'" "$ADN_DSP_PATH/$input_date_path/$area/$hhpath" done ETL_DSP_REQ_ORG_HOURS_PATH="${ETL_DSP_ORG_REQ_HOURS}/${output_date_path}" ETL_DSP_REQ_HOURS_PATH="${ETL_DSP_REQ_ETL_HOURS}/${output_date_path}" MDS_DSP_REQ_HOURS_PATH="${ETL_DSP_REQ_MDS_HOURS}/${output_date_path}" MDS_DSP_REQ_HOURS_EXPIRE_PATH="${ETL_DSP_REQ_MDS_HOURS}/${expire_date_path}" hadoop fs -rm -r $ETL_DSP_REQ_HOURS_PATH hadoop fs -rmr -r $MDS_DSP_REQ_HOURS_PATH spark-submit --class mobvista.dmp.datasource.dsp.DspOrgLogEtlHoursDemo \ --conf spark.yarn.executor.memoryOverhead=2048 \ --conf spark.sql.shuffle.partitions=2000 \ --master yarn --deploy-mode cluster --executor-memory 12g --driver-memory 6g --executor-cores 5 --num-executors 60 \ ../${JAR} -yyyymmddhh $input_date -output $ETL_DSP_REQ_HOURS_PATH -outputmds $MDS_DSP_REQ_HOURS_PATH -coalesce 2000 || exit 1 if [ $? -ne 0 ];then exit 255 fi ## mount_partition "etl_dsp_org_request_daily_hours" "dt='${dt}',hh='${hhpath}'" "$ETL_DSP_REQ_ORG_HOURS_PATH" mount_partition "etl_dsp_request_daily_hours" "dt='${dt}',hh='${hhpath}'" "$ETL_DSP_REQ_HOURS_PATH" mount_partition "mds_dsp_request_daily_hours" "dt='${dt}',hh='${hhpath}'" "$MDS_DSP_REQ_HOURS_PATH" unmount_partition "mds_dsp_request_daily_hours" "dt='${expire_date}',hh='${expire_hhpath}'" "$MDS_DSP_REQ_HOURS_EXPIRE_PATH" ## hadoop fs -touchz $ETL_DSP_REQ_ORG_HOURS_PATH/_SUCCESS hadoop fs -touchz $ETL_DSP_REQ_HOURS_PATH/_SUCCESS hadoop fs -touchz $MDS_DSP_REQ_HOURS_PATH/_SUCCESS