#!/usr/bin/env bash source ../dmp_env.sh echo "job begin!!!" dt_today=$(date -d "$ScheduleTime 2 hours ago" +"%Y/%m/%d/%H") dt_dash_today=$(date -d "$ScheduleTime 2 hours ago" +"%Y-%m-%d-%H") yearpath=$(date -d "2 hours ago $ScheduleTime" "+%Y") mothpath=$(date -d "2 hours ago $ScheduleTime" "+%m") daypath=$(date -d "2 hours ago $ScheduleTime" "+%d") hhpath=$(date -d "2 hours ago $ScheduleTime" "+%H") INPUT_PATH01="${ADN_DSP_PATH}/${yearpath}/${mothpath}/${daypath}/cn/${hhpath}" INPUT_PATH02="${ADN_DSP_PATH}/${yearpath}/${mothpath}/${daypath}/virginia/${hhpath}" INPUT_PATH03="${ADN_DSP_PATH}/${yearpath}/${mothpath}/${daypath}/tokyo/${hhpath}" check_await "${INPUT_PATH01}/_SUCCESS" check_await "${INPUT_PATH02}/_SUCCESS" check_await "${INPUT_PATH03}/_SUCCESS" OPPO_OUTPUT="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/tmp_dealid/oppooutput/${dt_today}" INMOBI_OUTPUT="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/tmp_dealid/inmobioutput/${dt_today}" hadoop fs -rm -r $OPPO_OUTPUT hadoop fs -rm -r $INMOBI_OUTPUT mount_partition "tmp_dealid" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',hh='${hhpath}',business='oppocn'" "${OPPO_OUTPUT}" mount_partition "tmp_dealid" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',hh='${hhpath}',business='inmobi'" "${INMOBI_OUTPUT}" spark-submit --class mobvista.dmp.datasource.taobao.EtlDealidDaily \ --conf spark.network.timeout=720s \ --conf spark.default.parallelism=2000 \ --conf spark.sql.shuffle.partitions=2000 \ --conf spark.sql.broadcastTimeout=1200 \ --conf spark.yarn.executor.memoryOverhead=4096 \ --conf spark.sql.autoBroadcastJoinThreshold=31457280 \ --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 6 --num-executors 30 \ ../${JAR} -dt_dash_today ${dt_dash_today} \ -oppooutput ${OPPO_OUTPUT} \ -inmobioutput ${INMOBI_OUTPUT} if [ $? -ne 0 ];then exit 255 fi echo "good job~~"