etl_dealid_hour.sh 1.89 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
#!/usr/bin/env bash


source ../dmp_env.sh
echo "job  begin!!!"


dt_today=$(date -d "$ScheduleTime 2 hours ago" +"%Y/%m/%d/%H")
dt_dash_today=$(date -d "$ScheduleTime 2 hours ago" +"%Y-%m-%d-%H")

yearpath=$(date -d "2 hours ago $ScheduleTime" "+%Y")
mothpath=$(date -d "2 hours ago $ScheduleTime" "+%m")
daypath=$(date -d "2 hours ago $ScheduleTime" "+%d")
hhpath=$(date -d "2 hours ago $ScheduleTime" "+%H")


INPUT_PATH01="${ADN_DSP_PATH}/${yearpath}/${mothpath}/${daypath}/cn/${hhpath}"
INPUT_PATH02="${ADN_DSP_PATH}/${yearpath}/${mothpath}/${daypath}/virginia/${hhpath}"
INPUT_PATH03="${ADN_DSP_PATH}/${yearpath}/${mothpath}/${daypath}/tokyo/${hhpath}"
check_await "${INPUT_PATH01}/_SUCCESS"
check_await "${INPUT_PATH02}/_SUCCESS"
check_await "${INPUT_PATH03}/_SUCCESS"

OPPO_OUTPUT="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/tmp_dealid/oppooutput/${dt_today}"
INMOBI_OUTPUT="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/tmp_dealid/inmobioutput/${dt_today}"
hadoop fs -rm -r $OPPO_OUTPUT
hadoop fs -rm -r $INMOBI_OUTPUT

mount_partition  "tmp_dealid" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',hh='${hhpath}',business='oppocn'" "${OPPO_OUTPUT}"
mount_partition  "tmp_dealid" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',hh='${hhpath}',business='inmobi'" "${INMOBI_OUTPUT}"

spark-submit --class mobvista.dmp.datasource.taobao.EtlDealidDaily \
 --conf spark.network.timeout=720s \
 --conf spark.default.parallelism=2000 \
 --conf spark.sql.shuffle.partitions=2000 \
 --conf spark.sql.broadcastTimeout=1200 \
 --conf spark.yarn.executor.memoryOverhead=4096 \
 --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
WangJinfeng committed
39
 --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 30 \
wang-jinfeng committed
40 41 42 43 44 45 46 47 48 49 50
 ../${JAR}  -dt_dash_today ${dt_dash_today} \
  -oppooutput ${OPPO_OUTPUT}   \
  -inmobioutput ${INMOBI_OUTPUT}


if [ $? -ne 0 ];then
  exit 255
fi

echo "good job~~"