dsp_dealer_daily.sh 921 Bytes
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
#!/bin/bash

source ../dmp_env.sh

ScheduleTime=${ScheduleTime:-$1}
dt=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d")
date_path=$(date -d "$ScheduleTime 1 days ago" "+%Y/%m/%d")

ETL_DSP_DEALERID_RETARGET_DAILY_PATH="${ETL_DSP_DEALERID_RETARGET_DAILY}/${date_path}"

hadoop fs -rm -r $ETL_DSP_DEALERID_RETARGET_DAILY_PATH

spark-submit --class  mobvista.dmp.datasource.dsp.DspDealeridRetarget \
 --conf spark.yarn.executor.memoryOverhead=2048 \
 --conf spark.sql.shuffle.partitions=2000 \
 --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 100 \
 ../${JAR} -yyyymmdd $dt  -output $ETL_DSP_DEALERID_RETARGET_DAILY_PATH  -appFile $ETL_DSP_DEALERID_PKG_MAPPING -coalesce 2000     || exit 1

mount_partition "etl_dsp_dealer_retarget_daily" "dt='${dt}'"  "$ETL_DSP_DEALERID_RETARGET_DAILY_PATH"

hadoop fs -touchz $ETL_DSP_DEALERID_RETARGET_DAILY_PATH/_SUCCESS