etl_lazada_ios_data_daily.sh 1.74 KB
Newer Older
fan.jiang committed
1 2 3 4 5 6 7 8 9 10 11 12
#!/usr/bin/env bash


source ../dmp_env.sh

dt_today=$(date -d "$ScheduleTime 2 days ago" +"%Y%m%d")
last_req_day=$(date -d "$ScheduleTime 31 days ago" +"%Y-%m-%d")

dt_slash_today=$(date -d "$ScheduleTime 1 days ago" +"%Y/%m/%d")
dt_slash_one_day=$(date -d "$ScheduleTime 2 days ago" +"%Y/%m/%d")

VN_IDFA_OUTPUT_PATH="${LAZADA_OUTPUT_PATH}/${dt_slash_today}/idfa/VN"
13 14 15 16 17
ID_IDFA_OUTPUT_PATH="${LAZADA_OUTPUT_PATH}/${dt_slash_today}/idfa/ID"
TH_IDFA_OUTPUT_PATH="${LAZADA_OUTPUT_PATH}/${dt_slash_today}/idfa/TH"
PH_IDFA_OUTPUT_PATH="${LAZADA_OUTPUT_PATH}/${dt_slash_today}/idfa/PH"
MY_IDFA_OUTPUT_PATH="${LAZADA_OUTPUT_PATH}/${dt_slash_today}/idfa/MY"
SG_IDFA_OUTPUT_PATH="${LAZADA_OUTPUT_PATH}/${dt_slash_today}/idfa/SG"
fan.jiang committed
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34



# check_await "${ODS_DMP_USER_INFO_DAILY}/${dt_today}/_SUCCESS"
check_await "${ODS_DMP_USER_INFO}/${dt_slash_one_day}/adn_request/_SUCCESS"
check_await "${ODS_DMP_USER_INFO}/${dt_slash_one_day}/dsp_req/_SUCCESS"

hadoop fs -rm -r "${VN_IDFA_OUTPUT_PATH}"


spark-submit --class mobvista.dmp.datasource.taobao.EtlLazadaIosActivitionDaily \
 --conf spark.network.timeout=720s \
 --conf spark.default.parallelism=2000 \
 --conf spark.sql.shuffle.partitions=2000 \
 --conf spark.sql.broadcastTimeout=1200 \
 --conf spark.yarn.executor.memoryOverhead=4096 \
 --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
WangJinfeng committed
35
 --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 90 ../${JAR} \
fan.jiang committed
36
 -today ${dt_today}  -last_req_day ${last_req_day} \
37 38
 -vn_idfaoutput "${VN_IDFA_OUTPUT_PATH}"  -id_idfaoutput "${ID_IDFA_OUTPUT_PATH}"  -th_idfaoutput "${TH_IDFA_OUTPUT_PATH}" \
 -ph_idfaoutput "${PH_IDFA_OUTPUT_PATH}"  -my_idfaoutput "${MY_IDFA_OUTPUT_PATH}"  -sg_idfaoutput "${SG_IDFA_OUTPUT_PATH}"
fan.jiang committed
39 40 41 42

if [ $? -ne 0 ];then
  exit 255
fi