#!/usr/bin/env bash source ../dmp_env.sh dt_today=$(date -d "$ScheduleTime 2 days ago" +"%Y%m%d") last_req_day=$(date -d "$ScheduleTime 31 days ago" +"%Y-%m-%d") dt_slash_today=$(date -d "$ScheduleTime 1 days ago" +"%Y/%m/%d") dt_slash_one_day=$(date -d "$ScheduleTime 2 days ago" +"%Y/%m/%d") VN_IDFA_OUTPUT_PATH="${LAZADA_OUTPUT_PATH}/${dt_slash_today}/idfa/VN" # check_await "${ODS_DMP_USER_INFO_DAILY}/${dt_today}/_SUCCESS" check_await "${ODS_DMP_USER_INFO}/${dt_slash_one_day}/adn_request/_SUCCESS" check_await "${ODS_DMP_USER_INFO}/${dt_slash_one_day}/dsp_req/_SUCCESS" hadoop fs -rm -r "${VN_IDFA_OUTPUT_PATH}" spark-submit --class mobvista.dmp.datasource.taobao.EtlLazadaIosActivitionDaily \ --conf spark.network.timeout=720s \ --conf spark.default.parallelism=2000 \ --conf spark.sql.shuffle.partitions=2000 \ --conf spark.sql.broadcastTimeout=1200 \ --conf spark.yarn.executor.memoryOverhead=4096 \ --conf spark.sql.autoBroadcastJoinThreshold=31457280 \ --files ${HIVE_SITE_PATH} \ --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 90 ../${JAR} \ -today ${dt_today} -last_req_day ${last_req_day} \ -vn_idfaoutput "${VN_IDFA_OUTPUT_PATH}" if [ $? -ne 0 ];then exit 255 fi