#!/usr/bin/env bash

source ../../dmp_env.sh

dt_slash_taobao_postback_day=$(date -d "$ScheduleTime" +"%Y/%m/%d")
dt_taobao_postback_day=$(date -d "$ScheduleTime" +"%Y%m%d")
dt_today=$(date -d "$ScheduleTime 2 days ago" +"%Y%m%d")
syn_3s_day=$(date -d "$ScheduleTime 2 days ago" +"%m%d")
dt_slash_today=$(date -d "$ScheduleTime 2 days ago" +"%Y/%m/%d")
dt_dash_today=$(date -d "$ScheduleTime 2 days ago" +"%Y-%m-%d")
dt_dash_rec15day=$(date -d "$ScheduleTime 16 days ago" +"%Y-%m-%d")


expire_date=$(date -d "$ScheduleTime 92 days ago" +"%Y%m%d")
expire_date_path=$(date -d "$ScheduleTime 92 days ago" +"%Y/%m/%d")
## 从oss拉取设备打标识  input_file_name()
#ALI_POSTBACK_PATH="${ALI_USER_ACTIVATION_POSTBACK_PATH}/${dt_slash_today}"
#check_await "${ALI_POSTBACK_PATH}/_SUCCESS"
#
#ALI_IOS_POSTBACK_PATH="${ALI_IOS_USER_ACTIVATION_POSTBACK_PATH}/${dt_slash_today}"
#check_await "${ALI_IOS_POSTBACK_PATH}/_SUCCESS"
#
#ALI_OAID_POSTBACK_PATH="${ALI_OAID_USER_ACTIVATION_POSTBACK_PATH}/${dt_slash_today}"
#check_await "${ALI_OAID_POSTBACK_PATH}/_SUCCESS"
##
##
#mount_partition "ali_user_postback_activation_daily" "dt='${dt_today}'" "$ALI_POSTBACK_PATH"
#mount_partition "ali_ios_user_postback_activation_daily" "dt='${dt_today}'" "$ALI_IOS_POSTBACK_PATH"
#mount_partition "ali_oaid_user_postback_activation_daily" "dt='${dt_today}'" "$ALI_OAID_POSTBACK_PATH"

POSTBACK_PATH="${TAOBAO_POSTBACK_DAILY_PATH}/${dt_slash_taobao_postback_day}"

dirs=$(hadoop fs -ls "${POSTBACK_PATH}" | grep -v Found   | grep -v "*" | awk -F'/' '{print $9}' )
for path in $dirs;do
  mount_partition  "ali_taobao_postback_activation_daily"  "dt='${dt_taobao_postback_day}', number='${path}'" "${POSTBACK_PATH}/${path}"
done


OUTPUT_PATH="${ETL_ALI_USERINFO_ACTIVATION_PATH}/${dt_slash_today}"
ALI_IOS_OUTPUT="${ETL_ALI_IOS_USERINFO_ACTIVATION_PATH}/${dt_slash_today}"
ALI_OAID_OUTPUT="${ETL_ALI_OAID_USERINFO_ACTIVATION_PATH}/${dt_slash_today}"
ALI_USER_ACTIVATION_SYS_TO3S_PATH="$ALI_USER_ACTIVATION_SYS_TO3S/${dt_slash_today}"

EXPIRE_PATH="${ETL_ALI_USERINFO_ACTIVATION_PATH}/${expire_date_path}"
EXPIRE_IOS_OUTPUT="${ETL_ALI_IOS_USERINFO_ACTIVATION_PATH}/${expire_date_path}"
EXPIRE_OAID_OUTPUT="${ETL_ALI_OAID_USERINFO_ACTIVATION_PATH}/${expire_date_path}"

hadoop fs -rm -r "${OUTPUT_PATH}"
hadoop fs -rm -r "${ALI_IOS_OUTPUT}"
hadoop fs -rm -r "${ALI_OAID_OUTPUT}"
hadoop fs -rm -r "${ALI_USER_ACTIVATION_SYS_TO3S_PATH}"

mount_partition "etl_ali_user_activation_total" "dt='${dt_today}'" "$OUTPUT_PATH"
mount_partition "etl_ali_ios_user_activation_total" "dt='${dt_today}'" "$ALI_IOS_OUTPUT"
mount_partition "etl_ali_oaid_user_activation_total" "dt='${dt_today}'" "$ALI_OAID_OUTPUT"

unmount_partition "etl_ali_user_activation_total" "dt='${expire_date}'" "$EXPIRE_PATH"
unmount_partition "etl_ali_ios_user_activation_total" "dt='${expire_date}'" "$EXPIRE_IOS_OUTPUT"
unmount_partition "etl_ali_oaid_user_activation_total" "dt='${expire_date}'" "$EXPIRE_OAID_OUTPUT"


spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDaily \
 --conf spark.network.timeout=720s \
 --conf spark.default.parallelism=2000 \
 --conf spark.sql.shuffle.partitions=2000 \
 --conf spark.sql.broadcastTimeout=1200 \
 --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
 --conf spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive=true \
 --files ${HIVE_SITE_PATH} \
 --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 70 \
 ../../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT}  -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 300 \
 -today ${dt_today} -update_date ${dt_dash_today}  -dt_taobao_postback_day ${dt_taobao_postback_day}\
 -dt_dash_rec15day ${dt_dash_rec15day} -syn_to_3s ${ALI_USER_ACTIVATION_SYS_TO3S_PATH} -syn_3s_day ${syn_3s_day}

if [ $? -ne 0 ];then
  exit 255
fi

echo "good job~~"
