#! /bin/bash

source ../dmp_env.sh


# 训练逻辑在go 54.88.197.144服务器, 16 9 * * * source ~/.bash_profile && cd /home/mobdev/yechao/gender_online;sh -x gender_predict.sh &>nohup.out


LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime")
dt=$(date +'%Y-%m-%d' -d "-1 day $ScheduleTime")
date_path=$(date +'%Y/%m/%d' -d "-1 day $ScheduleTime")
umount_date=$(date +%Y%m%d -d "-5 day $ScheduleTime")
umount_date_path=$(date +'%Y/%m/%d' -d "-5 day $ScheduleTime")

GENDER_PRIDICT_SCORE_INPUT=${GENDER_PRID_SCORE}/$LOG_TIME
GENDER_ORG_INPUT="${INSTALL_GENDER_LR}/${date_path}"
GENDER_DEVICE_OUTPUT="${GENDER_MERGE_DEVICE}/${date_path}"
UMOUNT_GENDER_DEVICE_OUTPUT="${GENDER_MERGE_DEVICE}/${umount_date_path}"

GDNEER_THRESHOLD_DICT_PATH="$GENDER_PRID_SCORE_THRESHOLD/$LOG_TIME/threshold.txt"

hadoop fs -rmr $GENDER_DEVICE_OUTPUT

check_await "${GENDER_PRIDICT_SCORE_INPUT}/_SUCCESS"
check_await "${GENDER_ORG_INPUT}/_SUCCESS"
check_await "${GDNEER_THRESHOLD_DICT_PATH}"


spark-submit --class mobvista.dmp.datasource.age_gender.MergeDeviceGenderLR \
     --name "MergeDeviceGenderLR" \
     --conf spark.yarn.executor.memoryOverhead=2048 --conf spark.network.timeout=720s \
     --conf spark.sql.shuffle.partitions=400 \
     --conf spark.default.parallelism=400 \
     --conf spark.speculation=false \
     --conf spark.speculation.quantile=0.9 \
     --conf spark.speculation.multiplier=1.5 \
     --master yarn --deploy-mode cluster --executor-memory 8g  --driver-memory 4g  --executor-cores 3  --num-executors 50 \
     ../${JAR} -date ${LOG_TIME}   -gender_predict_input ${GENDER_PRIDICT_SCORE_INPUT}  -gender_threshold_dict_input ${GDNEER_THRESHOLD_DICT_PATH}  -gender_device_output ${GENDER_DEVICE_OUTPUT} -parallelism 400


mount_partition "dmp_device_gender" "dt='${LOG_TIME}'" "${GENDER_DEVICE_OUTPUT}"
unmount_partition "dmp_device_gender" "dt='${umount_date}'" "${UMOUNT_GENDER_DEVICE_OUTPUT}"