#!/bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @file    : dm_interest_tag_daily.sh
# @author  : xiaobin.yan
# @time    : 2017-07-20
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime")
TWO_DAYS_AGO=$(date -d "$ScheduleTime 2 days ago" "+%Y%m%d")
THREE_DAYS_AGO=$(date -d "$ScheduleTime 3 days ago" "+%Y%m%d")
FOUR_DAYS_AGO=$(date -d "$ScheduleTime 4 days ago" "+%Y%m%d")
YEAR=${LOG_TIME:0:4}
MONTH=${LOG_TIME:4:2}
DAY=${LOG_TIME:6:2}

get_path_dir() {
    local path="$1"
    local expect_date="$2"
    local sub_partition="$3"
    local year=${expect_date:0:4}
    local month=${expect_date:4:2}
    local day=${expect_date:6:2}
    local check_path="$path/$year/$month/$day/$sub_partition"
    check_await "${check_path}/_SUCCESS"
    echo "$check_path"
}

INSTALL_DAILY_3S_INPUT=$(build_path_by_date "$INSTALL_DAILY_3S" "$LOG_TIME")
check_await "${INSTALL_DAILY_3S_INPUT}/_SUCCESS"
#AD_SERVER_PACKAGE_NAME_INPUT=$(build_path_by_date "$TMP_ADSERVER_PACKAGE_PATH" "$LOG_TIME")
#check_await "${AD_SERVER_PACKAGE_NAME_INPUT}/_SUCCESS"
ETL_ADN_INSTALL_TMP_DAILY_INPUT=$(build_path_by_date "$ETL_ADN_INSTALL_TMP_DAILY" "$LOG_TIME")
check_await "${ETL_ADN_INSTALL_TMP_DAILY_INPUT}/_SUCCESS"
ETL_ADN_SDK_REQUEST_DAILY_INPUT=$(build_path_by_date "$ETL_ADN_SDK_REQUEST_DAILY" "$LOG_TIME")
check_await "${ETL_ADN_SDK_REQUEST_DAILY_INPUT}/_SUCCESS"
ETL_DSP_REQ_DAILY_INPUT=$(build_path_by_date "$ETL_DSP_REQ_DAILY" "$LOG_TIME")
check_await "${ETL_DSP_REQ_DAILY_INPUT}/_SUCCESS"
GA_INSTALL_PATH_INPUT=$(build_path_by_date "$GA_INSTALL_PATH" "$TWO_DAYS_AGO")


INSTALL_3S_INPUT=$(get_path_dir "${DM_INTEREST_PATH}" "${LOG_TIME}" "3s")
ADN_INSTALL_INPUT=$(get_path_dir "${DM_INTEREST_PATH}" "${LOG_TIME}" "adn_install")
ADN_REQUEST_INPUT=$(get_path_dir "${DM_INTEREST_PATH}" "${LOG_TIME}" "adn_request_sdk")
DSP_REQUEST_INPUT=$(get_path_dir "${DM_INTEREST_PATH}" "${LOG_TIME}" "dsp_req")
#ADSERVER_INPUT=$(get_path_dir "${DM_INTEREST_PATH}" "${LOG_TIME}" "adserver")
GA_INSTALL_INPUT="$(build_path_by_date ${DM_INTEREST_PATH} ${TWO_DAYS_AGO})/ga"


INPUT_PATH_DAILY="${INSTALL_DAILY_3S_INPUT},${ETL_ADN_INSTALL_TMP_DAILY_INPUT},${ETL_ADN_SDK_REQUEST_DAILY_INPUT},${ETL_DSP_REQ_DAILY_INPUT}"
INPUT_PATH_INTEREST="${INSTALL_3S_INPUT},${ADN_INSTALL_INPUT},${ADN_REQUEST_INPUT},${DSP_REQUEST_INPUT}"
OUTPUT_PATH="${DM_INTEREST_DAILY_PATH}/${YEAR}/${MONTH}/${DAY}"

hadoop fs -test -e ${GA_INSTALL_PATH_INPUT}/_SUCCESS
if [ $? -eq 0 ];then
    INPUT_PATH_DAILY="$INPUT_PATH_DAILY,${GA_INSTALL_PATH_INPUT}"
fi

hadoop fs -test -e ${GA_INSTALL_PATH_INPUT}/_SUCCESS
if [ $? -eq 0 ]; then
    INPUT_PATH_INTEREST="$INPUT_PATH_INTEREST,${GA_INSTALL_INPUT}"
fi

hadoop fs -rm -r $OUTPUT_PATH

spark-submit --class mobvista.dmp.DmpInterestTagDailyJob --master yarn \
             --deploy-mode cluster \
             --driver-memory 4g \
             --executor-memory 10g \
             --executor-cores 4 \
             --num-executors 50 \
             --conf spark.yarn.executor.memoryOverhead=2048 \
             ../${JAR} "${INPUT_PATH_DAILY}"  "${INPUT_PATH_INTEREST}"  "${OUTPUT_PATH}"
if [ $? -ne 0 ]; then
    exit 1
fi

mount_partition "dm_interest_tag_daily" "year='$YEAR', month='$MONTH', day='$DAY'" "$OUTPUT_PATH"