#! /bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @file    :dm_interest_tag_all.sh
# @author  :liushuai
# @revision:2017-02-05 10:58
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime")
date_path=$(date +"%Y/%m/%d" -d "-1 day $ScheduleTime")
year=${LOG_TIME:0:4}
month=${LOG_TIME:4:2}
day=${LOG_TIME:6:2}

TWO_DAYS_AGO=$(date -d "$ScheduleTime 2 days ago" "+%Y%m%d")
old_date_path=$(date -d "$ScheduleTime 2 days ago" +"%Y/%m/%d")

RM_LOG_TIME=$(date +%Y/%m/%d -d "-10 day $ScheduleTime")
RM_DATE=$(date +%d -d "-10 day $ScheduleTime")



JOB_NAME="dm_interest_tag_all"

ADN_INSTALL_INPUT="${DM_INTEREST_PATH}/${date_path}/adn_install"
ADN_REQUEST_INPUT="${DM_INTEREST_PATH}/${date_path}/adn_request_sdk"
ADN_REQUEST_OTHER_INPUT="${DM_INTEREST_PATH}/${date_path}/adn_request_other"
DSP_REQUEST_INPUT="${DM_INTEREST_PATH}/${date_path}/dsp_req"
#AD_SERVER_INPUT="${DM_INTEREST_PATH}/${date_path}/adserver"
OTHER_INPUT="$DM_INTEREST_PATH/$old_date_path/other"
INSTALL_3S_INPUT="${DM_INTEREST_PATH}/${date_path}/3s"
MP_INSTALL_INPUT="${DM_INTEREST_PATH}/${date_path}/mp"
CLEVER_INSTALL_INPUT="${DM_INTEREST_PATH}/${date_path}/clever"
ADN_SDK_INPUT="${DM_INTEREST_PATH}/${date_path}/adn_sdk"
GA_INSTALL_INPUT=$(get_recently_dir "${DM_INTEREST_PATH}" "${TWO_DAYS_AGO}" "ga")
GA_DEVICE_INPUT=$(get_recently_dir "${DM_INTEREST_PATH}" "${TWO_DAYS_AGO}" "ga_device")

check_await $ADN_INSTALL_INPUT/_SUCCESS
check_await $ADN_REQUEST_INPUT/_SUCCESS
check_await $DSP_REQUEST_INPUT/_SUCCESS
#check_await $AD_SERVER_INPUT/_SUCCESS
check_await $INSTALL_3S_INPUT/_SUCCESS
check_await $MP_INSTALL_INPUT/_SUCCESS
check_await $CLEVER_INSTALL_INPUT/_SUCCESS
check_await $ADN_SDK_INPUT/_SUCCESS
check_await ${ADN_REQUEST_OTHER_INPUT}/_SUCCESS

REDUCE_NUM=$(calculate_reduce_num "${ADN_INSTALL_INPUT};${ADN_REQUEST_INPUT};${DSP_REQUEST_INPUT};${OTHER_INPUT};${INSTALL_3S_INPUT};${MP_INSTALL_INPUT};${GA_INSTALL_INPUT};${GA_DEVICE_INPUT};${CLEVER_INSTALL_INPUT};${ADN_SDK_INPUT};${ADN_REQUEST_OTHER_INPUT}")

TMP_OUTPUT_PATH="${TMP_INTEREST_PATH}/${year}/${month}/${day}"
OUTPUT_PATH="${DM_INTEREST_PATH}/${year}/${month}/${day}/all"
RM_OUTPUT_PATH="${DM_INTEREST_PATH}/${RM_LOG_TIME}/all"


INPUT_PATH="${ADN_INSTALL_INPUT},${ADN_REQUEST_INPUT},${DSP_REQUEST_INPUT},${GA_INSTALL_INPUT},${OTHER_INPUT},${INSTALL_3S_INPUT},${MP_INSTALL_INPUT},${CLEVER_INSTALL_INPUT},${ADN_SDK_INPUT},${ADN_REQUEST_OTHER_INPUT}"


hadoop fs -rm -r ${TMP_OUTPUT_PATH}
hadoop jar ../${JAR} mobvista.dmp.datasource.dm.mapreduce.DmInterestAllMR \
    -Dmapreduce.input.fileinputformat.inputdir=${INPUT_PATH} \
    -Dmapreduce.job.reduces=${REDUCE_NUM} \
    -Dtask.date=${LOG_TIME} \
    -Dmapreduce.map.speculative=true \
    -Dmapreduce.reduce.speculative=true \
    -Dmapreduce.job.reduce.slowstart.completedmaps=1.0 \
    -Dmapreduce.job.speculative.slowtaskthreshold=1.0 \
    -Dmapreduce.fileoutputcommitter.algorithm.version=2 \
    "${GA_DEVICE_INPUT}" "${TMP_OUTPUT_PATH}" || exit 1

hadoop fs -rm -r ${OUTPUT_PATH}
REDUCE_NUM=$(calculate_reduce_num "${TMP_OUTPUT_PATH}")
hadoop jar ../${JAR} mobvista.dmp.datasource.dm.mapreduce.DmInterestAllCombineMR \
    -Dmapreduce.job.reduces=${REDUCE_NUM} \
    -Dmapreduce.map.speculative=true \
    -Dmapreduce.reduce.speculative=true \
    -Dmapreduce.job.reduce.slowstart.completedmaps=1.0 \
    -Dmapreduce.job.speculative.slowtaskthreshold=1.0 \
    -Dmapreduce.fileoutputcommitter.algorithm.version=2 \
    "${TMP_OUTPUT_PATH}" ${OUTPUT_PATH} || exit 1

mount_partition "dm_interest_tag" "year='$year', month='$month', day='$day', business='all'" "$OUTPUT_PATH"

unmount_partition "dm_interest_tag" "year='$year', month='$month', day='$RM_DATE', business='all'"  "${RM_OUTPUT_PATH}"

hadoop fs -rmr $TMP_OUTPUT_PATH