#! /bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @file    :dm_interest_tag_all_v2.sh
# @author  :wangjf
# @revision:2018-12-27 11:13:06
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

today=${ScheduleTime:-$1}
date=$(date +"%Y%m%d" -d "-1 day $today")
year=${date:0:4}
month=${date:4:2}
day=${date:6:2}

beforeDate=$(date +"%Y%m%d" -d "-15 day $today")
beforeYear=${beforeDate:0:4}
beforeMonth=${beforeDate:4:2}
beforeDay=${beforeDate:6:2}

ga_date=$(date +"%Y%m%d" -d "-2 day $today")
date_path=$(date +%Y/%m/%d -d "-1 day $today")
before_date_path=$(date +%Y/%m/%d -d "-15 day $today")


ADN_INSTALL_INPUT="${DM_INTEREST_PATH}/${date_path}/adn_install"
ADN_REQUEST_INPUT="${DM_INTEREST_PATH}/${date_path}/adn_request_sdk"
ADN_REQUEST_OTHER_INPUT="${DM_INTEREST_PATH}/${date_path}/adn_request_other"
DSP_REQUEST_INPUT="${DM_INTEREST_PATH}/${date_path}/dsp_req"
OTHER_INPUT="$DM_INTEREST_PATH/$old_date_path/other"
INSTALL_3S_INPUT="${DM_INTEREST_PATH}/${date_path}/3s"
MP_INSTALL_INPUT="${DM_INTEREST_PATH}/${date_path}/mp"
CLEVER_INSTALL_INPUT="${DM_INTEREST_PATH}/${date_path}/clever"
ADN_SDK_INPUT="${DM_INTEREST_PATH}/${date_path}/adn_sdk"
GA_INSTALL_INPUT=$(get_recently_dir "${DM_INTEREST_PATH}" "${TWO_DAYS_AGO}" "ga")
GA_DEVICE_INPUT=$(get_recently_dir "${DM_INTEREST_PATH}" "${TWO_DAYS_AGO}" "ga_device")

check_await ${ADN_INSTALL_INPUT}/_SUCCESS
check_await ${ADN_REQUEST_INPUT}/_SUCCESS
check_await ${DSP_REQUEST_INPUT}/_SUCCESS
check_await ${INSTALL_3S_INPUT}/_SUCCESS
check_await ${MP_INSTALL_INPUT}/_SUCCESS
check_await ${CLEVER_INSTALL_INPUT}/_SUCCESS
check_await ${ADN_SDK_INPUT}/_SUCCESS
check_await ${ADN_REQUEST_OTHER_INPUT}/_SUCCESS

OUTPUT_PATH="${DM_DEV_INTEREST_PATH}_orc/${date_path}/all"
UNMOUNT_OUTPUT_PATH="${DM_DEV_INTEREST_PATH}_orc/${before_date_path}/all"

spark-submit --class mobvista.dmp.datasource.dm.DmInterestTagAllV2 \
     --name "mobvista.dmp.datasource.dm.DmInterestTagAllV2_wangjf_${date}" \
     --conf spark.sql.shuffle.partitions=20000 \
     --conf spark.default.parallelism=20000 \
     --conf spark.executor.memoryOverhead=2048 \
     --conf spark.kryoserializer.buffer.max=512m \
     --conf spark.kryoserializer.buffer=256k \
     --conf spark.sql.files.maxPartitionBytes=536870912 \
     --conf spark.sql.broadcastTimeout=3600 \
     --conf spark.sql.adaptive.enabled=true \
     --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
     --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
     --files ${HIVE_SITE_PATH} \
     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 3  --num-executors 256 \
     ../${JAR} \
     -output ${OUTPUT_PATH} -date ${date} -ga_date ${ga_date} -coalesce 5000

if [[ $? -ne 0 ]]; then
    exit 255
fi

common_mount_partition "dev" "dm_interest_tag_orc" "year='$year', month='$month', day='$day', business='all'" "$OUTPUT_PATH"

common_unmount_partition "dev" "dm_interest_tag_orc" "year='$beforeYear', month='$beforeMonth', day='$beforeDay', business='all'" "$UNMOUNT_OUTPUT_PATH"