#!/bin/sh


# # # # # # # # # # # # # # # # # # # # # #
# @author : 冯亮
# @date : 2018-04-25
# @desc : 为头条数据打标签
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

date_path=$(date +"%Y/%m/%d" -d "-1 day $ScheduleTime")
dt=$(date +"%Y%m%d" -d "-1 day $ScheduleTime")
old_path=$(date +"%Y/%m/%d" -d "-2 day $ScheduleTime")
expire_date=$(date +%Y%m%d -d "-4 day $ScheduleTime")
expire_date_path=$(date +%Y/%m/%d -d "-4 day $ScheduleTime")

business='toutiao'
source='toutiao'
old_path="${DM_DEVICE_TAG_PATH}/${old_path}/${source}/${business}"
output_path="${DM_DEVICE_TAG_PATH}/${date_path}/${source}/${business}"
expire_path="${DM_DEVICE_TAG_PATH}/${expire_date_path}/${source}/${business}"

reduce_num=`calculate_reduce_num "${old_path}"`
if [ $reduce_num -le 5 ]; then
    reduce_num=5
fi

hadoop fs -rm -r ${output_path}

spark-submit --class mobvista.dmp.datasource.toutiao.DmToutiaoDeviceTag \
    --conf spark.sql.shuffle.partitions=${reduce_num} \
    --conf spark.default.parallelism=200 \
    --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4 --num-executors 50 \
    ../${JAR} -date "$dt" -output ${output_path}
if [ $? -ne 0 ];then
  exit 255
fi

mount_partition "dm_device_tag" "dt='${dt}', source='${source}', business='${business}'" "${output_path}"

# 删除过期的分区及删除对应路径
unmount_partition "dm_device_tag" "dt='${expire_date}', source='${source}', business='${business}'" "${expire_path}"