mp_request_daily.sh

#! /bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @author : 冯亮
# @date : 2017-08-07
# @desc : 解析MP SDK 天数据
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

date_path=$(date +"%Y/%m/%d" -d "-1 day $ScheduleTime")
dt=$(date +"%Y%m%d" -d "-1 day $ScheduleTime")
old_path=$(date +"%Y/%m/%d" -d "-2 day $ScheduleTime")


INPUT_PATH="${MP_REQUEST_PATH}/$date_path/*"
OUTPUT_PATH="${MP_REQUEST_DAILY_PATH}/$date_path"
ID_MAPPING_PATH="${DMP_ADN_REQUEST_DEVICE_IDS}/$date_path/adn_request"

check_await ${MP_REQUEST_PATH}/$date_path/23
check_await ${ID_MAPPING_PATH}
export HADOOP_CLIENT_OPTS="-Xmx4096m $HADOOP_CLIENT_OPTS"

spark-submit --class mobvista.dmp.datasource.mpsdk.ParseMPSDKDaily \
  --conf spark.sql.shuffle.partitions=200 \
  --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 20 \
 ../${JAR} -input $INPUT_PATH -output $OUTPUT_PATH -mappingPath $ID_MAPPING_PATH -parallelism 100 -coalesce 20

if [ $? -ne 0 ];then
  exit 255
fi

mount_partition "etl_mpsdk_request_daily" "dt='$dt'" "$OUTPUT_PATH"