dmp_device_interest.sh 1.7 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
#!/bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @file    : dmp_interest_tag.sh
# @author  : jinfeng.wang
# @time    : 2020-05-22 16:06:07
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

LOG_TIME=${ScheduleTime:-$1}
date=$(date +%Y%m%d -d "-1 day $LOG_TIME")
date_path=$(date +"%Y/%m/%d" -d "-1 day ${LOG_TIME}")

DMP_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/${date_path}/14days"

check_await ${DMP_INSTALL_LIST_PATH}/_SUCCESS

WangJinfeng committed
19 20 21
app_tag_path="${APP_TAG_PATH}/${date_path}"

check_await "${app_tag_path}/_SUCCESS"
wang-jinfeng committed
22 23 24 25 26 27 28 29 30 31

OUTPUT_PATH="${DMP_INTEREST_PATH}/${date_path}"

expire_date=$(date +%Y%m%d -d "-10 day $LOG_TIME")

expire_date_path=$(date +"%Y/%m/%d" -d "-10 day ${LOG_TIME}")

EXPIRE_OUTPUT_PATH="${DMP_INTEREST_PATH}/${expire_date_path}"

spark-submit --class mobvista.dmp.datasource.dm.DmpDeviceInterest \
WangJinfeng committed
32 33 34 35 36 37 38 39 40 41
  --name "DmpDeviceInterest.${date}" \
  --conf spark.sql.shuffle.partitions=10000 \
  --conf spark.default.parallelism=1000 \
  --conf spark.kryoserializer.buffer.max=512m \
  --conf spark.kryoserializer.buffer=64m \
  --conf spark.sql.adaptive.enabled=true \
  --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 80 \
  ../${JAR} \
  -date ${date} -output ${OUTPUT_PATH} -coalesce 4000
wang-jinfeng committed
42 43

if [[ $? -ne 0 ]]; then
WangJinfeng committed
44
  exit 255
wang-jinfeng committed
45 46
fi

WangJinfeng committed
47
part_num=$(hadoop fs -ls ${OUTPUT_PATH} | wc -l)
wang-jinfeng committed
48

WangJinfeng committed
49 50 51
if [[ ${part_num} -le 1000 ]]; then
  echo "This Dir No Data, Pleasce Check Job !!!"
  exit 255
wang-jinfeng committed
52 53 54 55 56
fi

mount_partition "dmp_interest_tag" "dt='${date}'" "$OUTPUT_PATH"

# 删除过期的分区及删除对应路径
WangJinfeng committed
57
unmount_partition "dmp_interest_tag" "dt='${expire_date}'" "${EXPIRE_OUTPUT_PATH}"