dmp_install_list_daily_v2.sh 3.33 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
#!/bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @file    : dmp_install_list_daily.sh
# @author  : jinfeng.wang
# @time    : 2020-04-14 14:53:39
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

LOG_TIME=${ScheduleTime:-$1}
date=$(date +%Y%m%d -d "-1 day $LOG_TIME")
date_path=$(date +"%Y/%m/%d" -d "-1 day ${LOG_TIME}")

SSS_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/3s"
ADN_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_install"
ADN_REQUEST_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_request_sdk"
WangJinfeng committed
18
ADN_REQUEST_UNMATCH_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_request_unmatch"
wang-jinfeng committed
19 20 21 22
ADN_REQUEST_OTHER_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_request_other"
ADN_SDK_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_sdk"
ALLPB_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/allpb"
DSP_REQ_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/dsp_req"
WangJinfeng committed
23
DSP_REQ_UNMATCH_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/dsp_req_unmatch"
wang-jinfeng committed
24 25 26
CLEVER_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/clever"
BYTEDANCE_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/bytedance"
FACEBOOK_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/facebook"
wang-jinfeng committed
27
# JOYPAC_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/joypacios"
wang-jinfeng committed
28 29 30 31 32
MP_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/mp"

check_await ${SSS_INSTALL_LIST_PATH}/_SUCCESS
check_await ${ADN_INSTALL_LIST_PATH}/_SUCCESS
check_await ${ADN_REQUEST_INSTALL_LIST_PATH}/_SUCCESS
WangJinfeng committed
33
check_await ${ADN_REQUEST_UNMATCH_INSTALL_LIST_PATH}/_SUCCESS
wang-jinfeng committed
34 35 36
check_await ${ADN_REQUEST_OTHER_INSTALL_LIST_PATH}/_SUCCESS
check_await ${ADN_SDK_INSTALL_LIST_PATH}/_SUCCESS
check_await ${DSP_REQ_INSTALL_LIST_PATH}/_SUCCESS
WangJinfeng committed
37
check_await ${DSP_REQ_UNMATCH_INSTALL_LIST_PATH}/_SUCCESS
wang-jinfeng committed
38 39 40
check_await ${CLEVER_INSTALL_LIST_PATH}/_SUCCESS
check_await ${BYTEDANCE_INSTALL_LIST_PATH}/_SUCCESS
check_await ${FACEBOOK_INSTALL_LIST_PATH}/_SUCCESS
wang-jinfeng committed
41
# check_await ${JOYPAC_INSTALL_LIST_PATH}/_SUCCESS
wang-jinfeng committed
42 43 44 45 46 47
check_await ${MP_INSTALL_LIST_PATH}/_SUCCESS

BUSINESS="14days"

OUTPUT_PATH="${DMP_INSTALL_LIST}/${date_path}/${BUSINESS}"

WangJinfeng committed
48 49
mount_partition "dmp_install_list" "dt='$date', business='$BUSINESS'" "$OUTPUT_PATH"

wang-jinfeng committed
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
expire_date=$(date +%Y%m%d -d "-4 day $LOG_TIME")
expire_date_path=$(date +"%Y/%m/%d" -d "-4 day ${LOG_TIME}")
EXPIRE_OUTPUT_PATH="${DMP_INSTALL_LIST}/${expire_date_path}/${BUSINESS}"

spark-submit --class mobvista.dmp.common.InstallListDailyV2 \
     --name "InstallListDailyV2.${date}" \
     --conf spark.sql.shuffle.partitions=15000 \
     --conf spark.default.parallelism=2000 \
     --conf spark.kryoserializer.buffer.max=512m \
     --conf spark.kryoserializer.buffer=64m \
     --conf spark.driver.maxResultSize=8g \
     --conf spark.shuffle.io.retryWait=10s \
     --conf spark.shuffle.io.maxRetries=6 \
     --conf spark.sql.files.maxPartitionBytes=536870912 \
     --conf spark.sql.adaptive.enabled=true \
     --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
     --conf spark.kubernetes.memoryOverheadFactor=0.2 \
     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 3 --num-executors 256 \
     ../${JAR} -date ${date} -coalesce 4000 -output ${OUTPUT_PATH}

if [[ $? -ne 0 ]]; then
    exit 255
fi

# 删除过期的分区及删除对应路径
unmount_partition "dmp_install_list" "dt='${expire_date}', business='${BUSINESS}'" "${EXPIRE_OUTPUT_PATH}"