1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#!/bin/bash
# # # # # # # # # # # # # # # # # # # # # #
# @file : dmp_install_list_daily.sh
# @author : jinfeng.wang
# @time : 2020-04-14 14:53:39
# # # # # # # # # # # # # # # # # # # # # #
source ../dmp_env.sh
LOG_TIME=${ScheduleTime:-$1}
date=$(date +%Y%m%d -d "-1 day $LOG_TIME")
date_path=$(date +"%Y/%m/%d" -d "-1 day ${LOG_TIME}")
SSS_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/3s"
ADN_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_install"
ADN_REQUEST_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_request_sdk"
ADN_REQUEST_OTHER_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_request_other"
ADN_SDK_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_sdk"
ALLPB_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/allpb"
DSP_REQ_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/dsp_req"
CLEVER_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/clever"
BYTEDANCE_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/bytedance"
FACEBOOK_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/facebook"
JOYPAC_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/joypacios"
MP_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/mp"
# ALI_ACQUISITION_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/ali_acquisition"
# ALI_ACTIVATION_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/ali_activation"
# APPSFLYER_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/appsflyer"
# MOPUB_DEALERID_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/mopub_dealerid"
check_await ${SSS_INSTALL_LIST_PATH}/_SUCCESS
check_await ${ADN_INSTALL_LIST_PATH}/_SUCCESS
check_await ${ADN_REQUEST_INSTALL_LIST_PATH}/_SUCCESS
check_await ${ADN_REQUEST_OTHER_INSTALL_LIST_PATH}/_SUCCESS
check_await ${ADN_SDK_INSTALL_LIST_PATH}/_SUCCESS
check_await ${DSP_REQ_INSTALL_LIST_PATH}/_SUCCESS
check_await ${CLEVER_INSTALL_LIST_PATH}/_SUCCESS
check_await ${BYTEDANCE_INSTALL_LIST_PATH}/_SUCCESS
check_await ${FACEBOOK_INSTALL_LIST_PATH}/_SUCCESS
check_await ${JOYPAC_INSTALL_LIST_PATH}/_SUCCESS
check_await ${MP_INSTALL_LIST_PATH}/_SUCCESS
BUSINESS="day"
OUTPUT_PATH="${DMP_INSTALL_LIST}/${date_path}/${BUSINESS}"
expire_date=$(date +%Y%m%d -d "-4 day $LOG_TIME")
expire_date_path=$(date +"%Y/%m/%d" -d "-4 day ${LOG_TIME}")
EXPIRE_OUTPUT_PATH="${DMP_INSTALL_LIST}/${expire_date_path}/${BUSINESS}"
spark-submit --class mobvista.dmp.common.InstallListDaily \
--name "InstallListDaily.${date}" \
--conf spark.sql.shuffle.partitions=10000 \
--conf spark.default.parallelism=1000 \
--conf spark.kryoserializer.buffer.max=512m \
--conf spark.kryoserializer.buffer=64m \
--conf spark.driver.maxResultSize=8g \
--conf spark.sql.files.maxPartitionBytes=536870912 \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
--conf spark.kubernetes.memoryOverheadFactor=0.2 \
--files ${HIVE_SITE_PATH} \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors 160 \
../${JAR} -date ${date} -coalesce 3000 -output ${OUTPUT_PATH}
if [[ $? -ne 0 ]]; then
exit 255
fi
mount_partition "dmp_install_list" "dt='${date}', business='${BUSINESS}'" "$OUTPUT_PATH"
# 删除过期的分区及删除对应路径
unmount_partition "dmp_install_list" "dt='${expire_date}', business='${BUSINESS}'" "${EXPIRE_OUTPUT_PATH}"