#!/bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file : dmp_install_list_daily.sh # @author : jinfeng.wang # @time : 2020-04-14 14:53:39 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh LOG_TIME=${ScheduleTime:-$1} date=$(date +%Y%m%d -d "-1 day $LOG_TIME") date_path=$(date +"%Y/%m/%d" -d "-1 day ${LOG_TIME}") SSS_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/3s" ADN_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_install" ADN_REQUEST_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_request_sdk" ADN_REQUEST_OTHER_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_request_other" ADN_SDK_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/adn_sdk" ALLPB_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/allpb" DSP_REQ_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/dsp_req" CLEVER_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/clever" BYTEDANCE_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/bytedance" FACEBOOK_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/facebook" JOYPAC_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/joypacios" MP_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/mp" # ALI_ACQUISITION_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/ali_acquisition" # ALI_ACTIVATION_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/ali_activation" # APPSFLYER_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/appsflyer" # MOPUB_DEALERID_INSTALL_LIST_PATH="${DMP_INSTALL_LIST}/$date_path/mopub_dealerid" check_await ${SSS_INSTALL_LIST_PATH}/_SUCCESS check_await ${ADN_INSTALL_LIST_PATH}/_SUCCESS check_await ${ADN_REQUEST_INSTALL_LIST_PATH}/_SUCCESS check_await ${ADN_REQUEST_OTHER_INSTALL_LIST_PATH}/_SUCCESS check_await ${ADN_SDK_INSTALL_LIST_PATH}/_SUCCESS check_await ${DSP_REQ_INSTALL_LIST_PATH}/_SUCCESS check_await ${CLEVER_INSTALL_LIST_PATH}/_SUCCESS check_await ${BYTEDANCE_INSTALL_LIST_PATH}/_SUCCESS check_await ${FACEBOOK_INSTALL_LIST_PATH}/_SUCCESS check_await ${JOYPAC_INSTALL_LIST_PATH}/_SUCCESS check_await ${MP_INSTALL_LIST_PATH}/_SUCCESS BUSINESS="day" OUTPUT_PATH="${DMP_INSTALL_LIST}/${date_path}/${BUSINESS}" mount_partition "dmp_install_list" "dt='$date', business='$BUSINESS'" "$OUTPUT_PATH" expire_date=$(date +%Y%m%d -d "-8 day $LOG_TIME") expire_date_path=$(date +"%Y/%m/%d" -d "-8 day ${LOG_TIME}") EXPIRE_OUTPUT_PATH="${DMP_INSTALL_LIST}/${expire_date_path}/${BUSINESS}" spark-submit --class mobvista.dmp.common.InstallListDaily \ --name "InstallListDaily.${date}" \ --conf spark.sql.shuffle.partitions=10000 \ --conf spark.default.parallelism=1000 \ --conf spark.kryoserializer.buffer.max=512m \ --conf spark.kryoserializer.buffer=64m \ --conf spark.driver.maxResultSize=8g \ --conf spark.sql.files.maxPartitionBytes=536870912 \ --conf spark.sql.adaptive.enabled=true \ --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \ --conf spark.kubernetes.memoryOverheadFactor=0.2 \ --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors 160 \ ../${JAR} -date ${date} -coalesce 3000 -output ${OUTPUT_PATH} if [[ $? -ne 0 ]]; then exit 255 fi # 删除过期的分区及删除对应路径 unmount_partition "dmp_install_list" "dt='${expire_date}', business='${BUSINESS}'" "${EXPIRE_OUTPUT_PATH}"