#! /bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file :extract_device.sh # @author :liushuai # @revision:2017-01-18 16:12 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime") dt=$(date +'%Y-%m-%d' -d "-1 day $ScheduleTime") date_path=$(date +'%Y/%m/%d' -d "-1 day $ScheduleTime") # INSTALL_3S_INPUT="${DMP_INSTALL_LIST}/${date_path}/3s" # ADN_INSTALL_INPUT="${DMP_INSTALL_LIST}/${date_path}/adn_install" # ADN_REQUEST_INPUT="${DMP_INSTALL_LIST}/${date_path}/adn_request_sdk" # DSP_REQUEST_INPUT="${DMP_INSTALL_LIST}/${date_path}/dsp_req" # GA_INSTALL_DATE=$(get_recently_date "${DMP_INSTALL_LIST}" "${LOG_TIME}" "ga") # OTHER_DATE=$(get_recently_date "${DMP_INSTALL_LIST}" "${LOG_TIME}" "other") INSTALL_LIST_INPUT="${DMP_INSTALL_LIST}/${date_path}/14days" GENDER_OUTPUT="${GENDER_MERGE_INSTALL}/${date_path}" DSP_GENDER_PATH="${GENDER_GET_DSP_PATH}/${date_path}/" GA_GENDER_PATH="${GENDER_GET_GA_PATH}/${date_path}/" FB_GENDER_PATH="${GENDER_GET_FACEBOOK_PATH}/${date_path}/" TP_GENDER_PATH="${GENDER_GET_THIRDPART_PATH}/${date_path}/" # check_await ${INSTALL_3S_INPUT}/_SUCCESS # check_await ${ADN_INSTALL_INPUT}/_SUCCESS # check_await ${ADN_REQUEST_INPUT}/_SUCCESS # check_await ${DSP_REQUEST_INPUT}/_SUCCESS check_await ${INSTALL_LIST_INPUT}/_SUCCESS check_await ${GA_GENDER_PATH}/_SUCCESS check_await ${DSP_GENDER_PATH}/_SUCCESS check_await ${FB_GENDER_PATH}/_SUCCESS check_await ${TP_GENDER_PATH}/_SUCCESS spark-submit --class mobvista.dmp.datasource.age_gender.MergeInstallGender \ --name "MergeInstallGender.${LOG_TIME}" \ --conf spark.sql.shuffle.partitions=5000 \ --conf spark.default.parallelism=2000 \ --conf spark.storage.memoryFraction=0.4 \ --conf spark.shuffle.memoryFraction=0.4 \ --conf spark.sql.files.maxPartitionBytes=536870912 \ --conf spark.sql.adaptive.enabled=true \ --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \ --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 5 --num-executors 60 \ ../${JAR} -date ${LOG_TIME} \ -ga_gender_path ${GA_GENDER_PATH} -dsp_gender_path ${DSP_GENDER_PATH} -fb_gender_path ${FB_GENDER_PATH} -tp_gender_path ${TP_GENDER_PATH} -gender_output ${GENDER_OUTPUT} -parallelism 2000 if [ $? -ne 0 ]; then exit 255 fi