#!/bin/bash source ../dmp_env.sh business="iqiyi" today=${ScheduleTime} date=$(date +"%Y%m%d" -d "-1 day $today") date_path=$(date +%Y/%m/%d -d "-1 day $today") DSP_INSTALL="${DM_INSTALL_LIST}_v2/$date_path/dsp_req" ADN_REQUEST_INSTALL="${DM_INSTALL_LIST}_v2/$date_path/adn_request_sdk" check_await ${DSP_INSTALL}/_SUCCESS check_await ${ADN_REQUEST_INSTALL}/_SUCCESS OUTPUT="s3://mob-emr-test/dataplatform/rtdmp_request/${date_path}/${business}_adx" spark-submit --class mobvista.dmp.datasource.iqiyi.RTDmpIQiYiRequest \ --name "RTDmpIQiYiRequest" \ --conf spark.sql.shuffle.partitions=2000 \ --conf spark.default.parallelism=2000 \ --conf spark.kryoserializer.buffer.max=256m \ --conf spark.sql.adaptive.enabled=true \ --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \ --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 100 \ ../${JAR} -date ${date} -output ${OUTPUT} if [[ $? -ne 0 ]]; then exit 255 fi