#!/bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file :package_mapping.sh # @desc :package_name ID化 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh today=${ScheduleTime:-$1} date=$(date +"%Y%m%d" -d "-1 day $today") date_path=$(date +%Y/%m/%d -d "-1 day $today") spark-submit --class mobvista.dmp.datasource.retargeting.GetPackage \ --name "GetPackage.${date}" \ --conf spark.sql.shuffle.partitions=2000 \ --conf spark.default.parallelism=2000 \ --conf spark.sql.broadcastTimeout=1200 \ --conf spark.yarn.executor.memoryOverhead=2048 \ --conf spark.sql.files.maxPartitionBytes=268435456 \ --conf spark.sql.adaptive.enabled=true \ --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \ --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \ --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 128 \ ../${JAR} \ -date ${date} if [[ $? -ne 0 ]]; then exit 255 fi