#!/bin/sh source ../dmp_env.sh LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime") yes_bef1_slack=$(date +%Y/%m/%d -d "-1 day $ScheduleTime")
INPUT_PATH="s3://mob-emr-test/dataplatform/DataWareHouse/data/device_mapping/${yes_bef1_slack}"
OUTPUT_PATH="${ODS_DMP_USER_INFO_DAILY}_mapping/${yes_bef1_slack}" spark-submit --class mobvista.dmp.datasource.device.OdsDmpUserInfoDailyMapping \ --name "OdsDmpUserInfoDailyMapping.${date}.wangjf" \ --conf spark.network.timeout=720s \ --conf spark.sql.shuffle.partitions=5000 \ --conf spark.default.parallelism=5000 \ --conf spark.sql.files.maxPartitionBytes=536870912 \ --conf spark.kryoserializer.buffer.max=512m \ --conf spark.kryoserializer.buffer=64m \ --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \ --master yarn \ --deploy-mode cluster \ --executor-memory 10G \ --driver-memory 6G \ --executor-cores 4 \ --num-executors 100 \ ../${JAR} \ -date ${LOG_TIME} -input ${INPUT_PATH} -output ${OUTPUT_PATH} -coalesce 2000 if [[ $? -ne 0 ]]; then exit 255 fi mount_partition "ods_dmp_user_info_daily_mapping" "dt='${LOG_TIME}'" "${OUTPUT_PATH}"