#!/bin/sh # # # # # # # # # # # # # # # # # # # # # # # @author :fengliang # @revision: 2017-12-12 # @desc : 抽取dsp当天位置信息 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh today=${ScheduleTime:-$1} dt=$(date +"%Y%m%d" -d "-1 day $today") date_path=$(date +"%Y/%m/%d" -d "-1 day $today") INPUT_PATH="${MDS_DSP_REQ_DAILY}/$date_path" OUTPUT_PATH="${MDS_DMP_ADDRESS_DAILY}/$date_path/dsp_req" check_await "${INPUT_PATH}/_SUCCESS" hadoop fs -rm -r $OUTPUT_PATH spark-submit --class mobvista.dmp.datasource.address.AddressInfoTotal \ --conf spark.yarn.executor.memoryOverhead=1024 --conf spark.network.timeout=720s \ --conf spark.sql.shuffle.partitions=300 \ --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 100 \ ../${JAR} -input ${INPUT_PATH} -output ${OUTPUT_PATH} -dailyFormat "text" -parallelism 200 -coalesce 20 \ -indices "0,2,3,4,5,6,7" if [ $? -ne 0 ];then exit 255 fi mount_partition "mds_dmp_address_daily" "dt='$dt', business='dsp_req'" "$OUTPUT_PATH" if [ $? -ne 0 ];then exit 255 fi