rtdmp.sh

#!/bin/bash

source ../dmp_env.sh

today=${ScheduleTime}

date_time=$(date +"%Y-%m-%d.%H" -d "-1 hour $today")

date_path=$(date +%Y/%m/%d/%H -d "-1 hour $today")

part_num=$(hadoop fs -ls s3://mob-emr-test/dataplatform/rtdmp_pre/${date_path}/ | wc -l)
if [[ ${part_num} -le 50 ]]; then
  echo "This Dir No Data !!!"
  partition=10
  coalesce=10
  executor=2
  memory=4
  core=2
  flag=0
else
  partition=2000
  coalesce=200
  executor=8
  memory=10
  core=4
  flag=1
fi
INPUT="s3://mob-emr-test/dataplatform/rtdmp_pre/${date_path}"

OUTPUT="s3://mob-emr-test/dataplatform/rtdmp_deal/${date_path}"

before_date_path=$(date +%Y/%m/%d/%H -d "-2 hour $today")

BEFORE_OUTPUT="s3://mob-emr-test/dataplatform/rtdmp/${before_date_path}"

check_await "${BEFORE_OUTPUT}/_SUCCESS"

spark-submit --class mobvista.dmp.datasource.rtdmp.RTDmpMain \
  --name "RTDmpMain.${date_time}" \
  --conf spark.sql.shuffle.partitions=${partition} \
  --conf spark.default.parallelism=${partition} \
  --conf spark.kryoserializer.buffer.max=512m \
  --conf spark.kryoserializer.buffer=64m \
  --master yarn --deploy-mode cluster \
  --executor-memory ${memory}g --driver-memory 6g --executor-cores ${core} --num-executors ${executor} \
  .././DMP.jar \
  -flag ${flag} -time ${date_time} -input ${INPUT} -output ${OUTPUT} -coalesce ${coalesce}

if [[ $? -ne 0 ]]; then
  exit 255
fi