rtdmp_request_iqiyi_adx.sh 1011 Bytes
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#!/bin/bash

source ../dmp_env.sh

business="iqiyi"

today=${ScheduleTime}

date=$(date +"%Y%m%d" -d "-1 day $today")

date_path=$(date +%Y/%m/%d -d "-1 day $today")

DSP_INSTALL="${DM_INSTALL_LIST}_v2/$date_path/dsp_req"
ADN_REQUEST_INSTALL="${DM_INSTALL_LIST}_v2/$date_path/adn_request_sdk"

check_await ${DSP_INSTALL}/_SUCCESS
check_await ${ADN_REQUEST_INSTALL}/_SUCCESS

OUTPUT="s3://mob-emr-test/dataplatform/rtdmp_request/${date_path}/${business}_adx"

spark-submit --class mobvista.dmp.datasource.iqiyi.RTDmpIQiYiRequest \
  --name "RTDmpIQiYiRequest" \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.default.parallelism=2000 \
  --conf spark.kryoserializer.buffer.max=256m \
  --conf spark.sql.adaptive.enabled=true \
  --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 100 \
  ../${JAR} -date ${date} -output ${OUTPUT}


if [[ $? -ne 0 ]]; then
  exit 255
fi