dsp_org_etl_hours.sh 2.4 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
#!/bin/bash

source ../dmp_env.sh

ScheduleTime=${ScheduleTime:-$1}
output_date_path=$(date -d "1 hours ago $ScheduleTime" "+%Y/%m/%d/%H")
input_date=$(date -d "1 hours ago $ScheduleTime" "+%Y%m%d%H")

dt=$(date -d "1 hours ago $ScheduleTime" "+%Y%m%d")

input_date_path=$(date -d "1 hours ago $ScheduleTime" "+%Y/%m/%d")
yearpath=$(date -d "1 hours ago $ScheduleTime" "+%Y")
mothpath=$(date -d "1 hours ago $ScheduleTime" "+%m")
daypath=$(date -d "1 hours ago $ScheduleTime" "+%d")
hhpath=$(date -d "1 hours ago $ScheduleTime" "+%H")

expire_date_path=$(date -d "2161 hours ago $ScheduleTime" "+%Y/%m/%d/%H")
expire_date=$(date -d "2161 hours ago $ScheduleTime" "+%Y%m%d")
expire_hhpath=$(date -d "2161 hours ago $ScheduleTime" "+%H")

for area in cn tokyo virginia; do
  check_await "$ADN_DSP_PATH/$input_date_path/$area/$hhpath/_SUCCESS"
  common_mount_partition "adn_dsp" "log_adn_dsp_request_orc_hour" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='${area}',hh='${hhpath}'" "$ADN_DSP_PATH/$input_date_path/$area/$hhpath"
done

ETL_DSP_REQ_ORG_HOURS_PATH="${ETL_DSP_ORG_REQ_HOURS}/${output_date_path}"

ETL_DSP_REQ_HOURS_PATH="${ETL_DSP_REQ_ETL_HOURS}/${output_date_path}"
MDS_DSP_REQ_HOURS_PATH="${ETL_DSP_REQ_MDS_HOURS}/${output_date_path}"

MDS_DSP_REQ_HOURS_EXPIRE_PATH="${ETL_DSP_REQ_MDS_HOURS}/${expire_date_path}"

hadoop fs -rm -r $ETL_DSP_REQ_HOURS_PATH
hadoop fs -rmr -r $MDS_DSP_REQ_HOURS_PATH

spark-submit --class mobvista.dmp.datasource.dsp.DspOrgLogEtlHoursDemo \
  --conf spark.yarn.executor.memoryOverhead=2048 \
WangJinfeng committed
38 39
  --conf spark.sql.shuffle.partitions=3000 \
  --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 3 --num-executors 120 \
wang-jinfeng committed
40 41
  ../${JAR} -yyyymmddhh $input_date -output $ETL_DSP_REQ_HOURS_PATH -outputmds $MDS_DSP_REQ_HOURS_PATH -coalesce 2000 || exit 1

wang-jinfeng committed
42 43 44 45
if [ $? -ne 0 ];then
  exit 255
fi

wang-jinfeng committed
46 47 48 49 50 51 52 53 54 55 56
## mount_partition "etl_dsp_org_request_daily_hours" "dt='${dt}',hh='${hhpath}'"  "$ETL_DSP_REQ_ORG_HOURS_PATH"

mount_partition "etl_dsp_request_daily_hours" "dt='${dt}',hh='${hhpath}'" "$ETL_DSP_REQ_HOURS_PATH"

mount_partition "mds_dsp_request_daily_hours" "dt='${dt}',hh='${hhpath}'" "$MDS_DSP_REQ_HOURS_PATH"

unmount_partition "mds_dsp_request_daily_hours" "dt='${expire_date}',hh='${expire_hhpath}'" "$MDS_DSP_REQ_HOURS_EXPIRE_PATH"

## hadoop fs -touchz $ETL_DSP_REQ_ORG_HOURS_PATH/_SUCCESS
hadoop fs -touchz $ETL_DSP_REQ_HOURS_PATH/_SUCCESS
hadoop fs -touchz $MDS_DSP_REQ_HOURS_PATH/_SUCCESS