tracking_daily.sh 1.33 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
#! /bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @author : wangjf
# # # # # # # # # # # # # # # # # # # # # #

source ../../dmp_env.sh

dt=$(date +"%Y%m%d" -d "-1 day $ScheduleTime")
date_path=$(date +"%Y/%m/%d" -d "-1 day $ScheduleTime")
log_type=$1

hours="00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23"

if [[ ${log_type} = 'impression' ]]; then
  INPUT_PATH="${ADN_IMPRESSION_PATH}/${date_path}/beijing"
elif [[ ${log_type} = 'click' ]]; then
  INPUT_PATH="${ADN_CLICK_PATH}/${date_path}/beijing"
elif [[ ${log_type} = 'install' ]]; then
  INPUT_PATH="${ADN_INSTALL_PATH}/${date_path}/beijing"
fi

for hour in ${hours}; do
  check_await ${INPUT_PATH}/${hour}/_SUCCESS
done

if [[ ${log_type} = 'impression' ]]; then
28
  EXECUTORS=80
29 30
  OUTPUT_PATH="${OUTPUT_REYUN_IMPRESSION_PATH}/${date_path}"
elif [[ ${log_type} = 'click' ]]; then
31
  EXECUTORS=10
32 33
  OUTPUT_PATH="${OUTPUT_REYUN_CLICK_PATH}/${date_path}"
elif [[ ${log_type} = 'install' ]]; then
34
  EXECUTORS=10
35 36 37 38
  OUTPUT_PATH="${OUTPUT_REYUN_INSTALL_PATH}/${date_path}"
fi

spark-submit --class mobvista.dmp.output.reyun.TrackingLog \
WangJinfeng committed
39
  --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 4 --num-executors ${EXECUTORS} \
40
  ../../${JAR} -date ${dt} -log_type ${log_type} -output ${OUTPUT_PATH} -coalesce 100
41 42 43 44

if [ $? -ne 0 ]; then
  exit 255
fi