#!/bin/sh

source ../../dmp_env.sh
source ./../../ga_rawdata_analysis/common/tools.sh



ScheduleTime=${ScheduleTime:-$1}
starttime=$(date -d "4 hours ago $ScheduleTime" "+%Y%m%d%H")
endtime=$(date -d "1 hours ago $ScheduleTime" "+%Y%m%d%H")

datapath=$(date -d "1 hours ago $ScheduleTime" "+%Y/%m/%d")
yearpath=$(date -d "1 hours ago $ScheduleTime" "+%Y")
mothpath=$(date -d "1 hours ago $ScheduleTime" "+%m")
daypath=$(date -d "1 hours ago $ScheduleTime" "+%d")
hhpath=$(date -d "1 hours ago $ScheduleTime" "+%H")



## CNOUTPUT_PATH=s3://mob-emr-test/andy/log_adn_dsp_impression_hour_test/${datapath}/cn/${hhpath}
OUTPUT_PATH=s3://mob-ad/adn/dsp/impression/${datapath}

## OUTPUT_PATH=s3://mob-emr-test/andy/log_adn_dsp_impression_hour_test/${datapath}
CNOUTPUT_PATH=${OUTPUT_PATH}/cn/${hhpath}
TOYKOOUTPUT_PATH=${OUTPUT_PATH}/tokyo/${hhpath}
VIRGINIAOOUTPUT_PATH=${OUTPUT_PATH}/virginia/${hhpath}

hadoop fs -test -e ${OUTPUT_PATH}
if [ $? -ne 0 ];then
hadoop fs -mkdir -p  ${OUTPUT_PATH}
fi


## for area in cn tokyo virginia ;
## do
## dsp_impression_org_hour="s3://mob-ad/adn/dsp/impression_org/impression/${datapath}/${area}/${hhpath}"
## check_await "${dsp_impression_org_hour}/_SUCCESS"
## common_mount_partition "adn_dsp" "log_adn_dsp_impression_org_hour" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='${area}',hh='${hhpath}'" "${dsp_impression_org_hour}"
## done


for area in cn tokyo virginia ;
do
dsp_impression_org_hour="s3://mob-ad/adn/dsp_orc/impression/${datapath}/${area}/${hhpath}"
check_log_adn_dsp_impression_await "${dsp_impression_org_hour}/_SUCCESS"
common_mount_partition "adn_dsp" "log_adn_dsp_impression_org_orc_hour" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='${area}',hh='${hhpath}'" "${dsp_impression_org_hour}"
done

for area in cn tokyo virginia ;
do
dsp_request_orc_hour="s3://mob-ad/adn/dsp_orc/bid_request/${datapath}/${area}/${hhpath}"
check_log_adn_dsp_impression_await "${dsp_request_orc_hour}/_SUCCESS"
common_mount_partition "adn_dsp" "log_adn_dsp_bid_request_orc_hour" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='${area}',hh='${hhpath}'" "${dsp_request_orc_hour}"
done


 hadoop fs -rmr ${CNOUTPUT_PATH}
 hadoop fs -rmr ${TOYKOOUTPUT_PATH}
 hadoop fs -rmr ${VIRGINIAOOUTPUT_PATH}


: '
HIVE_CMD=$(hive_func)

$HIVE_CMD	-v  -hivevar  starttime=${starttime}   -hivevar endtime=${endtime}    -f	log_adn_dsp_impression_hour_combine.sql
'


spark-submit --class  mobvista.dmp.datasource.dsp.DspImpressionHourCombine \
     --name "mobvista.dmp.datasource.dsp.DspImpressionHourCombine" \
	--conf spark.sql.shuffle.partitions=4000 \
	--conf spark.default.parallelism=4000 \
	--conf spark.network.timeout=720s \
    --conf spark.sql.broadcastTimeout=1200 \
	 --conf spark.sql.files.maxPartitionBytes=268435456 \
     --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
	--conf spark.yarn.executor.memoryOverhead=4096 \
	--files ${HIVE_SITE_PATH} \
	--master yarn \
	--deploy-mode cluster \
	--executor-memory 10G \
	--driver-memory 4G \
	--executor-cores 3 \
	--num-executors 80 \
	../../${JAR}  \
	-starttime ${starttime}  -endtime ${endtime} -output ${OUTPUT_PATH}  -cnoutput ${CNOUTPUT_PATH} -vgoutput ${VIRGINIAOOUTPUT_PATH} -tkoutput ${TOYKOOUTPUT_PATH}  -hhpath ${hhpath}

if [ $? -ne 0 ]; then
    exit 255
fi


common_mount_partition "adn_dsp" "log_adn_dsp_impression_hour" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='cn',hh='${hhpath}'" "${CNOUTPUT_PATH}"
common_mount_partition "adn_dsp" "log_adn_dsp_impression_hour" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='tokyo',hh='${hhpath}'" "${TOYKOOUTPUT_PATH}"
common_mount_partition "adn_dsp" "log_adn_dsp_impression_hour" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='virginia',hh='${hhpath}'" "${VIRGINIAOOUTPUT_PATH}"


## common_mount_partition "adn_dsp" "log_adn_dsp_impression_hour_test" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='cn',hh='${hhpath}'" "${CNOUTPUT_PATH}"
## common_mount_partition "adn_dsp" "log_adn_dsp_impression_hour_test" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='tokyo',hh='${hhpath}'" "${TOYKOOUTPUT_PATH}"
## common_mount_partition "adn_dsp" "log_adn_dsp_impression_hour_test" "yr='${yearpath}', mt='${mothpath}',dt='${daypath}',rg='virginia',hh='${hhpath}'" "${VIRGINIAOOUTPUT_PATH}"



hadoop fs -touchz ${CNOUTPUT_PATH}/_SUCCESS
hadoop fs -touchz ${TOYKOOUTPUT_PATH}/_SUCCESS
hadoop fs -touchz ${VIRGINIAOOUTPUT_PATH}/_SUCCESS


