# !/bin/sh
#export SPARK_HOME=/data/liukai/spark
#export PATH=$PATH:$SPARK_HOME/bin
source ../dmp_env.sh
## date=$1
## deal_time="";
## year=""
## month=""
## day=""
## if [ "$date" ]; then
## deal_time=$(date +%Y%m%d -d "$date")
## year=$(date +%Y -d "$date")
## month=$(date +%m -d "$date")
## day=$(date +%d -d "$date")
## #cdndate=`date -d "$format_day" +%Y%m%d`
## echo "$deal_time"
## else
## #echo "dfdf"
## deal_time=$(date -d "$ScheduleTime 1 days ago" +"%Y%m%d")
## year=$(date -d "$ScheduleTime 1 days ago" +"%Y")
## month=$(date -d "$ScheduleTime 1 days ago" +"%m")
## day=$(date -d "$ScheduleTime 1 days ago" +"%d")
## echo "$deal_time"
##
## fi

dete_path=$(date -d "$ScheduleTime 1 days ago" +"%Y/%m/%d")
deal_time=$(date -d "$ScheduleTime 1 days ago" +"%Y%m%d")
echo "$deal_time"
sleep 10s

check_await ${EVENT_DEFINE_3S}/${dete_path}/_SUCCESS

# check dependencie tag file if is exist
## flag=false
## while(($flag==false))
## do
## hadoop fs -test -f ${EVENT_DEFINE_3S}/${year}/${month}/${day}/_SUCCESS
## if [ $? -eq 0 ] ;then
##     echo 'exist'
##       flag=true
##          break
## else
##     echo 'Error! file is not exist'
##      echo "sleep 1m"
##       sleep 10m
## fi
## done
echo "3ss_offer_event_spec.csv  file success exist, and then can start"
# check end
spark-submit --class mobvista.dmp.datasource.event_tag.Event_tag \
  --master yarn --deploy-mode cluster \
  --conf spark.akka.frameSize=100 \
  --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml \
  --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
  --conf spark.driver.memory=4G --queue root.dataplatform \
  --num-executors 40 --executor-cores 4 \
  --executor-memory 8G --conf spark.shuffle.file.buffer=256k \
  --conf spark.reducer.maxSizeInFlight=100m \
  --conf spark.shuffle.io.maxRetries=60 \
  --conf spark.shuffle.io.retryWait=60s \
  --conf spark.network.timeout=1200 \
  --conf spark.shuffle.compress=true \
  --conf spark.io.compression.codec=lz4 \
  --conf spark.driver.maxResultSize=2024M \
  --conf spark.shuffle.spill.compress=true \
  --name event_tag_daily --conf spark.app.name=event_tag_daily \
  --conf spark.app.db_name=dwh --conf spark.app.table=dmp_event_tag_daily \
  --conf spark.app.loadTime=${deal_time} \
  --conf spark.sql.shuffle.partitions=500 \
  --conf spark.yarn.executor.memoryOverhead=3048 \
  --conf spark.locality.wait=0 \
  --conf spark.shuffle.io.numConnectionsPerPeer=10 \
  --conf spark.shuffle.consolidateFiles=true \
  ../DMP.jar
