#!/usr/bin/env bash source ../../dmp_env.sh source ./../../ga_rawdata_analysis/common/tools.sh dt_today=$(date -d "$ScheduleTime 1 days ago" +"%Y%m%d") dt_slash_today=$(date -d "$ScheduleTime 2 days ago" +"%Y/%m/%d") check_await "${TMP_EGGPLANTS_OUTPUT_PATH}/${dt_slash_today}" hadoop fs -test -e "${ODS_OTHER_DEVICE_DAILY}/${dt_slash_today}" if [ $? -ne 0 ];then hadoop fs -mkdir -p "${ODS_OTHER_DEVICE_DAILY}/${dt_slash_today}" fi OUTPUT_PATH01="${RTDMP_NORMAL_PATH}/${dt_slash_today}/01" OUTPUT_PATH02="${RTDMP_NORMAL_PATH}/${dt_slash_today}/02" hadoop fs -rm -r "${OUTPUT_PATH01}" hadoop fs -rm -r "${OUTPUT_PATH02}" spark-submit --class mobvista.dmp.datasource.dm.RtdmpNormal \ --conf spark.yarn.executor.memoryOverhead=2048 \ --conf spark.default.parallelism=3000 \ --conf spark.sql.shuffle.partitions=3000 \ --conf spark.network.timeout=720s \ --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 3 --num-executors 30 \ ../../${JAR} -dt_today ${dt_today} -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02} -rtdmp_normal_count_result "${RTDMP_NORMAL_COUNT_RESULT}/${dt_slash_today}" -coalesce 200 if [[ $? -ne 0 ]]; then exit 255 fi hadoop distcp -m20 "${OUTPUT_PATH01}/*" "${TMP_EGGPLANTS_OUTPUT_PATH}/${dt_slash_today}/" hadoop distcp -m20 "${OUTPUT_PATH02}/*" "${ODS_OTHER_DEVICE_DAILY}/${dt_slash_today}/"