adn_install_click.sh 1.34 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
#!/usr/bin/env bash

source ../../dmp_env.sh

LOG_TIME=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d")

year=${LOG_TIME:0:4}
month=${LOG_TIME:4:2}
day=${LOG_TIME:6:2}


INPUT_ADN_INSTALL_PATH="${ETL_ADN_INSTALL_TMP_DAILY}/$year/$month/$day/"
INPUT_CLICK_PATH="${ETL_ADN_MERGE_CLICK_PRE_CLICK}/$year/$month/$day"
OUTPUT_ADN_INSTALL_DAILY="${ETL_ADN_INSTALL_DAILY}/$year/$month/$day"

hadoop fs -rm -r "$OUTPUT_ADN_INSTALL_DAILY/"

spark-submit --class mobvista.dmp.datasource.adn.AdnClickJoinInstallDaily \
  --name "AdnClickJoinInstallDaily.${LOG_TIME}" \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.default.parallelism=2000 \
  --conf spark.kryoserializer.buffer.max=512m \
  --conf spark.kryoserializer.buffer=64m \
  --master yarn --deploy-mode cluster \
  --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 50 \
  ../.././DMP.jar \
  -input_click ${INPUT_CLICK_PATH} -input_install ${INPUT_ADN_INSTALL_PATH} -output ${OUTPUT_ADN_INSTALL_DAILY} -coalesce 200

if [[ $? -ne 0 ]]; then
  exit 255
fi

: '
hadoop jar ../../${JAR} mobvista.dmp.datasource.adn.mapreduce.AdnClickJoinInstallJob \
    -Dmapreduce.fileoutputcommitter.algorithm.version=2 \
    "${INPUT_ADN_INSTALL_PATH}" "$INPUT_CLICK_PATH" "$OUTPUT_ADN_INSTALL_DAILY" || exit 1
'

mount_partition "etl_adn_install_daily" "\`date\`='$LOG_TIME'" "$OUTPUT_ADN_INSTALL_DAILY"