package_black_list.sh 1.35 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
#!/bin/bash

# # # # # # # # # # # # # # # # # # # # # # 
# @file  : collect_package_name.sh
# @author: houying
# @date  : 16-11-14
# @desc  : 收集需要抓取的package列表
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

LOG_DATE=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d")
date_path=$(date -d "$ScheduleTime 1 days ago" +"%Y/%m/%d")
yes_date=$(date -d "$ScheduleTime 2 days ago" +"%Y%m%d")
expire_date=$(date -d "$ScheduleTime 4 days ago" +"%Y%m%d")
expire_path=$(date -d "$ScheduleTime 4 days ago" +"%Y/%m/%d")

OUTPUT_PATH="${PACKAGE_BLACK_LIST}/$date_path"
EXPIRE_PATH="${PACKAGE_BLACK_LIST}/$expire_path"

spark-submit --class mobvista.dmp.main.PackageBlackList \
     --conf spark.sql.shuffle.partitions=10 \
WangJinfeng committed
23
     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
wang-jinfeng committed
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
     ../${JAR} \
     -date "${yes_date}" \
     -iosDailyPath "${TMP_IOS_APP_INFO_PATH}" -adrDailyPath "${TMP_ADR_APP_INFO_PATH}" \
     -toCrawlerPath "${TMP_CRAWLER_INFO_PATH}" -output $OUTPUT_PATH

if [ $? -ne 0 ];then
  exit 255
fi

hive_cmd "
use dev;
alter table dm_package_black_list add if not exists partition(dt='${LOG_DATE}') location '${OUTPUT_PATH}';
alter table dm_package_black_list drop if exists partition(dt='${expire_date}');
"
if [ $? -ne 0 ];then
  exit 255
fi

hadoop fs -rmr $EXPIRE_PATH

echo "[Finish]"