1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
#!/bin/bash
# # # # # # # # # # # # # # # # # # # # # #
# @file : collect_package_name.sh
# @author: houying
# @date : 16-11-14
# @desc : 收集需要抓取的package列表
# # # # # # # # # # # # # # # # # # # # # #
source ../dmp_env.sh
LOG_DATE=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d")
date_path=$(date -d "$ScheduleTime 1 days ago" +"%Y/%m/%d")
yes_date=$(date -d "$ScheduleTime 2 days ago" +"%Y%m%d")
expire_date=$(date -d "$ScheduleTime 4 days ago" +"%Y%m%d")
expire_path=$(date -d "$ScheduleTime 4 days ago" +"%Y/%m/%d")
OUTPUT_PATH="${PACKAGE_BLACK_LIST}/$date_path"
EXPIRE_PATH="${PACKAGE_BLACK_LIST}/$expire_path"
spark-submit --class mobvista.dmp.main.PackageBlackList \
--conf spark.sql.shuffle.partitions=10 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
../${JAR} \
-date "${yes_date}" \
-iosDailyPath "${TMP_IOS_APP_INFO_PATH}" -adrDailyPath "${TMP_ADR_APP_INFO_PATH}" \
-toCrawlerPath "${TMP_CRAWLER_INFO_PATH}" -output $OUTPUT_PATH
if [ $? -ne 0 ];then
exit 255
fi
hive_cmd "
use dev;
alter table dm_package_black_list add if not exists partition(dt='${LOG_DATE}') location '${OUTPUT_PATH}';
alter table dm_package_black_list drop if exists partition(dt='${expire_date}');
"
if [ $? -ne 0 ];then
exit 255
fi
hadoop fs -rmr $EXPIRE_PATH
echo "[Finish]"