1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
#!/bin/bash
# # # # # # # # # # # # # # # # # # # # # #
# @file :3s_tracking_install_daily.sh
# @author :liushuai
# @revision:2017-03-20 20:09
# # # # # # # # # # # # # # # # # # # # # #
source ../dmp_env.sh
LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime")
dt_yesterday=$(date +%Y%m%d -d "-2 day $ScheduleTime")
year=${LOG_TIME:0:4}
month=${LOG_TIME:4:2}
day=${LOG_TIME:6:2}
dt_today_dash=$(date +%Y-%m-%d -d "-2 day $ScheduleTime")
date_path="${year}/${month}/${day}"
old_path=$(date -d"$LOG_TIME -1 day " +"%Y/%m/%d")
CAMPAIGN_INPUT_PATH="${DIM_CAMPAIGN_PACKAGE_3S}/${year}/${month}-${day}/"
INPUT_PATH_3S="${PATH_3S}/$date_path/"
TMP_OUT_PATH="${TMP_INSTALL_DAILY_3S}/$date_path"
OUT_PATH="${INSTALL_DAILY_3S}/$date_path/"
check_await "${INPUT_PATH_3S}/23/45/*"
hadoop fs -rm -r ${TMP_OUT_PATH}
hadoop jar ../${JAR} mobvista.dmp.datasource.tracking.mapreduce.TrackingInstallDailyMR \
-Dtask.date=${year}-${month}-${day} \
-Dmapreduce.fileoutputcommitter.algorithm.version=2 \
"${CAMPAIGN_INPUT_PATH}" "${INPUT_PATH_3S}" "${TMP_OUT_PATH}"
if [ $? -ne 0 ];then
exit 255
fi
# 为bundleId 匹配 packageName
matchBundlePackage "$date_path" "$old_path" "2" "3" "$TMP_OUT_PATH" "$OUT_PATH" "3s" "../${JAR}"
mount_partition "etl_3s_install_daily" "\`date\`='$LOG_TIME'" "$OUT_PATH" || exit 1
# 国内大媒体入库dmp系统
hive_cmd "set mapred.reduce.tasks = 15;
set hive.exec.compress.output=true;
set mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec;
insert overwrite table dwh.etl_3s_install_daily_tmp
select lower(device_id) device_id,device_type,platform,package_name,update_date,country from dwh.etl_3s_install_daily where \`date\` = '${LOG_TIME}'
union
select lower(device_id) device_id,device_type,platform,package_name, '${dt_today_dash}' update_date,'CN' country from dwh.ods_bigmedia_domestic_daily where dt = '${dt_yesterday}' and package_name !=''
"
hadoop fs -rmr $OUT_PATH*
if [ $? -eq 0 ];then
hadoop fs -cp s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/tmp/etl_3s_install_daily_tmp/* $OUT_PATH
fi
hadoop fs -touchz ${OUT_PATH}_SUCCESS