1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
#!/bin/bash
# # # # # # # # # # # # # # # # # # # # # #
# @file : monitor_dmp_pipeline.sh
# @author: houying
# @date : 16-11-28
# # # # # # # # # # # # # # # # # # # # # #
source ../dmp_env.sh
LOG_TIME=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d")
year=${LOG_TIME:0:4}
month=${LOG_TIME:4:2}
# 获取各子目录大小
# $1 路径
get_child_data_size() {
for((i=7;i>=0;i--))
do
local date=$(date -d "$1 $i days ago" "+%Y%m%d")
local year=${date:0:4}
local month=${date:4:2}
local day=${date:6:2}
local path="$(echo "$2" |sed "s/#{year}/${year}/g" |sed "s/#{month}/${month}/g"|sed "s/#{day}/${day}/g")"
local table="$(sed 's/s3:\/\/mob-emr-test\/dataplatform\/DataWareHouse\/data\/dwh\/\([^\/\]\+\).*/\1/g' <<< "$path")"
local partition="$(sed "s/s3:\/\/mob-emr-test\/dataplatform\/DataWareHouse\/data\/dwh\/$table\/$year\/$month\/$day\/\?//g" <<< "$path")"
local line="$(hadoop fs -du -s "$path" )"
local size=0
if [[ ! -z "$line" ]]
then
local size="$(awk -v'FS= ' '{print $1}' <<< "$line")"
fi
if [[ -z "$partition" ]]
then
echo -e "$table\t$date\t$size"
else
echo -e "$table($partition)\t$date\t$size"
fi
done
}
# $1 路径
build_table() {
echo "<table class='gridtable'>" >> mail.txt
echo "<tr><th>table</th><th>date</th><th>size</th><th>rate</th></tr>" >> mail.txt
get_child_data_size "$LOG_TIME" "$1" | awk -v'FS=\t' 'BEGIN{
start=-1;
}{
if (start==-1) {
start=$3;
}
if ($3==0) {
print "<tr class=\"warn\"><td>"$1"</td><td>"$2"</td><td>"$3"</td><td>None%</td></tr>";
start=0;
} else if (start!=0) {
rate=($3-start)/start*100
if (rate<0) {
print "<tr class=\"warn\"><td>"$1"</td><td>"$2"</td><td>"$3"</td><td>"rate"%</td></tr>";
} else {
print "<tr><td>"$1"</td><td>"$2"</td><td>"$3"</td><td>"rate"%</td></tr>";
}
start=$3;
} else {
print "<tr class=\"warn\"><td>"$1"</td><td>"$2"</td><td>"$3"</td><td>None%</td></tr>";
start=$3;
}
}END{
print "</table>"
}'
}
echo '
<style type="text/css">
table.gridtable {
font-family: verdana,arial,sans-serif;
font-size:11px;
color:#333333;
border-width: 1px;
border-color: #666666;
border-collapse: collapse;
}
table.gridtable th {
border-width: 1px;
padding: 8px;
border-style: solid;
border-color: #666666;
background-color: #dedede;
}
table.gridtable td {
border-width: 1px;
padding: 8px;
border-style: solid;
border-color: #666666;
}
table.gridtable tr.warn {
border-width: 1px;
padding: 8px;
border-style: solid;
border-color: #666666;
background-color: #FF0000;
}
</style>
' > mail.txt
cat monitor.cnf|while read -r path
do
build_table "$path"
done >> mail.txt
send_mail "dmp_pipeline_monitor@mobvista.com" "ying.hou@mobvista.com" "dmp数据监控" "$(cat mail.txt)"
rm mail.txt