#!/bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file : monitor_dmp_pipeline.sh # @author: houying # @date : 16-11-28 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh LOG_TIME=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d") year=${LOG_TIME:0:4} month=${LOG_TIME:4:2} # 获取各子目录大小 # $1 路径 get_child_data_size() { for((i=7;i>=0;i--)) do local date=$(date -d "$1 $i days ago" "+%Y%m%d") local year=${date:0:4} local month=${date:4:2} local day=${date:6:2} local path="$(echo "$2" |sed "s/#{year}/${year}/g" |sed "s/#{month}/${month}/g"|sed "s/#{day}/${day}/g")" local table="$(sed 's/s3:\/\/mob-emr-test\/dataplatform\/DataWareHouse\/data\/dwh\/\([^\/\]\+\).*/\1/g' <<< "$path")" local partition="$(sed "s/s3:\/\/mob-emr-test\/dataplatform\/DataWareHouse\/data\/dwh\/$table\/$year\/$month\/$day\/\?//g" <<< "$path")" local line="$(hadoop fs -du -s "$path" )" local size=0 if [[ ! -z "$line" ]] then local size="$(awk -v'FS= ' '{print $1}' <<< "$line")" fi if [[ -z "$partition" ]] then echo -e "$table\t$date\t$size" else echo -e "$table($partition)\t$date\t$size" fi done } # $1 路径 build_table() { echo "<table class='gridtable'>" >> mail.txt echo "<tr><th>table</th><th>date</th><th>size</th><th>rate</th></tr>" >> mail.txt get_child_data_size "$LOG_TIME" "$1" | awk -v'FS=\t' 'BEGIN{ start=-1; }{ if (start==-1) { start=$3; } if ($3==0) { print "<tr class=\"warn\"><td>"$1"</td><td>"$2"</td><td>"$3"</td><td>None%</td></tr>"; start=0; } else if (start!=0) { rate=($3-start)/start*100 if (rate<0) { print "<tr class=\"warn\"><td>"$1"</td><td>"$2"</td><td>"$3"</td><td>"rate"%</td></tr>"; } else { print "<tr><td>"$1"</td><td>"$2"</td><td>"$3"</td><td>"rate"%</td></tr>"; } start=$3; } else { print "<tr class=\"warn\"><td>"$1"</td><td>"$2"</td><td>"$3"</td><td>None%</td></tr>"; start=$3; } }END{ print "</table>" }' } echo ' <style type="text/css"> table.gridtable { font-family: verdana,arial,sans-serif; font-size:11px; color:#333333; border-width: 1px; border-color: #666666; border-collapse: collapse; } table.gridtable th { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #dedede; } table.gridtable td { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; } table.gridtable tr.warn { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #FF0000; } </style> ' > mail.txt cat monitor.cnf|while read -r path do build_table "$path" done >> mail.txt send_mail "dmp_pipeline_monitor@mobvista.com" "ying.hou@mobvista.com" "dmp数据监控" "$(cat mail.txt)" rm mail.txt