1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
#!/bin/sh
# # # # # # # # # # # # # # # # # # # # # #
# @author : wangjf
# @date : 2019-08-28 18:06:59
# # # # # # # # # # # # # # # # # # # # # #
source ../dmp_env.sh
LOG_TIME=$(date -d "$ScheduleTime" +"%Y-%m-%d")
BEFORE_LOG_TIME=$(date -d "$ScheduleTime 1 days ago" +"%Y-%m-%d")
# params:appId appOs fors date
date_path=$(date -d "$ScheduleTime" +"%Y/%m/%d")
DAILY_INPUT="${IQiYi_DAILY_PATH}/${date_path}"
check_await "${DAILY_INPUT}/_SUCCESS"
check_process() {
shell=$1
pid=$(ps aux | grep "${shell}" |grep -v 'grep' |awk '{print $2}' | sort -n | head -n 1)
if [[ ${pid} != '' && ${pid} -gt 0 ]];then
sleep 60 && kill ${pid} && echo "ssh ${pid}.command is complete"
else
echo "ssh ${pid}.command is complete"
fi
}
LOG_TIME=$(date -d "$ScheduleTime" +"%Y-%m-%d")
if [[ ! -d "/home/hadoop/wangjf" ]]; then
mkdir /home/hadoop/wangjf
cd /home/hadoop/wangjf/
hdfs dfs -get s3://mob-emr-test/wangjf/data/pem/dataplatform_cn.pem .
chmod 600 dataplatform_cn.pem
fi
appId=2
appOs=1
fors=200
concurrent=10000
shell=" -cp /root/workspace/DMP-1.0.3-jar-with-dependencies.jar mobvista.dmp.datasource.baichuan.BaiChuanMainV2 ${appId} ${appOs} ${fors} ${concurrent} ${BEFORE_LOG_TIME}"
ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "sh -x /root/workspace/check_process.sh '${shell}'"
if [[ $? -ne 0 ]];then
exit 255
fi
ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "/root/workspace/exec_ali.sh ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME} >/root/workspace/exec_ali_2_1.log"
if [[ $? -ne 0 ]];then
exit 255
fi
# sleep $[fors * 30]
check_process "/root/workspace/exec_ali.sh ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME}"
shell=" -cp /root/workspace/DMP-1.0.3-jar-with-dependencies.jar mobvista.dmp.datasource.baichuan.BaiChuanMainV2 ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME}"
ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "sh -x /root/workspace/check_process.sh '${shell}'"
if [[ $? -ne 0 ]];then
exit 255
fi
appId=2
appOs=2
fors=200
concurrent=10000
shell=" -cp /root/workspace/DMP-1.0.3-jar-with-dependencies.jar mobvista.dmp.datasource.baichuan.BaiChuanMainV2 ${appId} ${appOs} ${fors} ${concurrent} ${BEFORE_LOG_TIME}"
ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "sh -x /root/workspace/check_process.sh '${shell}'"
if [[ $? -ne 0 ]];then
exit 255
fi
ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "/root/workspace/exec_ali.sh ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME} >/root/workspace/exec_ali_2_2.log"
if [[ $? -ne 0 ]];then
exit 255
fi
# sleep $[fors * 30]
check_process "/root/workspace/exec_ali.sh ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME}"
if [[ $? -ne 0 ]];then
exit 255
fi
shell=" -cp /root/workspace/DMP-1.0.3-jar-with-dependencies.jar mobvista.dmp.datasource.baichuan.BaiChuanMainV2 ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME}"
ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "sh -x /root/workspace/check_process.sh '${shell}'"
if [[ $? -ne 0 ]];then
exit 255
fi