ali_request.sh 3.25 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
#!/bin/sh

# # # # # # # # # # # # # # # # # # # # # #
# @author : wangjf
# @date : 2019-08-28 18:06:59
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

LOG_TIME=$(date -d "$ScheduleTime" +"%Y-%m-%d")

BEFORE_LOG_TIME=$(date -d "$ScheduleTime 1 days ago" +"%Y-%m-%d")
# params:appId appOs fors date

date_path=$(date -d "$ScheduleTime" +"%Y/%m/%d")
DAILY_INPUT="${IQiYi_DAILY_PATH}/${date_path}"
check_await "${DAILY_INPUT}/_SUCCESS"

check_process() {
    shell=$1
    pid=$(ps aux | grep "${shell}" |grep -v 'grep' |awk '{print $2}' | sort -n | head -n 1)
    if [[ ${pid} != '' && ${pid} -gt 0 ]];then
        sleep 60 && kill ${pid} && echo "ssh ${pid}.command is complete"
    else
        echo "ssh ${pid}.command is complete"
    fi
}

LOG_TIME=$(date -d "$ScheduleTime" +"%Y-%m-%d")

if [[ ! -d "/home/hadoop/wangjf" ]]; then
  mkdir /home/hadoop/wangjf

  cd /home/hadoop/wangjf/

  hdfs dfs -get s3://mob-emr-test/wangjf/data/pem/dataplatform_cn.pem .

  chmod 600 dataplatform_cn.pem
fi

appId=2
appOs=1
fors=200
concurrent=10000

shell=" -cp /root/workspace/DMP-1.0.3-jar-with-dependencies.jar mobvista.dmp.datasource.baichuan.BaiChuanMainV2 ${appId} ${appOs} ${fors} ${concurrent} ${BEFORE_LOG_TIME}"

ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "sh -x /root/workspace/check_process.sh '${shell}'"

if [[ $? -ne 0 ]];then
  exit 255
fi

ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "/root/workspace/exec_ali.sh ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME} >/root/workspace/exec_ali_2_1.log"

if [[ $? -ne 0 ]];then
  exit 255
fi

# sleep $[fors * 30]

check_process "/root/workspace/exec_ali.sh ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME}"

shell=" -cp /root/workspace/DMP-1.0.3-jar-with-dependencies.jar mobvista.dmp.datasource.baichuan.BaiChuanMainV2 ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME}"

ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "sh -x /root/workspace/check_process.sh '${shell}'"

if [[ $? -ne 0 ]];then
    exit 255
fi

appId=2
appOs=2
fors=200
concurrent=10000

shell=" -cp /root/workspace/DMP-1.0.3-jar-with-dependencies.jar mobvista.dmp.datasource.baichuan.BaiChuanMainV2 ${appId} ${appOs} ${fors} ${concurrent} ${BEFORE_LOG_TIME}"

ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "sh -x /root/workspace/check_process.sh '${shell}'"

if [[ $? -ne 0 ]];then
  exit 255
fi

ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "/root/workspace/exec_ali.sh ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME} >/root/workspace/exec_ali_2_2.log"

if [[ $? -ne 0 ]];then
  exit 255
fi

# sleep $[fors * 30]

check_process "/root/workspace/exec_ali.sh ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME}"

if [[ $? -ne 0 ]];then
  exit 255
fi

shell=" -cp /root/workspace/DMP-1.0.3-jar-with-dependencies.jar mobvista.dmp.datasource.baichuan.BaiChuanMainV2 ${appId} ${appOs} ${fors} ${concurrent} ${LOG_TIME}"

ssh -o "StrictHostKeyChecking no" -i /home/hadoop/wangjf/dataplatform_cn.pem -l root 182.92.177.185 "sh -x /root/workspace/check_process.sh '${shell}'"

if [[ $? -ne 0 ]];then
  exit 255
fi