datatory.sh

#!/bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @author  :wangjf
# @revision:2019-04-19 18:05:22
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

job_id=$1
tag=$2
json=$3

if [[ ${tag} == "pkg" ]]; then
    partition=2000
    execNum=10
    table="filter_pkg"
else
    partition=600
    execNum=8
    table="filter_adv"
fi

mysql -h dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -D datatory -u apptag_rw -p'7gyLEVtkER3u8c9' <<EOF
DELETE FROM result WHERE job_id = '${job_id}';
EOF

if [[ $? -ne 0 ]]; then
    exit 255
fi


spark-submit --class mobvista.dmp.datasource.datatory.DatatoryJob \
     --conf spark.sql.shuffle.partitions=${partition} \
     --conf spark.default.parallelism=${partition} \
     --conf spark.sql.files.maxPartitionBytes=536870912 \
     --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
     --files ${HIVE_SITE_PATH} \
     --jars ${JARS} \
     --master yarn --deploy-mode cluster --executor-memory ${execNum}g --driver-memory 6g  --executor-cores 4  --num-executors 150 \
     ../${JAR} -json ${json// /&@} -tag "${tag}" -coalesce 1

if [[ $? -ne 0 ]]; then

    update_time=`date '+%Y-%m-%d %H:%M:%S'`
mysql -h dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -D datatory -u apptag_rw -p'7gyLEVtkER3u8c9' <<EOF
UPDATE ${table} SET update_time = '${update_time}', flag = 2 WHERE job_id = '${job_id}';
UPDATE common_filter SET update_time = '${update_time}', flag = 2 WHERE job_id = '${job_id}';
EOF
    exit 255

fi

update_time=`date '+%Y-%m-%d %H:%M:%S'`

mysql -h dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -D datatory -u apptag_rw -p'7gyLEVtkER3u8c9' <<EOF
UPDATE ${table} SET update_time = '${update_time}', flag = 1 WHERE job_id = '${job_id}';
UPDATE common_filter SET update_time = '${update_time}', flag = 1 WHERE job_id = '${job_id}';

EOF

if [[ $? -ne 0 ]]; then
    exit 255
fi