#!/bin/bash # # # # # # # # # # # # # # # # # # # # # # # @author :wangjf # @revision:2019-04-19 18:05:22 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh job_id=$1 tag=$2 json=$3 if [[ ${tag} == "pkg" ]]; then partition=2000 execNum=10 table="filter_pkg" else partition=600 execNum=8 table="filter_adv" fi mysql -h dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -D datatory -u apptag_rw -p'7gyLEVtkER3u8c9' <<EOF DELETE FROM result WHERE job_id = '${job_id}'; EOF if [[ $? -ne 0 ]]; then exit 255 fi spark-submit --class mobvista.dmp.datasource.datatory.DatatoryJob \ --conf spark.sql.shuffle.partitions=${partition} \ --conf spark.default.parallelism=${partition} \ --conf spark.sql.files.maxPartitionBytes=536870912 \ --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \ --master yarn --deploy-mode cluster --executor-memory ${execNum}g --driver-memory 6g --executor-cores 4 --num-executors 150 \ ../${JAR} -json ${json// /&@} -tag "${tag}" -coalesce 1 if [[ $? -ne 0 ]]; then update_time=`date '+%Y-%m-%d %H:%M:%S'` mysql -h dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -D datatory -u apptag_rw -p'7gyLEVtkER3u8c9' <<EOF UPDATE ${table} SET update_time = '${update_time}', flag = 2 WHERE job_id = '${job_id}'; UPDATE common_filter SET update_time = '${update_time}', flag = 2 WHERE job_id = '${job_id}'; EOF exit 255 fi update_time=`date '+%Y-%m-%d %H:%M:%S'` mysql -h dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -D datatory -u apptag_rw -p'7gyLEVtkER3u8c9' <<EOF UPDATE ${table} SET update_time = '${update_time}', flag = 1 WHERE job_id = '${job_id}'; UPDATE common_filter SET update_time = '${update_time}', flag = 1 WHERE job_id = '${job_id}'; EOF if [[ $? -ne 0 ]]; then exit 255 fi