1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
#!/bin/bash
# # # # # # # # # # # # # # # # # # # # # #
# @author :wangjf
# @revision:2019-04-19 18:05:22
# # # # # # # # # # # # # # # # # # # # # #
source ../dmp_env.sh
job_id=$1
tag=$2
json=$3
if [[ ${tag} == "pkg" ]]; then
partition=2000
execNum=10
table="filter_pkg"
else
partition=600
execNum=8
table="filter_adv"
fi
mysql -h dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -D datatory -u apptag_rw -p'7gyLEVtkER3u8c9' <<EOF
DELETE FROM result WHERE job_id = '${job_id}';
EOF
if [[ $? -ne 0 ]]; then
exit 255
fi
spark-submit --class mobvista.dmp.datasource.datatory.DatatoryJob \
--conf spark.sql.shuffle.partitions=${partition} \
--conf spark.default.parallelism=${partition} \
--conf spark.sql.files.maxPartitionBytes=536870912 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory ${execNum}g --driver-memory 6g --executor-cores 4 --num-executors 150 \
../${JAR} -json ${json// /&@} -tag "${tag}" -coalesce 1
if [[ $? -ne 0 ]]; then
update_time=`date '+%Y-%m-%d %H:%M:%S'`
mysql -h dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -D datatory -u apptag_rw -p'7gyLEVtkER3u8c9' <<EOF
UPDATE ${table} SET update_time = '${update_time}', flag = 2 WHERE job_id = '${job_id}';
UPDATE common_filter SET update_time = '${update_time}', flag = 2 WHERE job_id = '${job_id}';
EOF
exit 255
fi
update_time=`date '+%Y-%m-%d %H:%M:%S'`
mysql -h dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com -P 3306 -D datatory -u apptag_rw -p'7gyLEVtkER3u8c9' <<EOF
UPDATE ${table} SET update_time = '${update_time}', flag = 1 WHERE job_id = '${job_id}';
UPDATE common_filter SET update_time = '${update_time}', flag = 1 WHERE job_id = '${job_id}';
EOF
if [[ $? -ne 0 ]]; then
exit 255
fi