spark.kubernetes.executor.limit.cores=5,update executor.cores=5,upgrade spark 3.1.1

azkaban/ali/TO/TO_daily.sh

@@ -27,7 +27,7 @@ spark-submit --class mobvista.dmp.datasource.TO.TODaily \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 20 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 20 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 200  -dt_dash_today ${dt_dash_today}
 

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_etl_postback_daily.sh

@@ -66,7 +66,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDail
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --conf spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive=true \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 70 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 70 \
  ../../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT}  -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 300 \
  -today ${dt_today} -update_date ${dt_dash_today}  -dt_taobao_postback_day ${dt_taobao_postback_day}\
  -dt_dash_rec15day ${dt_dash_rec15day} -syn_to_3s ${ALI_USER_ACTIVATION_SYS_TO3S_PATH} -syn_3s_day ${syn_3s_day}

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_extract_h_18_from_dsp_req.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlH18FromDmInstallListV2 \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 60 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
  ../../${JAR}  \
  -h18_imei ${IMEI_H_18_GUANGDIANTONG_RES_PATH} -h18_imeimd5 ${IMEIMD5_H_18_GUANGDIANTONG_RES_PATH}  \
  -dt_oneday_ago ${dt_oneday_ago}

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_insert_other_data_to_dmp.sh

@@ -43,7 +43,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlOtherDataFromPostBackDail
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 150 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 150 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_daily.sh

@@ -37,7 +37,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 120 ../../${JAR} \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 120 ../../${JAR} \
  -imeioutput "${ALIPAY_IMEIMD5_OUTPUT_PATH}"  \
  -today ${dt_today}  -last_req_day ${last_req_day} -dt_after_one_day ${dt_after_one_day}\
  -input_one_day ${INPUT_ONE_DAY}  -input_two_day ${INPUT_TWO_DAY}  -input_three_day ${INPUT_THREE_DAY} \

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df.sh

@@ -20,7 +20,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
+     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_02.sh

@@ -19,7 +19,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
+     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_03.sh

@@ -19,7 +19,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
+     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_04.sh

@@ -19,7 +19,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
+     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_02.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_03.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_04.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/etl_dealid_hour.sh

@@ -36,7 +36,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlDealidDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 6  --num-executors 30 \
+ --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 30 \
  ../${JAR}  -dt_dash_today ${dt_dash_today} \
   -oppooutput ${OPPO_OUTPUT}   \
   -inmobioutput ${INMOBI_OUTPUT}

azkaban/ali/other_single_jobs/etl_com_tencent_news_daily.sh

@@ -17,7 +17,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlComTencentNewsDaily \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 120 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 5  --num-executors 120 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 500 \
  -dt_today ${dt_today}  -dt_dash_rec7day  ${dt_dash_rec7day}  -dt_dash_rec15day  ${dt_dash_rec15day}

azkaban/ali/uc_lahuo/uc_other_data_to_dmp.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 150 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 150 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -output03 ${OUTPUT_PATH03} -output04 ${OUTPUT_PATH04}  \

azkaban/ali/uc_lahuo_to_guangdiantong/ali_extract_h_32_from_dsp_req.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlH32FromDmInstallListV2 \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 60 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
  ../../${JAR}  \
  -h32_imei ${IMEI_H_32_GUANGDIANTONG_RES_PATH} -h32_imeimd5 ${IMEIMD5_H_32_GUANGDIANTONG_RES_PATH}  \
  -dt_oneday_ago ${dt_oneday_ago}

azkaban/backflow/mapping/mapping.sh

@@ -63,6 +63,7 @@ spark-submit --class mobvista.dmp.datasource.backflow.BackFlow \
   --conf spark.default.parallelism=2000 \
   --conf spark.kryoserializer.buffer.max=512m \
   --conf spark.kryoserializer.buffer=64m \
+  --conf spark.kubernetes.container.image=818539432014.dkr.ecr.us-east-1.amazonaws.com/engineplus/spark:3.0.1-mobvista-v1.2.2 \
   --master yarn --deploy-mode cluster \
   --executor-memory 4g --driver-memory 4g --executor-cores 4 --num-executors 6 \
   ../.././DMP.jar \

azkaban/dm/pseudo_package_to_other_business/adx_packagename_synchronize.sh

@@ -21,7 +21,7 @@ spark-submit --class mobvista.dmp.datasource.dm.AdxPackagenameSynchronize \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 50 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 5  --num-executors 50 \
  ../../${JAR} -dt_today ${dt_today}    -output1 ${OUTPUT_PATH1} \
  -dt_day ${dt_day} \
  -coalesce 300

azkaban/dm/pseudo_package_to_other_business/btop_tiktokrv.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.dm.BtopTiktokrv \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 70 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 5  --num-executors 70 \
  ../../${JAR} -begin_day ${begin_day}   -begin_day02 ${begin_day02}  -end_day ${end_day} -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02}  -output03 ${OUTPUT_PATH03} -output04 ${OUTPUT_PATH04} -coalesce 200
 
 

azkaban/dm/pseudo_package_to_other_business/btop_tiktokrv_gaid.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.dm.BtopTiktokrvGaid \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 70 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 5  --num-executors 70 \
  ../../${JAR} -begin_day ${begin_day}   -begin_day02 ${begin_day02}  -end_day ${end_day} -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02}  -output03 ${OUTPUT_PATH03}  -coalesce 200
 
 

azkaban/dm/pseudo_package_to_other_business/cn_good_channel.sh

@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.datasource.dm.CnGoodChannel \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 100 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 5  --num-executors 100 \
  ../../${JAR}    -output1 ${OUTPUT_PATH1}  -output2 ${OUTPUT_PATH2}  -output3 ${OUTPUT_PATH3}   \
   -output4 ${OUTPUT_PATH4}  -output5 ${OUTPUT_PATH5}  -old_data_path ${OLD_DATA_PATH} \
   -package_name1 ${PACKAGE_NAME01}  -package_name2 ${PACKAGE_NAME02}  -package_name3 ${PACKAGE_NAME03}   \

azkaban/dm/pseudo_package_to_other_business/com_eg_android_AlipayGphone_reyun.sh

@@ -36,7 +36,7 @@ spark-submit --class mobvista.dmp.datasource.dm.ComEgAndroidAlipayGphoneReyun \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 10 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 5  --num-executors 10 \
  ../../${JAR} -dt_today ${dt_today}    -output1 ${OUTPUT_PATH1}  -output2 ${OUTPUT_PATH2} -update ${update}  \
   -coalesce 200
 

azkaban/dm/pseudo_package_to_other_business/phone_wax_nobid_to_dmp.sh

@@ -27,7 +27,7 @@ spark-submit --class mobvista.dmp.datasource.dm.YoukuPhoneWaxNobid \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 150 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 150 \
  ../../${JAR} -begin_day ${begin_day} -end_day ${end_day} -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 
 

azkaban/dm/pseudo_package_to_other_business/reyun_lahuo_list.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.dm.ReyunLahuoList \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
  ../../${JAR} -dt_today ${dt_today}  -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02}  -update_date ${update_date}\
   -coalesce 200
 

azkaban/dm/pseudo_package_to_other_business/rtdmp_tmp_id1142110895.sh

@@ -17,7 +17,7 @@ spark-submit --class mobvista.dmp.datasource.dm.RtdmpTmpId1142110895 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 70 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 5  --num-executors 70 \
  ../../${JAR} -dt_today ${dt_today}    -output1 ${OUTPUT_PATH1} \
   -coalesce 420
 

azkaban/fmp/fmp_insight.sh

@@ -28,7 +28,7 @@ spark-submit --class mobvista.dmp.datasource.fmp.FmpInsightJob \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=134217728 \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 6 --num-executors 64 \
+     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 5 --num-executors 64 \
      ../${JAR} \
      -data ${data// /###} -output ${output_path} -coalesce 1
 

pom.xml

@@ -25,7 +25,7 @@
         <commons.version>1.2</commons.version>
         <poi.version>3.16</poi.version>
         <hive.version>2.3.3</hive.version>
-        <spark.version>3.0.0</spark.version>
+        <spark.version>3.1.1</spark.version>
         <mysql.version>5.0.5</mysql.version>
         <gson.version>2.3.1</gson.version>
         <java.mail.version>1.4.7</java.mail.version>