switch spark 3.1

azkaban/joypac/joypac_result_etl.sh

@@ -26,6 +26,9 @@ spark-submit --class mobvista.dmp.datasource.joypac.JoypacResultEtl \
     --conf spark.network.timeout=720s \
     --conf spark.default.parallelism=20 \
     --conf spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive=true \
+    --conf spark.kubernetes.container.image=818539432014.dkr.ecr.us-east-1.amazonaws.com/engineplus/spark:engineplus-2.0.3-free \
+    --conf spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
+    --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
     --master yarn --deploy-mode cluster --name JoypacResutlEtl \
     --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
     --files ${HIVE_SITE_PATH} \

azkaban/joypac/joypc_sdk_daily.sh

@@ -25,6 +25,10 @@ hadoop fs -rm -r "$OUTPUT_PATH/"
 
 spark-submit --class mobvista.dmp.datasource.joypac.JoypcSdkDaily \
  --conf spark.yarn.executor.memoryOverhead=2048  --conf spark.network.timeout=720s \
+ --conf spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive=true \
+ --conf spark.kubernetes.container.image=818539432014.dkr.ecr.us-east-1.amazonaws.com/engineplus/spark:engineplus-2.0.3-free \
+ --conf spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
+ --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
  --master yarn --deploy-mode cluster --name joypc_sdk_daily --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml --conf spark.app.loadTime=${LOG_TIME} --conf spark.app.input_path=${INPUT_PATH} --conf spark.app.output_path=${OUTPUT_PATH} --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 \
  ../${JAR}
 if [ $? -ne 0 ];then