update dmp spark-submit

azkaban/3s/3s_install_device_tag.sh

@@ -32,7 +32,7 @@ hadoop fs -rmr $store_output_path
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=20 \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 4 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 4 \
      ../${JAR} \
      -date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 20
 if [ $? -ne 0 ]; then

azkaban/3s/3s_install_device_tag_daily.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.tracking_3s.TrackingTagDaily \
      --conf spark.sql.shuffle.partitions=5 \
      --conf spark.default.parallelism=5 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 3 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 3 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2
 

azkaban/3s/3s_install_device_tag_v2.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=8 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4
 

azkaban/3s/3s_tracking_install_total_orc.sh

@@ -40,7 +40,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=20 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 10
 
 if [[ $? -ne 0 ]];then

azkaban/3s_postback/3s_postback_daily.sh

@@ -41,7 +41,7 @@ spark-submit --class mobvista.dmp.datasource.postback_3s.PostBackDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
- --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 3  --num-executors 100 \
+ --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 3  --num-executors 100 \
  ../${JAR} -output ${OUTPUT_PATH}  -coalesce 100 \
  -today ${dt_today}  -last_sunday ${dt_yesterday}
 

azkaban/3s_postback/3s_postback_device_tag.sh

@@ -28,7 +28,7 @@ hadoop fs -rmr ${store_output_path}
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
   --conf spark.yarn.executor.memoryOverhead=2048  \
      --conf spark.sql.shuffle.partitions=1000 \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g  --executor-cores 3  --num-executors 40 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 6g  --executor-cores 3  --num-executors 40 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1000
 

azkaban/3s_postback/3s_postback_device_tag_daily.sh

@@ -28,7 +28,7 @@ spark-submit --class mobvista.dmp.datasource.postback_3s.PostBackTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 5
 

azkaban/3s_postback/3s_postback_device_tag_v2.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=100 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 40
 

azkaban/3s_postback/3s_postback_install_list.sh

@@ -24,7 +24,7 @@ spark-submit --class mobvista.dmp.datasource.postback_3s.PostBackInstallList \
  --conf spark.yarn.executor.memoryOverhead=2048  \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=1000 \
- --master yarn --deploy-mode cluster --name PostBackInstallList --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 20 \
+ --deploy-mode cluster --name PostBackInstallList --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 20 \
  ../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 1000 -coalesce 400
 
 

azkaban/3s_postback/3s_postback_install_total_orc.sh

@@ -39,9 +39,8 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.sql.shuffle.partitions=1000 \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
-     --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 25 \
-     ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 200
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 40 \
+     ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 400
 
 if [[ $? -ne 0 ]];then
   exit 255

azkaban/ad_server_campaign/merge_campaign_list.sh

@@ -48,7 +48,7 @@ spark-submit --class mobvista.dmp.datasource.packagelist.MergeCampaignList \
   --conf spark.sql.broadcastTimeout=1200 \
   --conf spark.yarn.executor.memoryOverhead=2048 \
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-  --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 4 \
+  --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 4 \
   ../${JAR} -input_dmp_data_adn ${INPUT_DMP_DATA_ADN} -input_campaign_adn ${INPUT_CAMPAIGN_ADN} -coalesce 20 \
   -output ${CAMPAIGN_TAG_OUTPUT} -today ${yester_year}-${yester_month}-${yester_day} || exit 1
 

azkaban/adn/address/mds_dmp_address_daily_adn.sh

@@ -22,7 +22,7 @@ hadoop fs -rm -r $OUTPUT_PATH
 spark-submit --class mobvista.dmp.datasource.address.AddressInfoTotal \
      --conf spark.yarn.executor.memoryOverhead=1024 --conf spark.network.timeout=720s \
      --conf spark.sql.shuffle.partitions=200 \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 40 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 40 \
      ../../${JAR}  -input ${INPUT_PATH} -output ${OUTPUT_PATH} -dailyFormat "rcfile"  -parallelism 200 -coalesce 20 \
      -indices "0,2,3,5,6"
 

azkaban/adn/adn_device_ids_daily.sh

@@ -25,15 +25,15 @@ OUTPUT_PATH="$DMP_ADN_REQUEST_DEVICE_IDS/$year/$month/$day/adn_request"
 ## REDUCE_NUM=$(calculate_reduce_num "$INPUT_ADN_DSP_PATH")
 
 hadoop fs -rm -r ${OUTPUT_PATH}/*
-export HADOOP_CLIENT_OPTS="-Xmx2096m $HADOOP_CLIENT_OPTS"
+export HADOOP_CLIENT_OPTS="-Xmx4096m $HADOOP_CLIENT_OPTS"
 
 hadoop jar ../${JAR} mobvista.dmp.datasource.adn.mapreduce.AdnDeviceIdsMR \
-    -Dmapreduce.job.reduces=1000 \
-    -Dmapred.child.java.opts=-Xmx4096m \
-    -Dmapreduce.map.memory.mb=8192 \
-    -Dmapreduce.map.java.opts=-Xmx4096m \
-    -Dmapreduce.reduce.memory.mb=8192 \
-    -Dmapreduce.reduce.java.opts=-Xmx4096m \
+    -Dmapreduce.job.reduces=2000 \
+    -Dmapred.child.java.opts=-Xmx8192m \
+    -Dmapreduce.map.memory.mb=10240 \
+    -Dmapreduce.map.java.opts=-Xmx8192m \
+    -Dmapreduce.reduce.memory.mb=10240 \
+    -Dmapreduce.reduce.java.opts=-Xmx8192m \
     -Dtask.date="$year-$month-$day" \
     -Dmapreduce.fileoutputcommitter.algorithm.version=2 \
     "$INPUT_ADN_DSP_PATH" "$OUTPUT_PATH" || exit 1

azkaban/adn/device/adn_org_etl_hours_frankfurt.sh

@@ -24,7 +24,7 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
 spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
   --conf spark.yarn.executor.memoryOverhead=3072 \
   --conf spark.sql.shuffle.partitions=2000 \
-  --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
+  --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
   ../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 200 -region frankfurt || exit 1
 
 if [[ $? -ne 0 ]]; then

azkaban/adn/device/adn_org_etl_hours_seoul.sh

@@ -24,7 +24,7 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
 spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
   --conf spark.yarn.executor.memoryOverhead=3072 \
   --conf spark.sql.shuffle.partitions=2000 \
-  --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
+  --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
   ../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 200 -region seoul || exit 1
 
 if [[ $? -ne 0 ]]; then

azkaban/adn/device/adn_org_etl_hours_singapore.sh

@@ -23,7 +23,7 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
 spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
   --conf spark.yarn.executor.memoryOverhead=3072 \
   --conf spark.sql.shuffle.partitions=2000 \
-  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 50 \
+  --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 50 \
   ../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 400 -region singapore || exit 1
 
 if [[ $? -ne 0 ]]; then

azkaban/adn/device/adn_org_etl_hours_virginia.sh

@@ -24,7 +24,7 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
 spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
   --conf spark.yarn.executor.memoryOverhead=3072 \
   --conf spark.sql.shuffle.partitions=2000 \
-  --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
+  --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
   ../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 200 -region virginia || exit 1
 
 if [[ $? -ne 0 ]]; then

azkaban/adn/package/adn_click_daily.sh

@@ -21,7 +21,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnClickDaily \
   --conf spark.default.parallelism=1000 \
   --conf spark.kryoserializer.buffer.max=512m \
   --conf spark.kryoserializer.buffer=64m \
-  --master yarn --deploy-mode cluster \
+  --deploy-mode cluster \
   --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 50 \
   ../.././DMP.jar \
   -datetime ${LOG_TIME} -output ${OUTPUT_ADN_INSTALL_DAILY} -coalesce 200

azkaban/adn/package/adn_install_click.sh

@@ -21,7 +21,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnClickJoinInstallDaily \
   --conf spark.default.parallelism=2000 \
   --conf spark.kryoserializer.buffer.max=512m \
   --conf spark.kryoserializer.buffer=64m \
-  --master yarn --deploy-mode cluster \
+  --deploy-mode cluster \
   --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 50 \
   ../.././DMP.jar \
   -input_click ${INPUT_CLICK_PATH} -input_install ${INPUT_ADN_INSTALL_PATH} -output ${OUTPUT_ADN_INSTALL_DAILY} -coalesce 200

azkaban/adn/package/adn_install_daily.sh

@@ -28,7 +28,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnInstallDaily \
   --conf spark.default.parallelism=100 \
   --conf spark.kryoserializer.buffer.max=512m \
   --conf spark.kryoserializer.buffer=64m \
-  --master yarn --deploy-mode cluster \
+  --deploy-mode cluster \
   --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
   ../.././DMP.jar \
   -datetime ${LOG_TIME} -output ${TMP_OUTPUT_ADN_INSTALL_DAILY} -coalesce 20

azkaban/adn/package/adn_install_device_tag.sh

@@ -32,7 +32,7 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=20 \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
      ../../${JAR} \
      -date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 20
 

azkaban/adn/package/adn_install_device_tag_daily.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnInstallTagDaily \
      --conf spark.sql.shuffle.partitions=50 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1
 

azkaban/adn/package/adn_install_device_tag_v2.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=8 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4
 

azkaban/adn/package/adn_install_total_orc.sh

@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.sql.shuffle.partitions=1000 \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 4  --num-executors 20 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 4  --num-executors 20 \
      ../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 100
 
 if [[ $? -ne 0 ]];then

azkaban/adn/package/adn_install_total_v1.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
+  --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
   ../../${JAR} \
   -date ${date} -coalesce 200 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
 

azkaban/adn/package/adn_install_total_v2.sh

@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
 '

azkaban/adn/package/adn_pre_click_daily.sh

@@ -21,7 +21,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnPreClickDaily \
   --conf spark.default.parallelism=1000 \
   --conf spark.kryoserializer.buffer.max=512m \
   --conf spark.kryoserializer.buffer=64m \
-  --master yarn --deploy-mode cluster \
+  --deploy-mode cluster \
   --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 50 \
   ../.././DMP.jar \
   -datetime ${LOG_TIME} -output ${OUTPUT_ADN_INSTALL_DAILY} -coalesce 200

azkaban/adn/package/adn_request_daily_v2.sh

@@ -32,7 +32,7 @@ spark-submit --class mobvista.dmp.datasource.adn_request_sdk.AdnRequestSdkEtlDai
     --conf spark.storage.memoryFraction=0.4 \
     --conf spark.sql.files.maxPartitionBytes=268435456 \
     --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
-    --master yarn --deploy-mode cluster --name "AdnRequestSdkEtlDaily.${LOG_TIME}" --executor-memory 10g --driver-memory 6g  --executor-cores 5 --num-executors 100 \
+    --deploy-mode cluster --name "AdnRequestSdkEtlDaily.${LOG_TIME}" --executor-memory 10g --driver-memory 6g  --executor-cores 5 --num-executors 100 \
     ../../${JAR} -appIdMapping ${INPUT_MAPPING_PATH} -manualAppIdMapping ${DIM_MANUAL_MAPPING} -output ${TMP_OUTPUT_PATH} -date ${LOG_TIME} -coalesce 2000
 
 if [[ $? -ne 0 ]]; then

azkaban/adn/package/adn_request_device_tag.sh

@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
     --conf spark.yarn.executor.memoryOverhead=2048 \
     --conf spark.sql.shuffle.partitions=4000 \
     --conf spark.default.parallelism=4000 \
-    --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 100 \
+    --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 100 \
     ../../${JAR} \
     -date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 400
 if [ $? -ne 0 ]; then

azkaban/adn/package/adn_request_device_tag_daily.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnRequestSdkTagDaily \
      --conf spark.sql.shuffle.partitions=20 \
      --conf spark.default.parallelism=5 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 2 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 2 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 20
 

azkaban/adn/package/adn_request_device_tag_v2.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=200 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100
 

azkaban/adn/package/adn_request_install_total_orc.sh

@@ -37,7 +37,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 3 --num-executors 200 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 3 --num-executors 200 \
      ../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 2000
 
 if [[ $? -ne 0 ]];then

azkaban/adn/package/adn_request_other_daily.sh

@@ -32,7 +32,7 @@ spark-submit --class mobvista.dmp.datasource.adn_request_other.EtlAdnRequestOthe
     --conf spark.storage.memoryFraction=0.4 \
     --conf spark.driver.maxResultSize=5g \
     --conf spark.executor.extraJavaOptions=-XX:+UseG1GC \
-    --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 40 \
+    --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 40 \
     ../../${JAR} -input "${INPUT_PATH}/*/*" -output $OUTPUT_PATH
 if [ $? -ne 0 ]; then
     exit 255

azkaban/adn/package/adn_request_other_device_tag.sh

@@ -33,7 +33,7 @@ hadoop fs -rmr ${store_output_path}
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=2000 \
       --conf spark.yarn.executor.memoryOverhead=3072 \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 80 \
+     --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 80 \
      ../../${JAR} \
      -date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000
 

azkaban/adn/package/adn_request_other_device_tag_daily.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn_request_other.AdnRequestOtherTa
      --conf spark.sql.shuffle.partitions=20 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10
 

azkaban/adn/package/adn_request_other_device_tag_v2.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=200 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100
 

azkaban/adn/package/adn_request_other_install.sh

@@ -27,7 +27,7 @@ hadoop fs -rm -r "$OUTPUT_PATH"
 spark-submit --class mobvista.dmp.datasource.adn_request_other.AdnRequestOtherInstall \
      --conf spark.yarn.executor.memoryOverhead=2048 \
      --conf spark.sql.shuffle.partitions=2000 \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 200 \
+     --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 200 \
      ../../${JAR} \
      -input "${INPUT_PATH}" -output $OUTPUT_PATH -date $date -oldInput $OLD_INPUT_PATH -parallelism 2000 -coalesce 2000
 if [ $? -ne 0 ]; then

azkaban/adn/package/adn_request_other_install_total_orc.sh

@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
+     --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 400
 
 if [[ $? -ne 0 ]];then

azkaban/adn/package/adn_request_other_install_v1.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
+  --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
   ../../${JAR} \
   -date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
 

azkaban/adn/package/adn_request_other_install_v2.sh

@@ -38,7 +38,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 30 \
+     --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 30 \
      ../../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
 '

azkaban/adn/package/adn_request_pkg_total_v1.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
+  --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
   ../../${JAR} \
   -date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
 

azkaban/adn/package/adn_request_pkg_total_v2.sh

@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 2  --num-executors 40 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 2  --num-executors 40 \
      ../../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
 '

azkaban/adn/package/adn_request_unmatch_install_total_orc.sh

@@ -32,7 +32,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=4000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC " \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 100 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 100 \
      ../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 2000
 
 if [[ $? -ne 0 ]];then

azkaban/adn_adx/adn_tencent_adx_device_tag.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.adn_adx.AdnAdxDeviceTag \
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
   --conf spark.sql.hive.verifyPartitionPath=true \
   --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
-  --master yarn --deploy-mode cluster --name AdnAdxDeviceTag --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 32 \
+  --deploy-mode cluster --name AdnAdxDeviceTag --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 32 \
   ../${JAR} -outputadxdevtag ${OUTPUT_ADN_ADX_DEVICE_TAG_PATH} \
   -coalesce 80 \
   -today ${dt_today} -yesterday ${dt_yesterday}

azkaban/adn_adx/adn_tencent_adx_package.sh

@@ -36,7 +36,7 @@ spark-submit --class mobvista.dmp.datasource.adn_adx.AdnTecentAdxDataMidWay \
  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
  --conf spark.sql.hive.verifyPartitionPath=true \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar,s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/json-serde-1.3.7-jar-with-dependencies.jar \
- --master yarn --deploy-mode cluster --name  AdnTecentAdxDataMidWay --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 100 \
+ --deploy-mode cluster --name  AdnTecentAdxDataMidWay --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 100 \
  ../${JAR}  -outputadxtmp ${OUTPUT_ODS_ADX_TMP_PATH} -dimadxpkg ${OUTPUT_DIM_ADN_ADX_PKG_PATH} \
  -coalesce 60 \
  -today ${dt_today}

azkaban/adn_sdk/adn_sdk_daily.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkDaily \
  --conf spark.default.parallelism=2000 \
  --conf spark.driver.maxResultSize=8g \
  --conf spark.app.coalesce=60000 \
- --master yarn --deploy-mode cluster --name adn_sdk_daily --executor-memory 10g --driver-memory 6g  --executor-cores 3 --num-executors 300 \
+ --deploy-mode cluster --name adn_sdk_daily --executor-memory 10g --driver-memory 6g  --executor-cores 3 --num-executors 300 \
  ../${JAR}
 
 if [[ $? -ne 0 ]];then

azkaban/adn_sdk/adn_sdk_device_tag.sh

@@ -34,7 +34,7 @@ hadoop fs -rmr ${store_output_path}
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=2000 \
      --conf spark.yarn.executor.memoryOverhead=4096 \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g  --executor-cores 3 --num-executors 200 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 6g  --executor-cores 3 --num-executors 200 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000
 

azkaban/adn_sdk/adn_sdk_device_tag_daily.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10
 

azkaban/adn_sdk/adn_sdk_device_tag_v2.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=1000 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100
 

azkaban/adn_sdk/adn_sdk_etl_hour.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkEtlHour \
  --conf spark.storage.memoryFraction=0.4 \
  --conf spark.driver.maxResultSize=8g \
  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
- --master yarn --deploy-mode cluster --name adn_sdk_etl_hour.${date}.${hour} --executor-memory 4g --driver-memory 4g  --executor-cores 4 --num-executors 20 \
+ --deploy-mode cluster --name adn_sdk_etl_hour.${date}.${hour} --executor-memory 4g --driver-memory 4g  --executor-cores 4 --num-executors 20 \
  ../${JAR} -coalesce 4000 -date ${date} -hour ${hour} -input ${INPUT_PATH} -output ${OUTPUT_PATH}
 
 if [[ $? -ne 0 ]];then

azkaban/adn_sdk/adn_sdk_install.sh

@@ -38,7 +38,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkInstallList \
  --conf spark.shuffle.memoryFraction=0.4 \
  --conf spark.sql.adaptive.enabled=true \
  --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
- --master yarn --deploy-mode cluster --name adn_sdk_install --executor-memory 15g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
+ --deploy-mode cluster --name adn_sdk_install --executor-memory 15g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
  ../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 2000 -coalesce 2000
 
 if [[ $? -ne 0 ]];then

azkaban/adn_sdk/adn_sdk_install_total_orc.sh

@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=4000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 100 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 100 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 2000
 
 if [[ $? -ne 0 ]];then

azkaban/adn_sdk/adn_sdk_install_v1.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 80 \
+  --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 80 \
   ../${JAR} \
   -date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
 

azkaban/adn_sdk/adn_sdk_install_v2.sh

@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
      ../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
 '

azkaban/adn_sdk/adn_sdk_v2_device_tag.sh

@@ -34,7 +34,7 @@ hadoop fs -rmr ${store_output_path}
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=2000 \
      --conf spark.yarn.executor.memoryOverhead=4096 \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 4  --num-executors 150 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 4  --num-executors 150 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000
 

azkaban/adn_sdk/adn_sdk_v2_device_tag_v2.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=100 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100
 

azkaban/adn_sdk/adn_sdk_v2_install.sh

@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkInstallList \
  --conf spark.sql.adaptive.enabled=true \
  --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
  --conf spark.app.version=1 \
- --master yarn --deploy-mode cluster --name "adn_sdk_v2_install.${LOG_TIME}" --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 50 \
+ --deploy-mode cluster --name "adn_sdk_v2_install.${LOG_TIME}" --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 50 \
  ../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 200 -coalesce 200
 
 if [[ $? -ne 0 ]];then

azkaban/adn_sdk/adn_sdk_v2_install_total_orc.sh

@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=500 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
+     --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 200
 
 if [[ $? -ne 0 ]];then

azkaban/adn_sdk/adn_sdk_v2_install_v1.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
+  --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
   ../${JAR} \
   -date ${date} -coalesce 200 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
 

azkaban/adn_sdk/adn_sdk_v2_install_v2.sh

@@ -43,7 +43,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
      ../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
 '

azkaban/age/calc_device_age.sh

@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.CalcDeviceAge \
      --conf spark.storage.memoryFraction=0.4 \
      --conf spark.shuffle.memoryFraction=0.4 \
      --conf spark.sql.files.maxPartitionBytes=134217728 \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 100 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 100 \
      ../${JAR} -date ${LOG_TIME} -merge_input ${INPUT_PATH} -dict_input ${FILEDICT} -output ${OUTPUT_PATH} -parallelism 500
 
 if [[ $? -ne 0 ]];then

azkaban/age/calc_package_dict_age.sh

@@ -27,7 +27,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.CalcPackageAge \
      --conf spark.storage.memoryFraction=0.4 \
      --conf spark.shuffle.memoryFraction=0.4 \
      --conf spark.sql.files.maxPartitionBytes=134217728 \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 100 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 100 \
      ../${JAR} -inputPath ${INPUT_PATH} -outputPath ${OUTPUT_PATH} -parallelism 1 -low ${LOW} -high ${HIGH} -unbelievable ${UNBELIEVABLE}
 
 if [[ $? -ne 0 ]];then

azkaban/age/get_dsp_all.sh

@@ -36,7 +36,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.GetAgeGender \
      --conf spark.sql.shuffle.partitions=2000 \
      --conf spark.default.parallelism=2000 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
      ../${JAR} -ageOutput ${AGE_OUTPUT_PATH} -genderOutput ${GENDER_OUTPUT_PATH} -date ${LOG_TIME} -business ${business}
 
 if [[ $? -ne 0 ]];then

azkaban/age/get_ga_all.sh

@@ -58,7 +58,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.GetAgeGender \
      --conf spark.speculation.quantile=0.9 \
      --conf spark.speculation.multiplier=1.5 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
+     --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -ageOutput ${AGE_OUTPUT_PATH} -genderOutput ${GENDER_OUTPUT_PATH} -date ${GA_TOTAL_DATE} -business ${business}
 
 if [[ $? -ne 0 ]];then

azkaban/age/merge_install_age.sh

@@ -52,7 +52,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.MergeInstallAge \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
      ../${JAR} -date ${LOG_TIME} \
      -ga_age_path ${GA_AGE_PATH}  -dsp_age_path ${DSP_AGE_PATH} -age_output ${OUTPUT_PATH} -parallelism 2000
 

azkaban/ali/TO/TO_daily.sh

@@ -27,7 +27,7 @@ spark-submit --class mobvista.dmp.datasource.TO.TODaily \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 20 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 20 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 200  -dt_dash_today ${dt_dash_today}
 

azkaban/ali/TO/TO_install_list.sh

@@ -23,7 +23,7 @@ spark-submit --class mobvista.dmp.datasource.TO.TOInstallList \
  --conf spark.yarn.executor.memoryOverhead=2048  \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=100 \
- --master yarn --deploy-mode cluster --name PostBackInstallList --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 30 \
+ --deploy-mode cluster --name PostBackInstallList --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 30 \
  ../../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 160 -coalesce 160
 
 

azkaban/ali/ali_ck.sh

@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.baichuan.BaiChuanJob \
      --conf spark.default.parallelism=100 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores 2  --num-executors 20 \
+     --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores 2  --num-executors 20 \
      ../${JAR} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/ali_daily.sh

@@ -24,7 +24,7 @@ spark-submit --class mobvista.dmp.datasource.baichuan.AliDaily \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 5 \
      ../${JAR} -date ${LOG_TIME} -partNum 10 -output ${OUTPUT_PATH} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/ali_device_tag.sh

@@ -36,7 +36,7 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=100 \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores 2  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 40
 

azkaban/ali/ali_device_tag_daily.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.baichuan.AliTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 5
 

azkaban/ali/ali_device_tag_v2.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=8 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 8
 

azkaban/ali/ali_install_list.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.baichuan.AliInstallList \
  --conf spark.yarn.executor.memoryOverhead=2048  \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=1000 \
- --master yarn --deploy-mode cluster --name ali_install_wangjf --executor-memory 8g --driver-memory 6g  --executor-cores 3  --num-executors 50 \
+ --deploy-mode cluster --name ali_install_wangjf --executor-memory 8g --driver-memory 6g  --executor-cores 3  --num-executors 50 \
  ../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 400 -coalesce 400
 
 

azkaban/ali/ali_install_list_v2.sh

@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
+     --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
      ../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
 

azkaban/ali/ali_install_total_orc.sh

@@ -36,7 +36,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=400 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 100
 
 if [[ $? -ne 0 ]];then

azkaban/ali/ali_userinfo_activation_daily_all_job/ali_ios_userinfo_activation_daily.sh

@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliIosActivitionDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
+ --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
  ../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OUTPUT_DAILY_PATH} -coalesce 500 \
  -yesterday ${yesterday} -today ${dt_today}  -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \
  -request_count_result "${IOS_REQUEST_COUNT_RESULT}/${dt_today}"  -last_req_day ${last_req_day}

azkaban/ali/ali_userinfo_activation_daily_all_job/ali_oaid_userinfo_activation_daily.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliOaidActivitionDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
+ --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
  ../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OAID_OUTPUT_DAILY_PATH} -coalesce 500 \
  -yesterday ${yesterday} -today ${dt_today}  -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \
  -request_count_result "${OAID_REQUEST_COUNT_RESULT}/${dt_today}" -last_req_day ${last_req_day}

azkaban/ali/ali_userinfo_activation_daily_all_job/ali_userinfo_activation_daily.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
+ --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
  ../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OUTPUT_DAILY_PATH} -coalesce 500 \
  -yesterday ${yesterday} -today ${dt_today}  -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \
  -request_count_result "${REQUEST_COUNT_RESULT}/${dt_today}" -last_req_day ${last_req_day}

azkaban/ali/ali_userinfo_postback_activation_daily.sh

@@ -84,7 +84,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDail
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 60 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 60 \
  ../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT}  -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 50 \
  -today ${dt_today} -update_date ${dt_dash_today}  \
  -dt_dash_rec15day ${dt_dash_rec15day} -syn_to_3s ${ALI_USER_ACTIVATION_SYS_TO3S_PATH} -syn_3s_day ${syn_3s_day}

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_etl_postback_daily.sh

@@ -66,7 +66,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDail
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --conf spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive=true \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 70 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 70 \
  ../../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT}  -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 300 \
  -today ${dt_today} -update_date ${dt_dash_today}  -dt_taobao_postback_day ${dt_taobao_postback_day}\
  -dt_dash_rec15day ${dt_dash_rec15day} -syn_to_3s ${ALI_USER_ACTIVATION_SYS_TO3S_PATH} -syn_3s_day ${syn_3s_day}

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_extract_h_18_from_dsp_req.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlH18FromDmInstallListV2 \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
  ../../${JAR}  \
  -h18_imei ${IMEI_H_18_GUANGDIANTONG_RES_PATH} -h18_imeimd5 ${IMEIMD5_H_18_GUANGDIANTONG_RES_PATH}  \
  -dt_oneday_ago ${dt_oneday_ago}

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_insert_other_data_to_dmp.sh

@@ -43,7 +43,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlOtherDataFromPostBackDail
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 150 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 150 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}

azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck.sh

@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck_02.sh

@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck_03.sh

@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck_04.sh

@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_daily.sh

@@ -37,7 +37,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 120 ../../${JAR} \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 120 ../../${JAR} \
  -imeioutput "${ALIPAY_IMEIMD5_OUTPUT_PATH}"  \
  -today ${dt_today}  -last_req_day ${last_req_day} -dt_after_one_day ${dt_after_one_day}\
  -input_one_day ${INPUT_ONE_DAY}  -input_two_day ${INPUT_TWO_DAY}  -input_three_day ${INPUT_THREE_DAY} \

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
 
@@ -62,7 +62,7 @@ fi
 # --conf spark.sql.broadcastTimeout=1200 \
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
+# --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}
 #

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_02.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
 
@@ -62,7 +62,7 @@ fi
 # --conf spark.sql.broadcastTimeout=1200 \
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
+# --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}
 #

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_03.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
 
@@ -62,7 +62,7 @@ fi
 # --conf spark.sql.broadcastTimeout=1200 \
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
+# --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}
 #

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_04.sh

@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
 
@@ -62,7 +62,7 @@ fi
 # --conf spark.sql.broadcastTimeout=1200 \
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
+# --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}
 #

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df.sh

@@ -20,7 +20,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_02.sh

@@ -19,7 +19,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_03.sh

@@ -19,7 +19,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_04.sh

@@ -19,7 +19,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
+     --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_02.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_03.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_04.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
+ --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_to_dmp/alipay_activation_install_list.sh

@@ -24,7 +24,7 @@ spark-submit --class mobvista.dmp.datasource.TO.TOInstallList \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=100 \
  --conf spark.read.textfile.tag=1 \
- --master yarn --deploy-mode cluster --name AlipayActivationInstallList --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 30 \
+ --deploy-mode cluster --name AlipayActivationInstallList --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 30 \
  ../../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 160 -coalesce 160