remove --files、--jars, upgrade to spark 3.1.1

azkaban/3s/3s_install_device_tag.sh

@@ -32,8 +32,6 @@ hadoop fs -rmr $store_output_path
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=20 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 4 \
      ../${JAR} \
      -date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 20

azkaban/3s/3s_install_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.tracking_3s.TrackingTagDaily \
      --conf spark.sql.shuffle.partitions=5 \
      --conf spark.default.parallelism=5 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 3 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2

azkaban/3s/3s_install_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=8 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4

azkaban/3s/3s_tracking_install_total_orc.sh

@@ -38,8 +38,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=20 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 10
 

azkaban/3s_postback/3s_postback_daily.sh

@@ -40,7 +40,6 @@ spark-submit --class mobvista.dmp.datasource.postback_3s.PostBackDaily \
  --conf spark.sql.shuffle.partitions=1000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
  --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 3  --num-executors 100 \
  ../${JAR} -output ${OUTPUT_PATH}  -coalesce 100 \

azkaban/3s_postback/3s_postback_device_tag.sh

@@ -28,8 +28,6 @@ hadoop fs -rmr ${store_output_path}
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
   --conf spark.yarn.executor.memoryOverhead=2048  \
      --conf spark.sql.shuffle.partitions=1000 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g  --executor-cores 3  --num-executors 40 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1000

azkaban/3s_postback/3s_postback_device_tag_daily.sh

@@ -28,8 +28,6 @@ spark-submit --class mobvista.dmp.datasource.postback_3s.PostBackTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 5

azkaban/3s_postback/3s_postback_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=100 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 40

azkaban/3s_postback/3s_postback_install_total_orc.sh

@@ -38,7 +38,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 25 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 200
 

azkaban/ad_server_campaign/merge_campaign_list.sh

@@ -48,7 +48,6 @@ spark-submit --class mobvista.dmp.datasource.packagelist.MergeCampaignList \
   --conf spark.sql.broadcastTimeout=1200 \
   --conf spark.yarn.executor.memoryOverhead=2048 \
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 4 \
   ../${JAR} -input_dmp_data_adn ${INPUT_DMP_DATA_ADN} -input_campaign_adn ${INPUT_CAMPAIGN_ADN} -coalesce 20 \
   -output ${CAMPAIGN_TAG_OUTPUT} -today ${yester_year}-${yester_month}-${yester_day} || exit 1

azkaban/adn/device/adn_org_etl_hours_frankfurt.sh

@@ -24,7 +24,6 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
 spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
   --conf spark.yarn.executor.memoryOverhead=3072 \
   --conf spark.sql.shuffle.partitions=2000 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
   ../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 200 -region frankfurt || exit 1
 

azkaban/adn/device/adn_org_etl_hours_seoul.sh

@@ -24,7 +24,6 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
 spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
   --conf spark.yarn.executor.memoryOverhead=3072 \
   --conf spark.sql.shuffle.partitions=2000 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
   ../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 200 -region seoul || exit 1
 

azkaban/adn/device/adn_org_etl_hours_singapore.sh

@@ -23,7 +23,6 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
 spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
   --conf spark.yarn.executor.memoryOverhead=3072 \
   --conf spark.sql.shuffle.partitions=2000 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 50 \
   ../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 400 -region singapore || exit 1
 

azkaban/adn/device/adn_org_etl_hours_virginia.sh

@@ -24,7 +24,6 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
 spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
   --conf spark.yarn.executor.memoryOverhead=3072 \
   --conf spark.sql.shuffle.partitions=2000 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
   ../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 200 -region virginia || exit 1
 

azkaban/adn/package/adn_install_device_tag.sh

@@ -32,8 +32,6 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=20 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
      ../../${JAR} \
      -date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 20

azkaban/adn/package/adn_install_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnInstallTagDaily \
      --conf spark.sql.shuffle.partitions=50 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1

azkaban/adn/package/adn_install_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=8 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4

azkaban/adn/package/adn_install_total_orc.sh

@@ -32,8 +32,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.sql.shuffle.partitions=400 \
      --conf spark.default.parallelism=400 \
      --conf spark.kryoserializer.buffer.max=256m \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 4  --num-executors 5 \
      ../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 100
 

azkaban/adn/package/adn_install_total_v1.sh

@@ -31,7 +31,6 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
   ../../${JAR} \
   -date ${date} -coalesce 200 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}

azkaban/adn/package/adn_install_total_v2.sh

@@ -39,8 +39,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/adn/package/adn_request_device_tag.sh

@@ -34,8 +34,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
     --conf spark.yarn.executor.memoryOverhead=2048 \
     --conf spark.sql.shuffle.partitions=4000 \
     --conf spark.default.parallelism=4000 \
-    --files ${HIVE_SITE_PATH} \
-    --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 100 \
     ../../${JAR} \
     -date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 400

azkaban/adn/package/adn_request_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnRequestSdkTagDaily \
      --conf spark.sql.shuffle.partitions=20 \
      --conf spark.default.parallelism=5 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 2 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 20

azkaban/adn/package/adn_request_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=200 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 10 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100

azkaban/adn/package/adn_request_install_total_orc.sh

@@ -32,7 +32,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.sql.shuffle.partitions=2000 \
      --conf spark.default.parallelism=2000 \
      --conf spark.kryoserializer.buffer.max=256m \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 18g --driver-memory 4g  --executor-cores 5 --num-executors 60 \
      ../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 1000
 

azkaban/adn/package/adn_request_other_daily.sh

@@ -32,7 +32,6 @@ spark-submit --class mobvista.dmp.datasource.adn_request_other.EtlAdnRequestOthe
     --conf spark.storage.memoryFraction=0.4 \
     --conf spark.driver.maxResultSize=5g \
     --conf spark.executor.extraJavaOptions=-XX:+UseG1GC \
-    --files ${HIVE_SITE_PATH} \
     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 40 \
     ../../${JAR} -input "${INPUT_PATH}/*/*" -output $OUTPUT_PATH
 if [ $? -ne 0 ]; then

azkaban/adn/package/adn_request_other_device_tag.sh

@@ -33,8 +33,6 @@ hadoop fs -rmr ${store_output_path}
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=2000 \
       --conf spark.yarn.executor.memoryOverhead=3072 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 80 \
      ../../${JAR} \
      -date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000

azkaban/adn/package/adn_request_other_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.adn_request_other.AdnRequestOtherTa
      --conf spark.sql.shuffle.partitions=20 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10

azkaban/adn/package/adn_request_other_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=200 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
      ../../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100

azkaban/adn/package/adn_request_other_install.sh

@@ -27,7 +27,6 @@ hadoop fs -rm -r "$OUTPUT_PATH"
 spark-submit --class mobvista.dmp.datasource.adn_request_other.AdnRequestOtherInstall \
      --conf spark.yarn.executor.memoryOverhead=2048 \
      --conf spark.sql.shuffle.partitions=2000 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 200 \
      ../../${JAR} \
      -input "${INPUT_PATH}" -output $OUTPUT_PATH -date $date -oldInput $OLD_INPUT_PATH -parallelism 2000 -coalesce 2000

azkaban/adn/package/adn_request_other_install_total_orc.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 400
 

azkaban/adn/package/adn_request_other_install_v1.sh

@@ -31,7 +31,6 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
   ../../${JAR} \
   -date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}

azkaban/adn/package/adn_request_other_install_v2.sh

@@ -38,8 +38,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 30 \
      ../../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/adn/package/adn_request_pkg_total_v1.sh

@@ -31,7 +31,6 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
   ../../${JAR} \
   -date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}

azkaban/adn/package/adn_request_pkg_total_v2.sh

@@ -39,8 +39,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 2  --num-executors 40 \
      ../../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/adn/package/adn_request_unmatch_install_total_orc.sh

@@ -30,8 +30,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC " \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 4  --num-executors 50 \
      ../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 200
 

azkaban/adn_adx/adn_tencent_adx_device_tag.sh

@@ -27,7 +27,6 @@ spark-submit --class mobvista.dmp.datasource.adn_adx.AdnAdxDeviceTag \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=10 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
  --master yarn --deploy-mode cluster --name  AdnAdxDeviceTag --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 32 \
  ../${JAR}  -outputadxdevtag ${OUTPUT_ADN_ADX_DEVICE_TAG_PATH} \

azkaban/adn_adx/adn_tencent_adx_package.sh

@@ -34,7 +34,6 @@ spark-submit --class mobvista.dmp.datasource.adn_adx.AdnTecentAdxDataMidWay \
  --conf spark.default.parallelism=1000 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar,s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/json-serde-1.3.7-jar-with-dependencies.jar \
  --master yarn --deploy-mode cluster --name  AdnTecentAdxDataMidWay --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 100 \
  ../${JAR}  -outputadxtmp ${OUTPUT_ODS_ADX_TMP_PATH} -dimadxpkg ${OUTPUT_DIM_ADN_ADX_PKG_PATH} \

azkaban/adn_sdk/adn_sdk_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkDaily \
  --conf spark.driver.maxResultSize=8g \
  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
  --conf spark.app.coalesce=60000 \
- --files ${HIVE_SITE_PATH} \
- --jars ${JARS} \
  --master yarn --deploy-mode cluster --name adn_sdk_daily --executor-memory 8g --driver-memory 6g  --executor-cores 4 --num-executors 200 \
  ../${JAR}
 

azkaban/adn_sdk/adn_sdk_device_tag.sh

@@ -34,8 +34,6 @@ hadoop fs -rmr ${store_output_path}
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=2000 \
      --conf spark.yarn.executor.memoryOverhead=4096 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g  --executor-cores 3 --num-executors 200 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000

azkaban/adn_sdk/adn_sdk_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10

azkaban/adn_sdk/adn_sdk_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=1000 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100

azkaban/adn_sdk/adn_sdk_install_total_orc.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=4000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 100 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 2000
 

azkaban/adn_sdk/adn_sdk_install_v1.sh

@@ -31,7 +31,6 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 80 \
   ../${JAR} \
   -date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}

azkaban/adn_sdk/adn_sdk_install_v2.sh

@@ -39,8 +39,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
      ../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/adn_sdk/adn_sdk_v2_device_tag.sh

@@ -34,8 +34,6 @@ hadoop fs -rmr ${store_output_path}
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=2000 \
      --conf spark.yarn.executor.memoryOverhead=4096 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 4  --num-executors 150 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000

azkaban/adn_sdk/adn_sdk_v2_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=100 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100

azkaban/adn_sdk/adn_sdk_v2_install_total_orc.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=500 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 200
 

azkaban/adn_sdk/adn_sdk_v2_install_v1.sh

@@ -31,7 +31,6 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.kryoserializer.buffer=64m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
   ../${JAR} \
   -date ${date} -coalesce 200 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}

azkaban/adn_sdk/adn_sdk_v2_install_v2.sh

@@ -43,8 +43,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
      ../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/age/get_dsp_all.sh

@@ -36,8 +36,6 @@ spark-submit --class mobvista.dmp.datasource.age_gender.GetAgeGender \
      --conf spark.sql.shuffle.partitions=2000 \
      --conf spark.default.parallelism=2000 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
      ../${JAR} -ageOutput ${AGE_OUTPUT_PATH} -genderOutput ${GENDER_OUTPUT_PATH} -date ${LOG_TIME} -business ${business}
 

azkaban/age/get_ga_all.sh

@@ -58,8 +58,6 @@ spark-submit --class mobvista.dmp.datasource.age_gender.GetAgeGender \
      --conf spark.speculation.quantile=0.9 \
      --conf spark.speculation.multiplier=1.5 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -ageOutput ${AGE_OUTPUT_PATH} -genderOutput ${GENDER_OUTPUT_PATH} -date ${GA_TOTAL_DATE} -business ${business}
 

azkaban/age/merge_install_age.sh

@@ -52,7 +52,6 @@ spark-submit --class mobvista.dmp.datasource.age_gender.MergeInstallAge \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
      ../${JAR} -date ${LOG_TIME} \
      -ga_age_path ${GA_AGE_PATH}  -dsp_age_path ${DSP_AGE_PATH} -age_output ${OUTPUT_PATH} -parallelism 2000

azkaban/ali/TO/TO_daily.sh

@@ -27,7 +27,6 @@ spark-submit --class mobvista.dmp.datasource.TO.TODaily \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 20 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 200  -dt_dash_today ${dt_dash_today}

azkaban/ali/ali_ck.sh

@@ -25,7 +25,6 @@ spark-submit --class mobvista.dmp.datasource.baichuan.BaiChuanJob \
      --conf spark.default.parallelism=100 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores 2  --num-executors 20 \
      ../${JAR} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/ali/ali_daily.sh

@@ -24,7 +24,6 @@ spark-submit --class mobvista.dmp.datasource.baichuan.AliDaily \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 5 \
      ../${JAR} -date ${LOG_TIME} -partNum 10 -output ${OUTPUT_PATH} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/ali/ali_device_tag.sh

@@ -36,8 +36,6 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=100 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 40

azkaban/ali/ali_device_tag_daily.sh

@@ -33,7 +33,6 @@ spark-submit --class mobvista.dmp.datasource.baichuan.AliTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 5

azkaban/ali/ali_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=8 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 8

azkaban/ali/ali_install_list_v2.sh

@@ -39,8 +39,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
      ../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/ali/ali_install_total_orc.sh

@@ -34,8 +34,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=400 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 100
 

azkaban/ali/ali_userinfo_activation_daily_all_job/ali_ios_userinfo_activation_daily.sh

@@ -34,7 +34,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliIosActivitionDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
  ../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OUTPUT_DAILY_PATH} -coalesce 500 \
  -yesterday ${yesterday} -today ${dt_today}  -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \

azkaban/ali/ali_userinfo_activation_daily_all_job/ali_oaid_userinfo_activation_daily.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliOaidActivitionDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
  ../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OAID_OUTPUT_DAILY_PATH} -coalesce 500 \
  -yesterday ${yesterday} -today ${dt_today}  -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \

azkaban/ali/ali_userinfo_activation_daily_all_job/ali_userinfo_activation_daily.sh

@@ -33,7 +33,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
  ../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OUTPUT_DAILY_PATH} -coalesce 500 \
  -yesterday ${yesterday} -today ${dt_today}  -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \

azkaban/ali/ali_userinfo_postback_activation_daily.sh

@@ -84,7 +84,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDail
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 60 \
  ../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT}  -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 50 \
  -today ${dt_today} -update_date ${dt_dash_today}  \

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_etl_postback_daily.sh

@@ -66,7 +66,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDail
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --conf spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive=true \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 70 \
  ../../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT}  -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 300 \
  -today ${dt_today} -update_date ${dt_dash_today}  -dt_taobao_postback_day ${dt_taobao_postback_day}\

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_extract_h_18_from_dsp_req.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlH18FromDmInstallListV2 \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 60 \
  ../../${JAR}  \
  -h18_imei ${IMEI_H_18_GUANGDIANTONG_RES_PATH} -h18_imeimd5 ${IMEIMD5_H_18_GUANGDIANTONG_RES_PATH}  \

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_insert_other_data_to_dmp.sh

@@ -43,7 +43,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlOtherDataFromPostBackDail
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 150 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} \

azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck.sh

@@ -25,7 +25,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
 

azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck_02.sh

@@ -25,7 +25,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
 

azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck_03.sh

@@ -25,7 +25,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
 

azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck_04.sh

@@ -25,7 +25,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
 

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_daily.sh

@@ -37,7 +37,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 120 ../../${JAR} \
  -imeioutput "${ALIPAY_IMEIMD5_OUTPUT_PATH}"  \
  -today ${dt_today}  -last_req_day ${last_req_day} -dt_after_one_day ${dt_after_one_day}\

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp.sh

@@ -33,7 +33,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
@@ -63,7 +62,6 @@ fi
 # --conf spark.sql.broadcastTimeout=1200 \
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-# --files ${HIVE_SITE_PATH} \
 # --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_02.sh

@@ -33,7 +33,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
@@ -63,7 +62,6 @@ fi
 # --conf spark.sql.broadcastTimeout=1200 \
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-# --files ${HIVE_SITE_PATH} \
 # --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_03.sh

@@ -33,7 +33,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
@@ -63,7 +62,6 @@ fi
 # --conf spark.sql.broadcastTimeout=1200 \
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-# --files ${HIVE_SITE_PATH} \
 # --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_04.sh

@@ -33,7 +33,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
@@ -63,7 +62,6 @@ fi
 # --conf spark.sql.broadcastTimeout=1200 \
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-# --files ${HIVE_SITE_PATH} \
 # --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df.sh

@@ -20,8 +20,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars s3://mob-emr-test/wangjf/jar/spark-clickhouse-connector_2.11-2.4.0_0.22.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_02.sh

@@ -19,8 +19,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars s3://mob-emr-test/wangjf/jar/spark-clickhouse-connector_2.11-2.4.0_0.22.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_03.sh

@@ -19,8 +19,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars s3://mob-emr-test/wangjf/jar/spark-clickhouse-connector_2.11-2.4.0_0.22.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_04.sh

@@ -19,8 +19,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars s3://mob-emr-test/wangjf/jar/spark-clickhouse-connector_2.11-2.4.0_0.22.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -cluster 'cluster_1st'  -hour ${hour}
 

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_02.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_03.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_04.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \

azkaban/ali/btop/btop_daily.sh

@@ -45,7 +45,6 @@ spark-submit --class mobvista.dmp.datasource.btop.BtopDaily \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 60 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 200  -dt_today ${dt_today} -dt_dash_tow_days ${dt_dash_tow_days}  -seven_days_ago ${seven_days_ago}  -fifteen_days_ago ${fifteen_days_ago} \

azkaban/ali/cainixihuan/cainixihuan01.sh

@@ -22,7 +22,6 @@ spark-submit \
 	--class mobvista.dmp.datasource.apptag.CaiNiXiHuanCrawlerSpark \
 	--conf spark.network.timeout=720s \
 	--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--name cainixihuan_AppInfoCrawlerSpark \

azkaban/ali/cainixihuan/cainixihuan02.sh

@@ -25,7 +25,6 @@ spark-submit \
 	--class mobvista.dmp.datasource.apptag.CaiNiXiHuanCrawlerSpark \
 	--conf spark.network.timeout=720s \
 	--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--name cainixihuan_AppInfoCrawlerSpark \

azkaban/ali/cainixihuan/cainixihuan03.sh

@@ -24,7 +24,6 @@ spark-submit \
 	--class mobvista.dmp.datasource.apptag.CaiNiXiHuanCrawlerSpark \
 	--conf spark.network.timeout=720s \
 	--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--name cainixihuan_AppInfoCrawlerSpark \

azkaban/ali/cainixihuan/cainixihuan04.sh

@@ -24,7 +24,6 @@ spark-submit \
 	--class mobvista.dmp.datasource.apptag.CaiNiXiHuanCrawlerSpark \
 	--conf spark.network.timeout=720s \
 	--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--name cainixihuan_AppInfoCrawlerSpark \

azkaban/ali/cainixihuan/cainixihuan05.sh

@@ -24,7 +24,6 @@ spark-submit \
 	--class mobvista.dmp.datasource.apptag.CaiNiXiHuanCrawlerSpark \
 	--conf spark.network.timeout=720s \
 	--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--name cainixihuan_AppInfoCrawlerSpark \

azkaban/ali/etl_dealid_hour.sh

@@ -36,7 +36,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlDealidDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 6  --num-executors 30 \
  ../${JAR}  -dt_dash_today ${dt_dash_today} \
   -oppooutput ${OPPO_OUTPUT}   \

azkaban/ali/etl_lazada_data_daily.sh

@@ -62,7 +62,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlLazadaActivitionDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 90 ../${JAR} \
  -gaidoutput "${GAID_OUTPUT_PATH}"   \
  -today ${dt_today}  -last_req_day ${last_req_day} \
@@ -107,7 +106,6 @@ fi
 # --conf spark.sql.broadcastTimeout=1200 \
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-# --files ${HIVE_SITE_PATH} \
 # --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 70 ../${JAR} \
 # -gaidoutput "${GAID_OUTPUT_PATH}" -gaidinput "${GAID_INPUT_PATH}"  -newoutput "${NEW_OUTPUT_PATH}" \
 # -today ${dt_today}  -dt_30days_ago ${dt_30days_ago}

azkaban/ali/etl_lazada_ios_data_daily.sh

@@ -32,7 +32,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlLazadaIosActivitionDaily
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 90 ../${JAR} \
  -today ${dt_today}  -last_req_day ${last_req_day} \
  -vn_idfaoutput "${VN_IDFA_OUTPUT_PATH}"  -id_idfaoutput "${ID_IDFA_OUTPUT_PATH}"  -th_idfaoutput "${TH_IDFA_OUTPUT_PATH}" \

azkaban/ali/other_single_jobs/etl_com_tencent_news_daily.sh

@@ -17,7 +17,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlComTencentNewsDaily \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 120 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 500 \

azkaban/ali/reyun/reyun_daily.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.reyun.ReyunDaily \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 3  --num-executors 60 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 400  -dt_today ${dt_today}

azkaban/ali/reyun/reyun_label_baijiu.sh

@@ -24,8 +24,6 @@ spark-submit --class mobvista.dmp.datasource.reyun.ReyunLabelBaijiu \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
- --jars ${JARS} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 170 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 680  -dt_today ${dt_today}

azkaban/ali/reyun/reyun_label_test.sh

@@ -26,7 +26,6 @@ spark-submit --class mobvista.dmp.datasource.reyun.ReyunLabelTest \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 3  --num-executors 40 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 40  -dt_today ${dt_today}

azkaban/ali/uc_lahuo/uc_imei_lahuo_ck.sh

@@ -24,7 +24,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCImeiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/ali/uc_lahuo/uc_lahuo_daily.sh

@@ -24,7 +24,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCLaHuoDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 80 ../../${JAR} \
  -imeioutput "${UC_IMEIMD5_OUTPUT_PATH}"  -oaidoutput "${UC_OAIDMD5_OUTPUT_PATH}"  \
  -today ${dt_today}  -last_req_day ${last_req_day}

azkaban/ali/uc_lahuo/uc_lahuo_data_to_dmp.sh

@@ -40,7 +40,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCTmpDataToDMP \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${UC_IMEIMD5_REQUEST_INPUT_PATH}  -oaidRequestInput ${UC_OAIDMD5_REQUEST_INPUT_PATH} \
   -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH}  -oaidResponseInput ${OAIDMD5_RESPONSE_INPUT_PATH}   \

azkaban/ali/uc_lahuo/uc_lahuo_df.sh

@@ -20,8 +20,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars s3://mob-emr-test/wangjf/jar/spark-clickhouse-connector_2.11-2.4.0_0.22.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -partNum 10 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -oaidOutput ${OAIDMD5_OUTPUT_PATH}  -cluster 'cluster_1st'
 

azkaban/ali/uc_lahuo/uc_oaid_lahuo_ck.sh

@@ -24,7 +24,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCOaidLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR}  -oaidInput ${OAIDMD5_INPUT} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/ali/uc_lahuo/uc_other_data_to_dmp.sh

@@ -33,7 +33,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCOtherDataToDmp \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 150 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \

azkaban/ali/uc_lahuo/uc_other_data_to_dmp_v2.sh

@@ -32,7 +32,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCOtherDataToDmpV2 \
   --conf spark.sql.shuffle.partitions=3000 \
   --conf spark.sql.broadcastTimeout=1200 \
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 5 --num-executors 120 \
   ../../${JAR} \
   -output ${OUTPUT_PATH} \

azkaban/ali/uc_lahuo_to_guangdiantong/ali_extract_h_32_from_dsp_req.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlH32FromDmInstallListV2 \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 60 \
  ../../${JAR}  \
  -h32_imei ${IMEI_H_32_GUANGDIANTONG_RES_PATH} -h32_imeimd5 ${IMEIMD5_H_32_GUANGDIANTONG_RES_PATH}  \

azkaban/ali/youku_laxin/youku_imei_laxin_ck.sh

@@ -23,7 +23,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.YOUKUImeiLaXinCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -imeiInput ${IMEIMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/ali/youku_laxin/youku_laxin_daily.sh

@@ -35,7 +35,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.YOUKULaXinDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 80 ../../${JAR} \
  -imeioutput "${YOUKU_IMEIMD5_OUTPUT_PATH}"  -oaidoutput "${YOUKU_OAIDMD5_OUTPUT_PATH}"  \
  -input_one_day ${INPUT_ONE_DAY}  -input_two_day ${INPUT_TWO_DAY}  -input_three_day ${INPUT_THREE_DAY} \

azkaban/ali/youku_laxin/youku_laxin_data_to_dmp.sh

@@ -25,7 +25,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.YoukuTmpDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -Input "${INPUT_PATH}/*/*"   -Output ${OUTPUT_PATH}   \
   -update ${update}
@@ -53,7 +52,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.YoukuLaXinPollingDataDedupli
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -dt_today ${dt_today}  -dt_begin_days ${dt_begin_days} \
  -AcquisitionOutput ${ACQUISITIONOUTPUT}

azkaban/ali/youku_laxin/youku_laxin_df.sh

@@ -20,8 +20,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.YOUKULaxinDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars s3://mob-emr-test/wangjf/jar/spark-clickhouse-connector_2.11-2.4.0_0.22.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 10 \
      ../../${JAR} -date ${LOG_TIME} -dt_three_days_ago ${dt_three_days_ago} -partNum 10 -imeiOutput ${IMEIMD5_OUTPUT_PATH}  -oaidOutput ${OAIDMD5_OUTPUT_PATH}  -cluster 'cluster_1st'
 

azkaban/ali/youku_laxin/youku_oaid_laxin_ck.sh

@@ -23,7 +23,6 @@ spark-submit --class mobvista.dmp.datasource.taobao.YOUKUOaidLaXinCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -oaidInput ${OAIDMD5_INPUT}  -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/app_info/app_info_adr_3s.sh

@@ -21,7 +21,7 @@ check_await "s3://mob-emr-test/leo.liang/dmp_dummper/all_3s_android.json.log"
 #--deploy-mode cluster 
 spark-submit --class mobvista.dmp.datasource.app_info_tag.App_info_adr_3s \
  --conf spark.yarn.executor.memoryOverhead=2048  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml --name app_info_adr_3s --conf spark.app.loadTime=${LOG_TIME} --conf spark.app.output_path=${OUTPUT_PATH} --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 \
+ --master yarn --deploy-mode cluster --name app_info_adr_3s --conf spark.app.loadTime=${LOG_TIME} --conf spark.app.output_path=${OUTPUT_PATH} --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 \
  ../${JAR}
 if [ $? -ne 0 ];then
   exit 255

azkaban/app_info/app_info_ios_3s.sh

@@ -19,7 +19,7 @@ check_await "s3://mob-emr-test/leo.liang/dmp_dummper/all_3s_ios.json.log"
 
 spark-submit --class mobvista.dmp.datasource.app_info_tag.App_info_ios_3s \
  --conf spark.yarn.executor.memoryOverhead=2048  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster --name app_info_ios_3s --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml --conf spark.app.loadTime=${LOG_TIME} --conf spark.app.output_path=${OUTPUT_PATH} --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 \
+ --master yarn --deploy-mode cluster --name app_info_ios_3s --conf spark.app.loadTime=${LOG_TIME} --conf spark.app.output_path=${OUTPUT_PATH} --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 \
  ../${JAR}
 if [ $? -ne 0 ];then
   exit 255

azkaban/app_info/collect_package_name.sh

@@ -83,7 +83,6 @@ install() {
     --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
     --conf spark.kryoserializer.buffer.max=512m \
     --conf spark.driver.maxResultSize=4g \
-    --files ${HIVE_SITE_PATH} \
     --master yarn --deploy-mode cluster --name CrawPkgsSpark --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 100 \
     ../${JAR} -pkginstallpath ${PACKAGE_INSTALL_PATH} -coalesce 10 \
     -yesday ${YESTERDAY}

azkaban/app_info/etl_app_info_adr.sh

@@ -24,15 +24,13 @@ echo "dim_app_info_adr file success exist, and then can start"
 : '
 spark-submit --class mobvista.dmp.datasource.app_info_tag.Etl_app_info_adr \
  --conf spark.yarn.executor.memoryOverhead=2048  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster --name etl_app_info_adr --conf spark.app.loadTime=${LOG_TIME} --conf spark.app.output_path=${OUTPUT_PATH} --executor-memory 6g --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml --driver-memory 4g --executor-cores 2 --num-executors 30 \
+ --master yarn --deploy-mode cluster --name etl_app_info_adr --conf spark.app.loadTime=${LOG_TIME} --conf spark.app.output_path=${OUTPUT_PATH} --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 \
  ../${JAR}
 '
 
 spark-submit --class mobvista.dmp.datasource.app_info_tag.Etl_app_info_adr_v2 \
      --name "mobvista.dmp.datasource.app_info_tag.Etl_app_info_adr_v2_wangjf_${LOG_TIME}" \
      --conf spark.yarn.executor.memoryOverhead=2048 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 2 \
      ../${JAR} \
      -date ${LOG_TIME} -output ${OUTPUT_PATH}

azkaban/app_info/etl_app_info_ios.sh

@@ -26,15 +26,13 @@ echo "dim_app_info_ios  file success exist, and then can start"
 : '
 spark-submit --class mobvista.dmp.datasource.app_info_tag.Etl_app_info_ios \
  --conf spark.yarn.executor.memoryOverhead=2048  --conf spark.network.timeout=720s \
---master yarn --deploy-mode cluster --name etl_app_info_ios --conf spark.app.loadTime=${LOG_TIME} --conf spark.app.output_path=${OUTPUT_PATH} --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml \
+--master yarn --deploy-mode cluster --name etl_app_info_ios --conf spark.app.loadTime=${LOG_TIME} --conf spark.app.output_path=${OUTPUT_PATH} --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 \
  ../${JAR}
 '
 
 spark-submit --class mobvista.dmp.datasource.app_info_tag.Etl_app_info_ios_v2 \
      --name "mobvista.dmp.datasource.app_info_tag.Etl_app_info_ios_v2_wangjf_${LOG_TIME}" \
      --conf spark.yarn.executor.memoryOverhead=2048 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 2 \
      ../${JAR} \
      -date ${LOG_TIME} -output ${OUTPUT_PATH}

azkaban/app_info/import_campaign_tags.sh

@@ -28,7 +28,6 @@ hadoop fs -rmr $output_path
 spark-submit --class mobvista.dmp.datasource.newtag.ImportCampaignTags \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=5 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
     ../${JAR} -newPath ${new_path} -output $output_path
 

azkaban/app_info/import_package_tags.sh

@@ -31,7 +31,6 @@ hadoop fs -rmr $output_path
 spark-submit --class mobvista.dmp.datasource.newtag.ImportPkgTags \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=5 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 5 \
     ../${JAR} -newPath ${new_path} -output $output_path
 

azkaban/app_info/package_black_list.sh

@@ -20,7 +20,6 @@ EXPIRE_PATH="${PACKAGE_BLACK_LIST}/$expire_path"
 
 spark-submit --class mobvista.dmp.main.PackageBlackList \
      --conf spark.sql.shuffle.partitions=10 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date "${yes_date}" \

azkaban/app_tag/app_tag.sh

@@ -15,7 +15,6 @@ spark-submit --class mobvista.dmp.datasource.apptag.AppTagJob \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf dfs.socket.timeout=300000 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} -output ${output_path}
 

azkaban/appsflyer/appsflyer_total.sh

@@ -62,7 +62,6 @@ $HIVE_CMD	-v  -hivevar  dt_today ${dt_today}     -hivevar update_date ${dt_today
   --conf spark.network.timeout=720s \
   --conf spark.default.parallelism=10 \
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-  --files ${HIVE_SITE_PATH} \
   --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
   --master yarn --deploy-mode cluster --name  apps_flyer_total --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 5 \
   ../${JAR}  -outputtotal ${OUTPUT_TOTAL_PATH} -dmpuserinfo ${DMP_USER_INFO_OUTPUT_PATH} \

azkaban/bigmedia_domestic/bigmedia_domestic_launch_total.sh

@@ -52,7 +52,6 @@ spark-submit --class mobvista.dmp.datasource.bigmedia_domestic.BigMediaDomestic
  --conf spark.default.parallelism=400 \
  --conf spark.sql.shuffle.partitions=400 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
  --master yarn --deploy-mode cluster --name BigMediaDomestic --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 36  \
  ../${JAR}  -bigmediainput  ${INPUT_PATH_DAILY} -outputdaily ${OUTPUT_BIGMEDIA_DAILY_PATH} -outputgender ${GENDER_BIG_MEDIA_OUTPUT_PATH} -coalesce 50 \

azkaban/bytedance/bytedance_device_tag.sh

@@ -33,8 +33,6 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=50 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g  --executor-cores 3  --num-executors 20 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10

azkaban/bytedance/bytedance_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.bytedance.ByteDanceTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 5

azkaban/bytedance/bytedance_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=8 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 8

azkaban/bytedance/bytedance_install_total_orc.sh

@@ -35,8 +35,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=20 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 10
 

azkaban/clever/adn_clever_device_tag.sh

@@ -33,8 +33,6 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=100 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g  --executor-cores 3  --num-executors 20 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100

azkaban/clever/adn_clever_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.clever.AdnCleverTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1

azkaban/clever/adn_clever_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=4 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4

azkaban/clever/adn_clever_install_total_orc.sh

@@ -34,8 +34,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=20 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 10
 

azkaban/clever/adn_clever_install_v2.sh

@@ -36,8 +36,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../${JAR} \
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/datatory/datatory.sh

@@ -35,8 +35,6 @@ spark-submit --class mobvista.dmp.datasource.datatory.DatatoryJob \
      --conf spark.default.parallelism=${partition} \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory ${execNum}g --driver-memory 6g  --executor-cores 4  --num-executors 150 \
      ../${JAR} -json ${json// /&@} -tag "${tag}" -coalesce 1
 

azkaban/datatory/tracking/3s_tracking_event_daily.sh

@@ -28,8 +28,6 @@ spark-submit --class mobvista.dmp.datasource.datatory.TrackingEventDaily \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../../${JAR} -date ${date} -output ${output_path} -info_output ${info_output_path} -before_date ${before_date}
 

azkaban/datatory/tracking/3s_trackingnew_install_daily.sh

@@ -26,8 +26,6 @@ spark-submit --class mobvista.dmp.datasource.datatory.TrackingInstallDaily \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../../${JAR} -date ${date} -output ${output_path}
 

azkaban/datatory/tracking/adn_tracking_ck.sh

@@ -23,7 +23,6 @@ spark-submit --class mobvista.dmp.clickhouse.tracking.AdnTrackingDaily \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../../${JAR} -date ${date} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/datatory/tracking/adn_trackingnew_event_daily.sh

@@ -25,8 +25,6 @@ spark-submit --class mobvista.dmp.datasource.datatory.AdnTrackingEventDaily \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../../${JAR} -date ${date} -output ${output_path}
 

azkaban/datatory/tracking/adn_trackingnew_install_daily.sh

@@ -25,8 +25,6 @@ spark-submit --class mobvista.dmp.datasource.datatory.AdnTrackingInstallDaily \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../../${JAR} -date ${date} -output ${output_path}
 

azkaban/datatory/tracking/adn_trackingnew_merge_daily.sh

@@ -25,8 +25,6 @@ spark-submit --class mobvista.dmp.datasource.datatory.AdnTrackingMergeDaily \
      --conf spark.sql.shuffle.partitions=100 \
      --conf spark.default.parallelism=100 \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 3  --num-executors 5 \
      ../../${JAR} -date ${date} -output ${output_path}
 

azkaban/datatory/tracking/tracking_ck.sh

@@ -30,7 +30,6 @@ spark-submit --class mobvista.dmp.clickhouse.tracking.TrackingDaily \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../../${JAR} -date ${date} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/datatory/tracking/trackingnew_merge_daily.sh

@@ -23,8 +23,6 @@ spark-submit --class mobvista.dmp.datasource.datatory.TrackingMergeDaily \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../../${JAR} -date ${date} -output ${output_path}
 

azkaban/datatory/user_info/user_info_ck.sh

@@ -32,7 +32,6 @@ spark-submit --class mobvista.dmp.clickhouse.feature.UserInfo \
      --conf spark.kryoserializer.buffer.max=512m \
      --conf spark.kryoserializer.buffer=64m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 10 \
      ../../${JAR} -date ${date} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/dm/dm_active_tag_month.sh

@@ -64,8 +64,6 @@ spark-submit --class mobvista.dmp.datasource.dm.ActiveTag \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 18g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
      ../${JAR} \
      -date ${date} -output ${OUTPUT_PATH} -coalesce 1000 -days 29

azkaban/dm/dm_active_tag_week.sh

@@ -66,8 +66,6 @@ spark-submit --class mobvista.dmp.datasource.dm.ActiveTag \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 18g --driver-memory 4g  --executor-cores 5  --num-executors 40 \
      ../${JAR} \
      -date ${date} -output ${OUTPUT_PATH} -coalesce 1000 -days 6

azkaban/dm/dm_interest_tag_all_v2.sh

@@ -48,8 +48,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmpDeviceInterest \
      --conf spark.kryoserializer.buffer=64m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 100 \
      ../${JAR} \
      -date ${date} -output ${OUTPUT_PATH}

azkaban/dm/dm_interest_tag_all_v3.sh

@@ -60,8 +60,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInterestTagAllV2 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 3  --num-executors 256 \
      ../${JAR} \
      -output ${OUTPUT_PATH} -date ${date} -ga_date ${ga_date} -coalesce 5000

azkaban/dm/dmp_device_id_md5.sh

@@ -26,7 +26,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmpDeviceIdMd5 \
     --conf spark.kryoserializer.buffer=64m \
     --conf spark.sql.adaptive.enabled=true \
     --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-    --files ${HIVE_SITE_PATH} \
     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 60 \
     ../${JAR} \
     -date ${date} -coalesce 1000 -output $OUTPUT_PATH

azkaban/dm/dmp_device_interest.sh

@@ -34,8 +34,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmpDeviceInterest \
     --conf spark.kryoserializer.buffer=64m \
     --conf spark.sql.adaptive.enabled=true \
     --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-    --files ${HIVE_SITE_PATH} \
-    --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 80 \
     ../${JAR} \
     -date ${date} -output ${OUTPUT_PATH} -coalesce 4000

azkaban/dm/dmp_device_tag_daily.sh

@@ -39,7 +39,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.kryoserializer.buffer=64m \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 18g --driver-memory 6g  --executor-cores 5  --num-executors 80 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4000

azkaban/dm/dmp_install_list_common.sh

@@ -27,7 +27,6 @@ spark-submit --class mobvista.dmp.common.Old2NewInstallList \
      --conf spark.default.parallelism=${PARTITION} \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors ${EXECUTOR} \
      ../${JAR} -date ${LOG_TIME} -business ${BUSINESS} -output ${OUTPUT}  -coalesce ${COALESCE}
 

azkaban/dm/dmp_install_list_daily.sh

@@ -60,7 +60,6 @@ spark-submit --class mobvista.dmp.common.InstallListDaily \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
      --conf spark.kubernetes.memoryOverheadFactor=0.2 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3 --num-executors 160 \
      ../${JAR} -date ${date} -coalesce 3000 -output ${OUTPUT_PATH}
 

azkaban/dm/dmp_install_list_daily_v2.sh

@@ -58,7 +58,6 @@ spark-submit --class mobvista.dmp.common.InstallListDailyV2 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
      --conf spark.kubernetes.memoryOverheadFactor=0.2 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 3 --num-executors 256 \
      ../${JAR} -date ${date} -coalesce 4000 -output ${OUTPUT_PATH}
 

azkaban/dm/dmp_install_list_merge.sh

@@ -41,7 +41,6 @@ spark-submit --class mobvista.dmp.common.InstallListMerge \
      --conf spark.driver.maxResultSize=8g \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.sql.files.maxPartitionBytes=134217728 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 5 --num-executors 100 \
      ../${JAR} -date ${date} -coalesce 4000 -output ${OUTPUT_PATH}
 

azkaban/dm/fix_dmp_install_list.sh

@@ -24,7 +24,6 @@ spark-submit --class mobvista.dmp.datasource.dm.FixDmpInstallList \
     --conf spark.kryoserializer.buffer=64m \
     --conf spark.sql.adaptive.enabled=true \
     --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-    --files ${HIVE_SITE_PATH} \
     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
     ../${JAR} \
     -date ${date} -coalesce 4000 -output $OUTPUT_PATH -business ${BUSINESS}

azkaban/dm/fix_install_list_ruid.sh

@@ -27,7 +27,6 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
   ../${JAR} \
   -date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}

azkaban/dm/install_list_other_v2.sh

@@ -37,8 +37,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../${JAR}
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/dm/install_other_device_tag.sh

@@ -33,8 +33,6 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=200 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 40 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 200

azkaban/dm/other_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=4 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4

azkaban/dm/other_install_total_orc.sh

@@ -29,8 +29,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=8 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 4
 

azkaban/dm/pseudo_package_to_other_business/Age_Package_Names.sh

@@ -26,7 +26,6 @@ spark-submit --class mobvista.dmp.datasource.dm.AgePackageNames \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 2 --num-executors 220 \
  ../../${JAR} -dt_today ${dt_today}   -update ${update}  -Age_Package_Names ${Age_Package_Names}   -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 

azkaban/dm/pseudo_package_to_other_business/Canglan_Package_Names.sh

@@ -26,7 +26,6 @@ spark-submit --class mobvista.dmp.datasource.dm.CanglanPackageNames \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 2 --num-executors 220 \
  ../../${JAR} -dt_today ${dt_today}   -update ${update}  -Canglan_Package_Names ${Canglan_Package_Names}   -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 

azkaban/dm/pseudo_package_to_other_business/Three_Kingdoms_Game.sh

@@ -26,7 +26,6 @@ spark-submit --class mobvista.dmp.datasource.dm.ThreeKingdomsGame \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 2 --num-executors 220 \
  ../../${JAR} -dt_today ${dt_today}   -update ${update}  -package_names_input ${Three_Kingdoms_Package_Names}   -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 

azkaban/dm/pseudo_package_to_other_business/adx_packagename_synchronize.sh

@@ -21,7 +21,6 @@ spark-submit --class mobvista.dmp.datasource.dm.AdxPackagenameSynchronize \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 50 \
  ../../${JAR} -dt_today ${dt_today}    -output1 ${OUTPUT_PATH1} \
  -dt_day ${dt_day} \

azkaban/dm/pseudo_package_to_other_business/btop_tiktokrv.sh

@@ -30,7 +30,6 @@ spark-submit --class mobvista.dmp.datasource.dm.BtopTiktokrv \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 70 \
  ../../${JAR} -begin_day ${begin_day}   -begin_day02 ${begin_day02}  -end_day ${end_day} -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02}  -output03 ${OUTPUT_PATH03} -output04 ${OUTPUT_PATH04} -coalesce 200
 

azkaban/dm/pseudo_package_to_other_business/btop_tiktokrv_gaid.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.dm.BtopTiktokrvGaid \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 70 \
  ../../${JAR} -begin_day ${begin_day}   -begin_day02 ${begin_day02}  -end_day ${end_day} -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02}  -output03 ${OUTPUT_PATH03}  -coalesce 200
 

azkaban/dm/pseudo_package_to_other_business/cn_good_channel.sh

@@ -35,7 +35,6 @@ spark-submit --class mobvista.dmp.datasource.dm.CnGoodChannel \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 100 \
  ../../${JAR}    -output1 ${OUTPUT_PATH1}  -output2 ${OUTPUT_PATH2}  -output3 ${OUTPUT_PATH3}   \
   -output4 ${OUTPUT_PATH4}  -output5 ${OUTPUT_PATH5}  -old_data_path ${OLD_DATA_PATH} \

azkaban/dm/pseudo_package_to_other_business/com_eg_android_AlipayGphone_reyun.sh

@@ -36,7 +36,6 @@ spark-submit --class mobvista.dmp.datasource.dm.ComEgAndroidAlipayGphoneReyun \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 10 \
  ../../${JAR} -dt_today ${dt_today}    -output1 ${OUTPUT_PATH1}  -output2 ${OUTPUT_PATH2} -update ${update}  \
   -coalesce 200

azkaban/dm/pseudo_package_to_other_business/phone_wax_nobid_to_dmp.sh

@@ -27,7 +27,6 @@ spark-submit --class mobvista.dmp.datasource.dm.YoukuPhoneWaxNobid \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 150 \
  ../../${JAR} -begin_day ${begin_day} -end_day ${end_day} -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 

azkaban/dm/pseudo_package_to_other_business/reyun_lahuo_list.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.dm.ReyunLahuoList \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 10 \
  ../../${JAR} -dt_today ${dt_today}  -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02}  -update_date ${update_date}\
   -coalesce 200

azkaban/dm/pseudo_package_to_other_business/rtdmp_normal.sh

@@ -26,7 +26,6 @@ spark-submit --class mobvista.dmp.datasource.dm.RtdmpNormal \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 30 \
  ../../${JAR} -dt_today ${dt_today}  -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02}  -rtdmp_normal_count_result "${RTDMP_NORMAL_COUNT_RESULT}/${dt_slash_today}"  -coalesce 200
 

azkaban/dm/pseudo_package_to_other_business/rtdmp_tmp_id1142110895.sh

@@ -17,7 +17,6 @@ spark-submit --class mobvista.dmp.datasource.dm.RtdmpTmpId1142110895 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.driver.maxResultSize=4g \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 70 \
  ../../${JAR} -dt_today ${dt_today}    -output1 ${OUTPUT_PATH1} \
   -coalesce 420

azkaban/dm/pseudo_package_to_other_business/shinny.sh

@@ -27,7 +27,6 @@ spark-submit --class mobvista.dmp.datasource.dm.ShinnyPackageNames \
  --conf spark.default.parallelism=3000 \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 2 --num-executors 220 \
  ../../${JAR} -dt_today ${dt_today}   -update ${update}  -Shinny_Package_Names ${Shinny_Package_Names}   -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 

azkaban/dmp_env.sh

@@ -849,8 +849,6 @@ userInfoJob() {
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.9 \
      --conf spark.speculation.multiplier=1 \
-     --jars ${JARS} \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
      ${jar} -date $LOG_TIME -dailyPath $dailyPath -agePath $agePath -genderPath $genderPath \
      -dailyFormat ${dailyFormat} -dailyDidIndex $dailyDidIndex -dailyDidTypeIndex $dailyDidTypeIndex -dailyPltIndex $dailyPltIndex -dailyCountryIndex $dailyCountryIndex \
@@ -898,8 +896,6 @@ userInfoJob_dsp_req() {
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.9 \
      --conf spark.speculation.multiplier=1 \
-     --jars ${JARS} \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 80 \
      ${jar} -date $LOG_TIME -dailyPath $dailyPath -agePath $agePath -genderPath $genderPath \
      -dailyFormat ${dailyFormat} -dailyDidIndex $dailyDidIndex -dailyDidTypeIndex $dailyDidTypeIndex -dailyPltIndex $dailyPltIndex -dailyCountryIndex $dailyCountryIndex \
@@ -963,8 +959,6 @@ installListJob() {
         --conf spark.default.parallelism=${partitions} \
         --conf spark.kryoserializer.buffer.max=256m \
         --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-        --files ${HIVE_SITE_PATH} \
-        --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
         --master yarn --deploy-mode cluster --executor-memory ${exec_memory}g --driver-memory ${driver_memory}g  --executor-cores 2  --num-executors ${executors} \
         ${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce ${coalesce}
     if [[ $? -ne 0 ]];then

azkaban/dmp_event_tag/dmp_event_tag.sh

@@ -17,8 +17,6 @@ spark-submit --class mobvista.dmp.datasource.event_tag.DmpEventTag \
       --conf spark.sql.shuffle.partitions=600 \
       --conf spark.default.parallelism=600 \
       --conf spark.sql.broadcastTimeout=600 \
-      --files ${HIVE_SITE_PATH} \
-      --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
       --master yarn --deploy-mode cluster   --executor-cores 3 --executor-memory 6G --num-executors 20 \
       ../${JAR} \
       -today ${date} -output  ${OUTPUT_PATH}  -coalesce 600

azkaban/dmp_event_tag/dmp_event_tag_pre.sh

@@ -10,8 +10,6 @@ date_path=$(date +%Y/%m/%d -d "-1 day $today")
 spark-submit --class mobvista.dmp.datasource.newtag.CalInterestTag \
       --conf spark.network.timeout=720s \
       --conf spark.sql.broadcastTimeout=1200\
-      --files ${HIVE_SITE_PATH} \
-      --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
       --master yarn --deploy-mode cluster --executor-cores 2 --driver-memory 4g --executor-memory 6G  --num-executors 1 \
       ../${JAR} \
       -today ${date}

azkaban/dmp_event_tag/event_3s_postback_tag.sh

@@ -46,7 +46,6 @@ spark-submit --class mobvista.dmp.datasource.event_tag.PostBackEvent \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --conf spark.sql.adaptive.enabled=true \
  --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 4 --num-executors 50 \
  ../${JAR} -output ${OUTPUT_PATH}  -coalesce 200 \

azkaban/dmp_event_tag/event_3s_tag.sh

@@ -27,7 +27,6 @@ spark-submit --class mobvista.dmp.datasource.event_tag.Dmp3sEventTag \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
   --name dmp_3s_event_tag \
   --conf spark.akka.frameSize=100 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --driver-memory 4g --executor-cores 2 --executor-memory 4G --num-executors 20 \
   ../${JAR} \
   -today ${date} -input ${input_path} -output ${OUTPUT_PATH} -coalesce 100

azkaban/dmp_event_tag/event_ga_tag.sh

@@ -25,7 +25,6 @@ spark-submit --class mobvista.dmp.datasource.event_tag.DmpGaEventTag \
   --conf spark.shuffle.compress=true \
   --conf spark.shuffle.spill.compress=true \
   --name dmp_ga_event_tag \
-  --files ${HIVE_SITE_PATH} \
   --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/json-serde-1.3.7-jar-with-dependencies.jar \
   --master yarn --deploy-mode cluster --driver-memory 4g --executor-cores 2 --executor-memory 4G --num-executors 160 \
   ../${JAR} \

azkaban/dmp_event_tag/event_ss_tag.sh

@@ -36,7 +36,6 @@ spark-submit --class mobvista.dmp.datasource.event_tag.DmpSsEventTag \
       --conf spark.shuffle.consolidateFiles=true \
       --conf spark.dynamicAllocation.enabled=false \
       --conf spark.akka.frameSize=100 \
-      --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --driver-memory 4g --executor-cores 2 --executor-memory 4G --num-executors 20 \
      ../${JAR} \
       -today ${date}  -input "${input_path}/*/*" -output  ${OUTPUT_PATH}  -coalesce 100

azkaban/dsp/dsp_dc_interest.sh

@@ -39,7 +39,6 @@ hadoop fs -rm -r ${OUTPUT_PATH}
 spark-submit --class mobvista.dmp.datasource.dsp.dc.interest.DmDCInterestTagV2 \
  --conf spark.yarn.executor.memoryOverhead=3072 \
  --conf spark.sql.shuffle.partitions=500 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 5  --num-executors 40 \
  ../${JAR} -input $DSP_REQ_DAILY_PATH -output $OUTPUT_PATH -yestoday $old_date \
   -dictPath $SEGMENT_TAG_PATH -parallelism 500 -coalesce 500 -date $modify_date

azkaban/dsp/dsp_device_mapping.sh

@@ -20,7 +20,6 @@ spark-submit --class mobvista.dmp.datasource.dsp.DspDeviceIdMapping \
   --conf spark.sql.files.maxPartitionBytes=536870912 \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 5 --num-executors 100 \
   ../${JAR} -date ${dt} -output ${OUTPUT} -coalesce 100
 

azkaban/dsp/dsp_etl_daily_ext.sh

@@ -51,7 +51,6 @@ spark-submit --class mobvista.dmp.datasource.dsp.DspReqImeiDealDaily \
   --conf spark.sql.shuffle.partitions=1000 \
   --conf spark.default.parallelism=200 \
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-  --files ${HIVE_SITE_PATH} \
   --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
   --master yarn --deploy-mode cluster --name DspReqImeiDealDaily --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors 20 \
   ../${JAR} -output "$OUTPUT_PATH" -coalesce 200 \

azkaban/dsp/dsp_org_etl_daily.sh

@@ -20,7 +20,6 @@ hadoop fs -rm -r ${TMP_OUTPUT_PATH}
 spark-submit  --class mobvista.dmp.datasource.dsp.DspOrgEtlDailys \
  --conf spark.yarn.executor.memoryOverhead=3072 \
  --conf spark.sql.shuffle.partitions=10000 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 4  --num-executors 100 \
  ../${JAR} -input $ETL_DSP_REQ_ETL_HOURS_INPUT_PATH  \
  -output ${TMP_OUTPUT_PATH}  \

azkaban/dsp/dsp_org_etl_hours.sh

@@ -36,7 +36,6 @@ hadoop fs -rmr -r $MDS_DSP_REQ_HOURS_PATH
 spark-submit --class mobvista.dmp.datasource.dsp.DspOrgLogEtlHoursDemo \
   --conf spark.yarn.executor.memoryOverhead=2048 \
   --conf spark.sql.shuffle.partitions=2000 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 12g --driver-memory 6g --executor-cores 5 --num-executors 60 \
   ../${JAR} -yyyymmddhh $input_date -output $ETL_DSP_REQ_HOURS_PATH -outputmds $MDS_DSP_REQ_HOURS_PATH -coalesce 2000 || exit 1
 

azkaban/dsp/dsp_req_device_tag.sh

@@ -41,7 +41,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.shuffle.file.buffer.kb=128 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 3 --num-executors 256 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 5000

azkaban/dsp/dsp_req_device_tag_daily.sh

@@ -35,8 +35,6 @@ spark-submit --class mobvista.dmp.datasource.dsp.DspReqTagDaily \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.memory.offHeap.enabled=true \
      --conf spark.memory.offHeap.size=4294967296 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 150 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 200

azkaban/dsp/dsp_req_device_tag_v2.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.storage.memoryFraction=0.5 \
      --conf spark.shuffle.memoryFraction=0.3 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 200 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1000

azkaban/dsp/dsp_req_install_total_orc.sh

@@ -36,7 +36,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 18g --driver-memory 6g  --executor-cores 5 --num-executors 100 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 4000
 

azkaban/dsp/dsp_req_pkg_total_v1.sh

@@ -33,7 +33,6 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
   --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
   ../${JAR} \
   -date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}

azkaban/dsp/dsp_req_pkg_total_v2.sh

@@ -39,8 +39,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 180 \
      ../${JAR}
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/dsp/dsp_req_profile_total.sh

@@ -25,8 +25,6 @@ spark-submit --class mobvista.dmp.datasource.dsp.DspDeviceProfile \
     --conf spark.default.parallelism=2000 \
     --conf spark.kryoserializer.buffer.max=512m \
     --conf spark.kryoserializer.buffer=64m \
-    --files ${HIVE_SITE_PATH} \
-    --jars ${JARS} \
     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 4 --num-executors 100 \
     ../${JAR} \
     -date ${LOG_TIME} -output ${OUTPUT}

azkaban/dsp/dsp_req_unmatch_install_total_orc.sh

@@ -39,8 +39,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 100
 

azkaban/dsp/impre_info_compl/log_adn_dsp_click_impression_combine.sh

@@ -34,7 +34,6 @@ spark-submit --class  mobvista.dmp.datasource.dsp.DspClickImpressionCombineOrc \
     --conf spark.sql.broadcastTimeout=1200 \
 	 --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--executor-memory 8G \

azkaban/dsp/impre_info_compl/log_adn_dsp_impression_hour_combine.sh

@@ -75,7 +75,6 @@ spark-submit --class  mobvista.dmp.datasource.dsp.DspImpressionHourCombine \
 	 --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
 	--conf spark.yarn.executor.memoryOverhead=4096 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--executor-memory 10G \

azkaban/dsp/impre_info_compl/log_adn_dsp_impression_hour_full_cn.sh

@@ -42,7 +42,6 @@ spark-submit --class  mobvista.dmp.datasource.dsp.DspImpressionHourFullCn \
 	--conf spark.default.parallelism=6 \
 	--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
 	--conf spark.yarn.executor.memoryOverhead=1024 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--executor-memory 2G \

azkaban/dsp/impre_info_compl/log_adn_dsp_impression_hour_full_tk.sh

@@ -45,7 +45,6 @@ spark-submit --class  mobvista.dmp.datasource.dsp.DspImpressionHourFullOther \
 	--conf spark.default.parallelism=2000 \
 	--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
 	--conf spark.yarn.executor.memoryOverhead=2048 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--executor-memory 6G \

azkaban/dsp/impre_info_compl/log_adn_dsp_impression_hour_full_vg.sh

@@ -42,7 +42,6 @@ spark-submit --class  mobvista.dmp.datasource.dsp.DspImpressionHourFullOther \
 	--conf spark.default.parallelism=2500 \
 	--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
 	--conf spark.yarn.executor.memoryOverhead=2048 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--executor-memory 6G \

azkaban/dsp/log_adn_dsp_impression_hour_full.sh

@@ -39,7 +39,6 @@ spark-submit --class  mobvista.dmp.datasource.dsp.DspImpressionHourFull \
 	--conf spark.default.parallelism=6 \
 	--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
 	--conf spark.yarn.executor.memoryOverhead=1024 \
-	--files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--executor-memory 2G \

azkaban/dsp/tmp_extract_data_from_dsp_req.sh

@@ -21,7 +21,6 @@ hadoop fs -rm -r ${Tmp_Extract_Data_From_DspReq_Path}
 spark-submit  --class mobvista.dmp.datasource.dsp.TmpExtractDataFromDspReq \
  --conf spark.yarn.executor.memoryOverhead=3072 \
  --conf spark.sql.shuffle.partitions=10000 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 4  --num-executors 100 \
  ../${JAR} -input $ETL_DSP_REQ_ETL_HOURS_INPUT_PATH  \
  -output ${Tmp_Extract_Data_From_DspReq_Path}  \

azkaban/event_tag/Ga_purchase_event.sh

@@ -51,7 +51,5 @@ spark-submit --class mobvista.dmp.datasource.event_tag.Ga_purchase_event \
   --conf spark.shuffle.io.numConnectionsPerPeer=10 \
   --conf spark.shuffle.consolidateFiles=true \
   --conf spark.dynamicAllocation.enabled=false \
-  --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml \
-  --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
   --master yarn --deploy-mode cluster --conf spark.akka.frameSize=100 --num-executors 35 --executor-cores 2 --executor-memory 5G \
   ../${JAR}

azkaban/event_tag/behavior_thirdparty_datasource_manual_daily.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.behavior.ThirdPartySourceDaily \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=10 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
  --master yarn --deploy-mode cluster --name behavior_from_third_party_daily --executor-memory 2g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
  ../${JAR}  -outputtotal ${OUTPUT_TOTAL_PATH}  -coalesce 10 \

azkaban/event_tag/behavior_thirdparty_datasource_total.sh

@@ -28,7 +28,6 @@ spark-submit --class mobvista.dmp.datasource.behavior.ThirdPartySourceTotal \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=10 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
  --master yarn --deploy-mode cluster --name behavior_from_third_party_total --executor-memory 2g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
  ../${JAR}  -outputtotal ${OUTPUT_TOTAL_PATH} -dmpevent ${DMP_EVENT_TAG_PATH} -coalesce 10  \

azkaban/event_tag/event_tag.sh

@@ -51,7 +51,6 @@ echo "3ss_offer_event_spec.csv  file success exist, and then can start"
 spark-submit --class mobvista.dmp.datasource.event_tag.Event_tag \
   --master yarn --deploy-mode cluster \
   --conf spark.akka.frameSize=100 \
-  --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml \
   --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
   --conf spark.driver.memory=4G --queue root.dataplatform \
   --num-executors 40 --executor-cores 4 \

azkaban/event_tag/event_tag_source.sh

@@ -27,4 +27,4 @@ deal_time=$(date -d "$ScheduleTime 1 days ago" +"%Y%m%d")
 echo  "$deal_time"
 sleep 10s
 
-spark-submit --class mobvista.dmp.datasource.event_tag.Dmp_event_source --master yarn --deploy-mode cluster --conf spark.akka.frameSize=100 --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf spark.driver.memory=2G --queue root.dataplatform --num-executors 35 --executor-cores 2 --executor-memory 5G --conf spark.shuffle.file.buffer=256k --conf spark.reducer.maxSizeInFlight=100m --conf spark.shuffle.io.maxRetries=60  --conf spark.shuffle.io.retryWait=60s --conf spark.network.timeout=1200 --conf spark.shuffle.compress=true --conf spark.io.compression.codec=lz4 --conf spark.driver.maxResultSize=2024M --conf spark.shuffle.spill.compress=true --name event_tag_source_daily --conf spark.app.loadTime=${deal_time} --conf spark.app.coalesce_num=500 --conf spark.yarn.executor.memoryOverhead=3048 --conf spark.locality.wait=0 --conf spark.shuffle.io.numConnectionsPerPeer=10 --conf spark.shuffle.consolidateFiles=true --conf spark.dynamicAllocation.enabled=false ../DMP.jar
+spark-submit --class mobvista.dmp.datasource.event_tag.Dmp_event_source --master yarn --deploy-mode cluster --conf spark.akka.frameSize=100 --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf spark.driver.memory=2G --queue root.dataplatform --num-executors 35 --executor-cores 2 --executor-memory 5G --conf spark.shuffle.file.buffer=256k --conf spark.reducer.maxSizeInFlight=100m --conf spark.shuffle.io.maxRetries=60  --conf spark.shuffle.io.retryWait=60s --conf spark.network.timeout=1200 --conf spark.shuffle.compress=true --conf spark.io.compression.codec=lz4 --conf spark.driver.maxResultSize=2024M --conf spark.shuffle.spill.compress=true --name event_tag_source_daily --conf spark.app.loadTime=${deal_time} --conf spark.app.coalesce_num=500 --conf spark.yarn.executor.memoryOverhead=3048 --conf spark.locality.wait=0 --conf spark.shuffle.io.numConnectionsPerPeer=10 --conf spark.shuffle.consolidateFiles=true --conf spark.dynamicAllocation.enabled=false ../DMP.jar

azkaban/facebook/facebook_daily.sh

@@ -48,8 +48,6 @@ spark-submit --class mobvista.dmp.datasource.facebook.FaceBookDaily \
   --conf spark.network.timeout=720s \
   --conf spark.default.parallelism=500 \
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-  --files ${HIVE_SITE_PATH} \
-  --jars ${path_dir}/hive-hcatalog-core-2.3.3.jar \
   --master yarn --deploy-mode cluster --name facebook_daily_kehan --executor-memory 4g --driver-memory 4g --executor-cores 3 --num-executors 50 \
   ../${JAR} -input ${INPUT_PATH} -output ${OUTPUT_PATH} -unmatched ${UNMATCHED_OUTPUT_PATH} -coalesce 200 \
   -today ${dt_today} -yesterday ${dt_yesterday} -last_sunday ${dt_yesterday}

azkaban/facebook/facebook_device_tag.sh

@@ -33,8 +33,6 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=100 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 5 --num-executors 10 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10

azkaban/facebook/facebook_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.facebook.FaceBookTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 5

azkaban/facebook/facebook_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=4 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4

azkaban/facebook/facebook_install_total_orc.sh

@@ -34,8 +34,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=10 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 4
 

azkaban/facebook/facebook_total.sh

@@ -38,7 +38,6 @@ spark-submit --class mobvista.dmp.datasource.facebook.FaceBookTotal \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=10 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
  --master yarn --deploy-mode cluster --name facebook_device_total --executor-memory 2g --driver-memory 2g  --executor-cores 2  --num-executors 3 \
  ../${JAR}  -outputtotal ${OUTPUT_TOTAL_PATH} -outputgender ${GENDER_OUTPUT_PATH} -coalesce 10 \

azkaban/fmp/fmp_insight.sh

@@ -28,7 +28,6 @@ spark-submit --class mobvista.dmp.datasource.fmp.FmpInsightJob \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=134217728 \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 6 --num-executors 64 \
      ../${JAR} \
      -data ${data// /###} -output ${output_path} -coalesce 1

azkaban/ga/ga_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=100 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 40

azkaban/ga/ga_install_device_tag.sh

@@ -32,8 +32,6 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=200 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 20 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 200

azkaban/ga/ga_install_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.ga.GaTagDaily \
      --conf spark.sql.shuffle.partitions=15 \
      --conf spark.default.parallelism=5 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10

azkaban/ga/ga_install_total_orc.sh

@@ -35,8 +35,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 200
 

azkaban/ga/ga_install_total_v2.sh

@@ -34,8 +34,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 2  --num-executors 15 \
      ../${JAR}
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/ga/ga_prepare.sh

@@ -32,7 +32,5 @@ spark-submit --class mobvista.dmp.datasource.ga.GaParser \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.9 \
      --conf spark.speculation.multiplier=1.2 \
-     --files /data/azkaban-hadoop/command-home/spark-offline/conf/hive-site.xml \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 180 \
      ${JAR} -date_str_midline ${MID_TIME} -input ${INPUT_PATH} -output ${OUTPUT_PATH} -coalesce 2000
\ No newline at end of file

azkaban/gender/gender_thirdparty_datasource_manual_daily.sh

@@ -28,7 +28,6 @@ spark-submit --class mobvista.dmp.datasource.age_gender.ThirdPartySourceDaily \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=10 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
  --master yarn --deploy-mode cluster --name gender_from_third_party_daily --executor-memory 2g --driver-memory 2g  --executor-cores 2  --num-executors 2 \
  ../${JAR}  -outputtotal ${OUTPUT_TOTAL_PATH}  -coalesce 10 \

azkaban/gender/gender_thirdparty_datasource_total.sh

@@ -31,7 +31,6 @@ spark-submit --class mobvista.dmp.datasource.age_gender.ThirdPartySourceTotal \
  --conf spark.network.timeout=720s \
  --conf spark.default.parallelism=10 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
  --master yarn --deploy-mode cluster --name gender_from_third_party --executor-memory 2g --driver-memory 2g  --executor-cores 2  --num-executors 3 \
  ../${JAR}  -outputtotal ${OUTPUT_TOTAL_PATH} -outputgender ${GENDER_OUTPUT_PATH} -coalesce 10  \

azkaban/gender/merge_device_gender.sh

@@ -34,8 +34,6 @@ spark-submit --class mobvista.dmp.datasource.age_gender.MergeDeviceGenderLR \
      --conf spark.speculation=false \
      --conf spark.speculation.quantile=0.9 \
      --conf spark.speculation.multiplier=1.5 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 8g  --driver-memory 4g  --executor-cores 3  --num-executors 50 \
      ../${JAR} -date ${LOG_TIME}   -gender_predict_input ${GENDER_PRIDICT_SCORE_INPUT}  -gender_threshold_dict_input ${GDNEER_THRESHOLD_DICT_PATH}  -gender_device_output ${GENDER_DEVICE_OUTPUT} -parallelism 400
 

azkaban/gender/merge_install_gender.sh

@@ -49,7 +49,6 @@ spark-submit --class mobvista.dmp.datasource.age_gender.MergeInstallGender \
   --conf spark.sql.files.maxPartitionBytes=536870912 \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 5 --num-executors 60 \
   ../${JAR} -date ${LOG_TIME} \
   -ga_gender_path ${GA_GENDER_PATH} -dsp_gender_path ${DSP_GENDER_PATH} -fb_gender_path ${FB_GENDER_PATH} -tp_gender_path ${TP_GENDER_PATH} -gender_output ${GENDER_OUTPUT} -parallelism 2000

azkaban/gender/merge_install_gender_v2.sh

@@ -81,8 +81,6 @@ spark-submit --class mobvista.dmp.datasource.age_gender.MergeInstallGenderLR \
      --conf spark.shuffle.memoryFraction=0.4 \
      --conf spark.sql.files.maxPartitionBytes=134217728 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar,s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/json-serde-1.3.7-jar-with-dependencies.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g  --driver-memory 6g  --executor-cores 5  --num-executors 100 \
      ../${JAR} -date ${LOG_TIME}  -dt_yesterday ${dt_yesterday} -ga_date ${GA_INSTALL_DATE}  -other_date ${OTHER_DATE}  \
     -gender_output ${GENDER_OUTPUT}  -gender_bin_output ${GENDER_BIN_OUTPUT} -org_gender_bin_output ${ORG_OUTPUT_PATH}    -calc_gender_bin_output ${CALC_OUTPUT_PATH}  -parallelism 1000

azkaban/install_ruid/etl_ruid_mapping.sh

@@ -34,7 +34,6 @@ spark-submit --class mobvista.dmp.datasource.dm.EtlRuidMapping \
   --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
   ../${JAR} \
   -date ${date} -output $OUTPUT_PATH

azkaban/iqiyi/foractivation_qiyi_oppo.sh

@@ -21,7 +21,6 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.EtlOtherIqiyiActivationData \
  --conf spark.sql.shuffle.partitions=2000 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 12g --driver-memory 4g  --executor-cores 5  --num-executors 80 \
  ../${JAR}  \
  -output ${OUTPUT_PATH} \

azkaban/iqiyi/iqiyi_ck.sh

@@ -28,7 +28,6 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.IQiYiEtl2CK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/iqiyi/iqiyi_daily.sh

@@ -19,7 +19,6 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.IQiYiDaily \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files /data/hadoop-config/command-home/hdp-spark-2.3.1-offline/conf/hive-site.xml \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 10 \
      ../${JAR} -date ${LOG_TIME} -partNum 10 -output ${OUTPUT_PATH} -cluster 'cluster_1st'
 

azkaban/iqiyi/iqiyi_install_total_orc.sh

@@ -29,8 +29,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=200 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 40
 

azkaban/iqiyi/iqiyi_lahuo_ck.sh

@@ -26,7 +26,6 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.IQiYiLaHuoCK \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf hive.exec.orc.default.stripe.size=268435456 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../${JAR} -input ${INPUT} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 

azkaban/iqiyi/iqiyi_lahuo_daily.sh

@@ -32,7 +32,6 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.IQiYiLaHuoDaily \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 50 \
      ../${JAR} -date ${LOG_TIME} -days 7 -output ${OUTPUT_PATH}
 

azkaban/iqiyi/iqiyi_lahuo_df.sh

@@ -20,7 +20,6 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.IQiYiLaHuoDF \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files /data/hadoop-config/command-home/hdp-spark-2.3.1-offline/conf/hive-site.xml \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 10 \
      ../${JAR} -date ${LOG_TIME} -partNum 10 -output ${OUTPUT_PATH} -cluster 'cluster_1st'
 

azkaban/iqiyi/iqiyi_tmp_daily_data_to_dmp.sh

@@ -27,7 +27,6 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.IQiYiTmpDataToDMP \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 30 \
  ../${JAR}  -input ${INPUT} \
   -output ${OUTPUT}   \
@@ -52,7 +51,6 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.IQiYiLaHuoFourDaysDataDedupli
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
   -output ${FOUR_DAYS_OUTPUT}

azkaban/joypac/joypac_device_tag.sh

@@ -33,8 +33,6 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=10 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10

azkaban/joypac/joypac_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.joypac.JoypacTagDaily \
      --conf spark.sql.shuffle.partitions=5 \
      --conf spark.default.parallelism=5 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 3 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1

azkaban/joypac/joypac_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=8 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4

azkaban/joypac/joypac_install_total_orc.sh

@@ -34,8 +34,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=10 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 4
 

azkaban/joypac/joypac_result_all.sh

@@ -22,8 +22,6 @@ spark-submit --class mobvista.dmp.datasource.joypac.JoypacResultAll \
     --conf spark.default.parallelism=20 \
     --master yarn --deploy-mode cluster --name JoypacResutlAll \
     --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
-    --files ${HIVE_SITE_PATH} \
-    --jars ${JARS} \
     ../${JAR} -output ${OUTPUT_PATH} -coalesce 10 -date ${LOG_TIME}
 
 if [[ $? -ne 0 ]];then

azkaban/joypac/joypac_result_daily.sh

@@ -24,8 +24,6 @@ spark-submit --class mobvista.dmp.datasource.joypac.JoypacResultDaily \
     --conf spark.default.parallelism=20 \
     --master yarn --deploy-mode cluster --name JoypacResutlDaily \
     --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
-    --files ${HIVE_SITE_PATH} \
-    --jars ${JARS} \
     ../${JAR} -dict ${DICT_PATH} -output ${OUTPUT_PATH} -coalesce 10 -date ${LOG_TIME} -cluster ${cluster}
 
 if [[ $? -ne 0 ]];then

azkaban/joypac/joypac_result_etl.sh

@@ -30,7 +30,6 @@ spark-submit --class mobvista.dmp.datasource.joypac.JoypacResultEtl \
     --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
     --master yarn --deploy-mode cluster --name JoypacResutlEtl \
     --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
-    --jars ${JARS} \
     ../${JAR} -input ${INPUT_PATH} -output ${OUTPUT_PATH} -coalesce 10 -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
 
 if [[ $? -ne 0 ]];then

azkaban/joypac/joypac_user_info.sh

@@ -21,8 +21,6 @@ spark-submit --class mobvista.dmp.datasource.joypac.JoypacUserFeatureJob \
  --conf spark.kryoserializer.buffer.max=512m \
  --conf spark.kryoserializer.buffer=64m \
  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
- --files ${HIVE_SITE_PATH} \
- --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
  --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 100 \
  ../${JAR} -date ${LOG_TIME} -output ${OUTPUT_PATH} -dict_output ${DICT_PATH} -coalesce 200
 

azkaban/joypac/joypac_user_info_cassandra.sh

@@ -19,8 +19,6 @@ spark-submit --class mobvista.dmp.datasource.retargeting.UserFeatureCassandra \
     --conf spark.default.parallelism=1000 \
     --conf spark.yarn.executor.memoryOverhead=2048 \
     --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-    --files ${HIVE_SITE_PATH} \
-    --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2 --num-executors 3 \
     ../${JAR} -input ${INPUT_PATH} -region ${REGION}
 

azkaban/mp/mp_request_daily.sh

@@ -23,7 +23,6 @@ export HADOOP_CLIENT_OPTS="-Xmx4096m $HADOOP_CLIENT_OPTS"
 
 spark-submit --class mobvista.dmp.datasource.mpsdk.ParseMPSDKDaily \
   --conf spark.sql.shuffle.partitions=200 \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 20 \
  ../${JAR} -input $INPUT_PATH -output $OUTPUT_PATH -mappingPath $ID_MAPPING_PATH -parallelism 100 -coalesce 20
 

azkaban/mp/mp_request_device_tag.sh

@@ -33,8 +33,6 @@ hadoop fs -rmr ${store_output_path}
 
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
      --conf spark.sql.shuffle.partitions=1000 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 4 --num-executors 50 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1000

azkaban/mp/mp_request_device_tag_daily.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.mpsdk.MpSdkTagDaily \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10

azkaban/mp/mp_request_device_tag_v2.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
      --conf spark.default.parallelism=200 \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100

azkaban/mp/mp_request_install_list_v2.sh

@@ -35,8 +35,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.8 \
      --conf spark.speculation.multiplier=1.2 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 3  --num-executors 80 \
      ../${JAR}
      -output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}

azkaban/mp/mp_request_install_total_orc.sh

@@ -34,8 +34,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=400 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 25 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 200
 

azkaban/mparticle/mparticle_install_total_orc.sh

@@ -31,8 +31,6 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
      --conf spark.default.parallelism=200 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 10 \
      ../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT}  -coalesce 40
 

azkaban/output/reyun/user_info.sh

@@ -7,9 +7,9 @@
 source ../../dmp_env.sh
 
 dt=$(date +"%Y%m%d" -d "-1 day $ScheduleTime")
-date_path=$(date +"%Y%m/%d" -d "-1 day $ScheduleTime")
+date_path=$(date +"%Y/%m/%d" -d "-1 day $ScheduleTime")
 
-check_await ${ODS_DMP_USER_INFO_ALL}/${dt}/_SUCCESS
+check_await ${ODS_DMP_USER_INFO_ALL}_v2/${dt}/_SUCCESS
 
 OUTPUT_PATH="${OUTPUT_REYUN_USER_INFO_PATH}/${date_path}"
 

azkaban/package/get_package.sh

@@ -22,8 +22,6 @@ spark-submit --class mobvista.dmp.datasource.retargeting.GetPackage \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 5 --num-executors 128 \
      ../${JAR} \
      -date ${date}

azkaban/package/merge_pkg_tag.sh

@@ -24,8 +24,6 @@ spark-submit --class mobvista.dmp.datasource.apptag.MergeAppTagID \
      --conf spark.kryoserializer.buffer.max=512m \
      --conf spark.kryoserializer.buffer=64m \
      --conf dfs.socket.timeout=300000 \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores 2  --num-executors 2 \
      ../${JAR} -output ${output_path} -date ${date}
 

azkaban/package/package_mapping.sh

@@ -19,8 +19,6 @@ spark-submit --class mobvista.dmp.datasource.retargeting.PackageMapping \
      --conf spark.sql.shuffle.partitions=10 \
      --conf spark.default.parallelism=10 \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 1 --num-executors 1 \
      ../${JAR} \
      -output ${mount_path}

azkaban/realtime/cassandra_sink.sh

@@ -33,8 +33,6 @@ spark-submit --class mobvista.dmp.datasource.retargeting.UserFeatureCassandra \
      --conf spark.yarn.executor.memoryOverhead=2048 \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${JARS} \
      --master yarn --deploy-mode cluster --executor-memory 2g --driver-memory 2g --executor-cores 2 --num-executors ${cores} \
      ../${JAR} \
      -date ${date} -region ${region} -input ${ODS_USER_INFO_REGION_PATH}

azkaban/realtime/dm_realtime_service.sh

@@ -37,7 +37,6 @@ spark-submit --class mobvista.dmp.datasource.retargeting.DeviceInfoJob \
      --conf spark.sql.files.maxPartitionBytes=536870912 \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 8g --executor-cores 3 --num-executors 100 \
      ../${JAR} \
      -date ${date} -output ${output_path} -coalesce 2000

azkaban/realtime/dm_realtime_service_region.sh

@@ -34,7 +34,6 @@ spark-submit --class mobvista.dmp.datasource.retargeting.UserFeatureFilterJob \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 3 --num-executors 100 \
      ../${JAR} \
      -date ${date} -output ${output_path} -coalesce 1000 -days ${days}

azkaban/realtime/dmp_protal_job.sh

@@ -13,7 +13,6 @@ spark-submit --class mobvista.dmp.datasource.retargeting.RetargetingCassandra \
      --conf spark.yarn.executor.memoryOverhead=2048 \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 2g --driver-memory 2g  --executor-cores 2 --num-executors 2 \
      ../${JAR} -region ${region}
 

azkaban/realtime_v2/adn_etl_hour.sh

@@ -47,7 +47,6 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnRequestSdkHour \
     --conf spark.sql.shuffle.partitions=${partition} \
     --conf spark.kryoserializer.buffer.max=256m \
     --conf spark.sql.files.maxPartitionBytes=134217728 \
-    --files ${HIVE_SITE_PATH} \
     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores ${cores} --num-executors ${executors} \
     ../${JAR} -input ${INPUT_ADN_PATH} -output ${OUTPUT_PATH} -coalesce ${coalesce} -input_dict1 ${INPUT_MAPPING_PATH} -input_dict2 ${APPID_PACKAGE}
 

azkaban/realtime_v2/dsp_etl_hour.sh

@@ -57,7 +57,6 @@ spark-submit --class mobvista.dmp.datasource.dsp.DspEtlHour \
     --conf spark.sql.files.maxPartitionBytes=134217728 \
     --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
     --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-    --files ${HIVE_SITE_PATH} \
     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores ${cores} --num-executors ${executors} \
     ../${JAR} -input ${INPUT_DSP_PATH} -output ${OUTPUT_PATH} -detailOutPath ${detailOutPath} -coalesce ${coalesce}
 

azkaban/realtime_v2/etl_hour_ck.sh

@@ -51,7 +51,6 @@ spark-submit --class mobvista.dmp.clickhouse.realtime.MergeEtlHourToCK \
      --conf spark.sql.files.maxPartitionBytes=268435456 \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g  --executor-cores 2  --num-executors ${executors} \
      ../${JAR} -date ${date} -host ${host} -cluster ${cluster} -database ${database} -table ${table} \
      --input_dsp ${INPUT_DSP_PATH} --input_adn ${INPUT_ADN_PATH} --region ${region} --hour ${hour} --app_tag_input ${INPUT_APP_TAG_PATH}

azkaban/realtime_v2/realtime_hour_cassandra_sink.sh

@@ -36,7 +36,6 @@ spark-submit --class mobvista.dmp.clickhouse.realtime.ReadFromCKWriteCS \
      --conf spark.yarn.executor.memoryOverhead=2048 \
      --conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 2g --executor-cores 2 --num-executors ${cores} \
      ../${JAR} \
      -date ${date} -hour ${hour} -region ${region} -host ${host} -cluster ${cluster} -database ${database} -table ${table} --output ${S3_OUTPUT} -part ${partition}

azkaban/retarget_dealerid/dsp_dealer_daily.sh

@@ -13,7 +13,6 @@ hadoop fs -rm -r $ETL_DSP_DEALERID_RETARGET_DAILY_PATH
 spark-submit --class  mobvista.dmp.datasource.dsp.DspDealeridRetarget \
  --conf spark.yarn.executor.memoryOverhead=2048 \
  --conf spark.sql.shuffle.partitions=2000 \
- --files ${HIVE_SITE_PATH} \
  --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 5  --num-executors 100 \
  ../${JAR} -yyyymmdd $dt  -output $ETL_DSP_DEALERID_RETARGET_DAILY_PATH  -appFile $ETL_DSP_DEALERID_PKG_MAPPING -coalesce 2000     || exit 1
 

azkaban/retarget_dealerid/dsp_dealer_device_tag.sh

@@ -28,8 +28,6 @@ hadoop fs -rmr ${store_output_path}
 spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
   --conf spark.yarn.executor.memoryOverhead=2048  \
      --conf spark.sql.shuffle.partitions=1000 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g  --executor-cores 5 --num-executors 40 \
      ../${JAR} \
      -date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1000

azkaban/rtdmp/device_info_calc.sh

@@ -20,7 +20,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.DeviceInfoCalc \
      --conf spark.default.parallelism=4000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 4  --num-executors 100 \
      ../${JAR} -date ${date} -output ${OUTPUT}
 

azkaban/rtdmp/device_region_calc.sh

@@ -24,8 +24,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.DeviceRegionCalc \
      --conf spark.default.parallelism=200 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -date ${date} -output ${OUTPUT}
 

azkaban/rtdmp/device_region_merge.sh

@@ -18,8 +18,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.DeviceRegionMerge \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -date ${date} -output ${OUTPUT} -input ${INPUT} -coalesce 200
 

azkaban/rtdmp/device_region_result.sh

@@ -15,8 +15,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.DeviceRegionResult \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -date ${date} -output ${OUTPUT}
 

azkaban/rtdmp/device_region_write.sh

@@ -16,7 +16,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.DeviceRegionWrite \
      --conf spark.default.parallelism=1000 \
      --conf spark.kryoserializer.buffer.max=256m \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 4 \
      ../${JAR} -input ${INPUT}
 

azkaban/rtdmp/lazada/etl_job.sh

@@ -29,7 +29,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.lazada.ETLJob \
   --conf spark.default.parallelism=100 \
   --conf spark.kryoserializer.buffer.max=256m \
   --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 4 --num-executors 5 \
   ../../${JAR} -dt ${date} -output ${OUTPUT} -tb_type ${tb_type}
 

azkaban/rtdmp/lazada/lazada_rtdmp.sh

@@ -16,7 +16,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.lazada.ProcessRTJob \
   --conf spark.default.parallelism=4000 \
   --conf spark.kryoserializer.buffer.max=256m \
   --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 4 --num-executors 5 \
   ../../${JAR} -dt ${date} -output ${OUTPUT}
 

azkaban/rtdmp/lazada/merge_install.sh

@@ -22,7 +22,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.lazada.MergeInstallJob \
   --conf spark.default.parallelism=4000 \
   --conf spark.kryoserializer.buffer.max=256m \
   --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-  --files ${HIVE_SITE_PATH} \
   --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 4 --num-executors 5 \
   ../../${JAR} -dt ${date} -output ${OUTPUT}
 

azkaban/rtdmp/rtdmp_as.sh

@@ -24,7 +24,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.RTDmpAS \
      --conf spark.speculation.quantile=0.9 \
      --conf spark.speculation.multiplier=1.3 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2 --num-executors 20 \
      ../${JAR} -input_audience ${INPUT_AUDIENCE} -input_data ${INPUT_DATA} -output ${OUTPUT} -coalesce 100 -time "${date_time}"
 

azkaban/rtdmp/rtdmp_merge.sh

@@ -49,8 +49,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.RTDmpMerge \
      --conf spark.speculation.quantile=0.9 \
      --conf spark.speculation.multiplier=1.3 \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-     --files ${HIVE_SITE_PATH} \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 18g --driver-memory 6g  --executor-cores 5 --num-executors 20 \
      ../${JAR} -date_time "${curr_time}" -old_time "${old_time}" -input ${INPUT} -output ${OUTPUT} -partition 100
 

azkaban/rtdmp/rtdmp_repair.sh

@@ -21,7 +21,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.RTDmpMain \
      --conf spark.speculation=true \
      --conf spark.speculation.quantile=0.9 \
      --conf spark.speculation.multiplier=1.3 \
-     --jars ${SPARK_HOME}/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g  --executor-cores 3 --num-executors 20 \
      ../${JAR} -time "${date_time}" -data_utime "${date_time}" -output ${OUTPUT} -coalesce 100 -partition 2000
 

azkaban/rtdmp/rtdmp_request.sh

@@ -96,8 +96,6 @@ spark-submit --class mobvista.dmp.datasource.rtdmp.RTDmpRequest \
   --conf spark.kryoserializer.buffer.max=256m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
-  --files ${HIVE_SITE_PATH} \
-  --jars ${JARS} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors ${executors} \
   ../${JAR} -date "${date}" -hh "0${hh}" -output ${OUTPUT} -coalesce ${coalesce} -business ${business} -table ${table}
 

azkaban/rtdmp/rtdmp_request_iqiyi_adx.sh

@@ -25,8 +25,6 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.RTDmpIQiYiRequest \
   --conf spark.kryoserializer.buffer.max=256m \
   --conf spark.sql.adaptive.enabled=true \
   --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
-  --files ${HIVE_SITE_PATH} \
-  --jars ${JARS} \
   --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 100 \
   ../${JAR} -date ${date} -output ${OUTPUT}
 

azkaban/setting/appid_package.sh

@@ -37,8 +37,6 @@ spark-submit --class mobvista.dmp.datasource.setting.SettingTotal \
   --conf spark.network.timeout=720s \
   --conf spark.default.parallelism=30 \
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
-  --files ${HIVE_SITE_PATH} \
-  --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
   --master yarn --deploy-mode cluster --name  apps_flyer_total --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 5 \
   ../${JAR}  -outputtotal ${APP_ID_MAPPING_TMP}  \
   -coalesce 30 \

azkaban/statistics/dm_device_tag_statistics.sh

@@ -35,7 +35,6 @@ spark-submit --class mobvista.dmp.datasource.dm.DmDeviceTagStatistics \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --conf spark.sql.adaptive.enabled=true \
      --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
-     --files ${HIVE_SITE_PATH} \
      --master yarn --deploy-mode cluster --executor-memory 18g --driver-memory 4g  --executor-cores 5  --num-executors 60 \
      ../${JAR} \
      -output ${mount_path} -date ${date} -coalesce 500

azkaban/toutiao/dm_toutiao_launch_total.sh

@@ -25,7 +25,6 @@ hadoop fs -rm -r ${OUTPUT_PATH}
 spark-submit --class mobvista.dmp.datasource.toutiao.DmToutiaoTotal \
     --conf spark.sql.shuffle.partitions=${reduce_num} \
     --conf spark.default.parallelism=200 \
-    --files ${HIVE_SITE_PATH} \
     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4 --num-executors 50 \
     ../${JAR} -output $OUTPUT_PATH -date ${dt} -yestoday ${old_date} -updateDate ${update_date}
 if [ $? -ne 0 ];then

azkaban/toutiao/toutiao_dmp_device_tag.sh

@@ -21,8 +21,6 @@ spark-submit --class mobvista.dmp.datasource.toutiao.GameDeviceToutiao \
      --conf spark.yarn.executor.memoryOverhead=2048 \
      --conf spark.sql.shuffle.partitions=4000 \
      --conf spark.default.parallelism=300 \
-     --files ${HIVE_SITE_PATH} \
-     --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
      --master yarn --deploy-mode cluster --executor-memory 12g --driver-memory 6g  --executor-cores 4  --num-executors 120 \
      ${JAR} \
      -date ${date} -output ${output_path} -startDate ${startDate} -endDate ${endDate} -file ${file}

azkaban/toutiao/toutiao_launch_device_tag.sh

@@ -31,7 +31,6 @@ hadoop fs -rm -r ${output_path}
 spark-submit --class mobvista.dmp.datasource.toutiao.DmToutiaoDeviceTag \
     --conf spark.sql.shuffle.partitions=${reduce_num} \
     --conf spark.default.parallelism=200 \
-    --files ${HIVE_SITE_PATH} \
     --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g  --executor-cores 4 --num-executors 50 \
     ../${JAR} -date "$dt" -output ${output_path}
 if [ $? -ne 0 ];then

azkaban/userinfo/ods_dmp_user_info_all.sh

@@ -98,7 +98,6 @@ spark-submit --class  mobvista.dmp.datasource.device.OdsDmpUserInfoAll \
     --conf spark.sql.files.maxPartitionBytes=268435456 \
     --conf spark.sql.adaptive.enabled=true \
     --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
-    --files ${HIVE_SITE_PATH} \
     --master yarn \
 	  --deploy-mode cluster \
 	  --executor-memory 10G \

azkaban/userinfo/ods_dmp_user_info_all_v2.sh

@@ -34,7 +34,6 @@ spark-submit --class  mobvista.dmp.datasource.device.OdsDmpUserInfoAllV2 \
     --conf spark.shuffle.memoryFraction=0.4 \
     --conf spark.sql.adaptive.enabled=true \
     --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
-    --files ${HIVE_SITE_PATH} \
     --master yarn \
 	  --deploy-mode cluster \
 	  --executor-memory 12G \

azkaban/userinfo/ods_dmp_user_info_daily.sh

@@ -121,7 +121,6 @@ spark-submit --class  mobvista.dmp.datasource.device.OdsDmpUserInfoDailyV2 \
     --conf spark.sql.adaptive.enabled=true \
     --conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
     --conf spark.sql.autoBroadcastJoinThreshold=-1 \
-    --files ${HIVE_SITE_PATH} \
     --master yarn \
 	  --deploy-mode cluster \
 	  --executor-memory 12G \

azkaban/userinfo/ods_dmp_user_info_daily_v2.sh

@@ -116,14 +116,12 @@ spark-submit --class  mobvista.dmp.datasource.device.OdsDmpUserInfoDailyV3 \
 	--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
 	--conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
     --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
-    --files ${HIVE_SITE_PATH} \
 	--master yarn \
 	--deploy-mode cluster \
 	--executor-memory 10G \
 	--driver-memory 6G \
 	--executor-cores 4 \
 	--num-executors 180 \
-	--jars ${JARS} \
 	../${JAR}  \
     -cur_day ${date} -gender_date ${GET_GENDER_DATE} -output ${OUTPUT_PATH} -coalesce 2000
 

src/main/scala/mobvista/dmp/output/reyun/Constant.scala

@@ -37,7 +37,7 @@ object Constant {
   val user_info =
     """
       |SELECT dev_id, dev_id_md5, LOWER(dev_type) dev_type, LOWER(platform) platform, UPPER(country) country, install, interest, update_date
-      |  FROM dwh.ods_dmp_user_info_all
+      |  FROM dwh.ods_dmp_user_info_all_v2
       |  WHERE dt = '@date' AND UPPER(country) = 'CN'
       |""".stripMargin
 }