增大 driver-memory，避免出现 driver pod OOMKilled

azkaban/adn/package/adn_request_other_device_tag.sh

@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
       --conf spark.yarn.executor.memoryOverhead=3072 \
      --files ${HIVE_SITE_PATH} \
      --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 5g  --executor-cores 2  --num-executors 80 \
+     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 80 \
      ../../${JAR} \
      -date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000
 

azkaban/adn/package/adn_request_other_install.sh

@@ -28,7 +28,7 @@ spark-submit --class mobvista.dmp.datasource.adn_request_other.AdnRequestOtherIn
      --conf spark.yarn.executor.memoryOverhead=2048 \
      --conf spark.sql.shuffle.partitions=2000 \
      --files ${HIVE_SITE_PATH} \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 3g  --executor-cores 2  --num-executors 200 \
+     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 200 \
      ../../${JAR} \
      -input "${INPUT_PATH}" -output $OUTPUT_PATH -date $date -oldInput $OLD_INPUT_PATH -parallelism 2000 -coalesce 2000
 if [ $? -ne 0 ]; then

azkaban/adn_adx/adn_tencent_adx_device_tag.sh

@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.adn_adx.AdnAdxDeviceTag \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
- --master yarn --deploy-mode cluster --name  AdnAdxDeviceTag --executor-memory 4g --driver-memory 2g  --executor-cores 2  --num-executors 32 \
+ --master yarn --deploy-mode cluster --name  AdnAdxDeviceTag --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 32 \
  ../${JAR}  -outputadxdevtag ${OUTPUT_ADN_ADX_DEVICE_TAG_PATH} \
  -coalesce 80 \
  -today ${dt_today} -yesterday ${dt_yesterday}

azkaban/age/get_ga_all.sh

@@ -60,7 +60,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.GetAgeGender \
      --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
      --files ${HIVE_SITE_PATH} \
      --jars ${JARS} \
-     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 2g  --executor-cores 2  --num-executors 50 \
+     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
      ../${JAR} -ageOutput ${AGE_OUTPUT_PATH} -genderOutput ${GENDER_OUTPUT_PATH} -date ${GA_TOTAL_DATE} -business ${business}
 
 if [[ $? -ne 0 ]];then

azkaban/ali/TO/TO_daily.sh

@@ -28,7 +28,7 @@ spark-submit --class mobvista.dmp.datasource.TO.TODaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 20 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 20 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 200  -dt_dash_today ${dt_dash_today}
 

azkaban/ali/ali_userinfo_activation_daily_all_job/ali_ios_userinfo_activation_daily.sh

@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliIosActivitionDaily \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 9g --driver-memory 3g  --executor-cores 4  --num-executors 60 \
+ --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
  ../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OUTPUT_DAILY_PATH} -coalesce 500 \
  -yesterday ${yesterday} -today ${dt_today}  -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \
  -request_count_result "${IOS_REQUEST_COUNT_RESULT}/${dt_today}"  -last_req_day ${last_req_day}

azkaban/ali/ali_userinfo_activation_daily_all_job/ali_oaid_userinfo_activation_daily.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliOaidActivitionDaily \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 9g --driver-memory 3g  --executor-cores 4  --num-executors 60 \
+ --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
  ../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OAID_OUTPUT_DAILY_PATH} -coalesce 500 \
  -yesterday ${yesterday} -today ${dt_today}  -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \
  -request_count_result "${OAID_REQUEST_COUNT_RESULT}/${dt_today}" -last_req_day ${last_req_day}

azkaban/ali/ali_userinfo_activation_daily_all_job/ali_userinfo_activation_daily.sh

@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionDaily \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 9g --driver-memory 3g  --executor-cores 4  --num-executors 60 \
+ --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 60 \
  ../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OUTPUT_DAILY_PATH} -coalesce 500 \
  -yesterday ${yesterday} -today ${dt_today}  -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \
  -request_count_result "${REQUEST_COUNT_RESULT}/${dt_today}" -last_req_day ${last_req_day}

azkaban/ali/ali_userinfo_postback_activation_daily.sh

@@ -85,7 +85,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDail
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 3  --num-executors 60 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 3  --num-executors 60 \
  ../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT}  -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 50 \
  -today ${dt_today} -update_date ${dt_dash_today}  \
  -dt_dash_rec15day ${dt_dash_rec15day} -syn_to_3s ${ALI_USER_ACTIVATION_SYS_TO3S_PATH} -syn_3s_day ${syn_3s_day}

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_etl_postback_daily.sh

@@ -62,7 +62,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDail
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 6  --num-executors 70 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 70 \
  ../../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT}  -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 300 \
  -today ${dt_today} -update_date ${dt_dash_today}  -dt_taobao_postback_day ${dt_taobao_postback_day}\
  -dt_dash_rec15day ${dt_dash_rec15day} -syn_to_3s ${ALI_USER_ACTIVATION_SYS_TO3S_PATH} -syn_3s_day ${syn_3s_day}

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_extract_h_18_from_dsp_req.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlH18FromDmInstallListV2 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 60 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 60 \
  ../../${JAR}  \
  -h18_imei ${IMEI_H_18_GUANGDIANTONG_RES_PATH} -h18_imeimd5 ${IMEIMD5_H_18_GUANGDIANTONG_RES_PATH}  \
  -dt_oneday_ago ${dt_oneday_ago}

azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_insert_other_data_to_dmp.sh

@@ -44,7 +44,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlOtherDataFromPostBackDail
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 150 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 150 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_daily.sh

@@ -38,7 +38,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDaily \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 6  --num-executors 120 ../../${JAR} \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 120 ../../${JAR} \
  -imeioutput "${ALIPAY_IMEIMD5_OUTPUT_PATH}"  \
  -today ${dt_today}  -last_req_day ${last_req_day} -dt_after_one_day ${dt_after_one_day}\
  -input_one_day ${INPUT_ONE_DAY}  -input_two_day ${INPUT_TWO_DAY}  -input_three_day ${INPUT_THREE_DAY} \

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp.sh

@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
 
@@ -64,7 +64,7 @@ fi
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
 # --files ${HIVE_SITE_PATH} \
-# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}
 #

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_02.sh

@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
 
@@ -64,7 +64,7 @@ fi
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
 # --files ${HIVE_SITE_PATH} \
-# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}
 #

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_03.sh

@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
 
@@ -64,7 +64,7 @@ fi
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
 # --files ${HIVE_SITE_PATH} \
-# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}
 #

azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_04.sh

@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH}  -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
   -output01 ${OUTPUT01}  -output02 ${OUTPUT02}
 
@@ -64,7 +64,7 @@ fi
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
 # --files ${HIVE_SITE_PATH} \
-# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
 # ../../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
 #  -ActivationOutput ${ACTIVATIONOUTPUT}  -AcquisitionOutput ${ACQUISITIONOUTPUT}
 #

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 140 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_02.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 140 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_03.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 140 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_04.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 140 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 140 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}  -hour ${hour}

azkaban/ali/etl_dealid_hour.sh

@@ -37,7 +37,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlDealidDaily \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 9g --driver-memory 3g  --executor-cores 6  --num-executors 30 \
+ --master yarn --deploy-mode cluster  --executor-memory 10g --driver-memory 4g  --executor-cores 6  --num-executors 30 \
  ../${JAR}  -dt_dash_today ${dt_dash_today} \
   -oppooutput ${OPPO_OUTPUT}   \
   -inmobioutput ${INMOBI_OUTPUT}

azkaban/ali/etl_lazada_data_daily.sh

@@ -60,7 +60,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlLazadaActivitionDaily \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 90 ../${JAR} \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 90 ../${JAR} \
  -gaidoutput "${GAID_OUTPUT_PATH}"   \
  -today ${dt_today}  \
  -input_one_day ${INPUT_ONE_DAY}  -input_two_day ${INPUT_TWO_DAY}  -input_three_day ${INPUT_THREE_DAY} \
@@ -105,7 +105,7 @@ fi
 # --conf spark.yarn.executor.memoryOverhead=4096 \
 # --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
 # --files ${HIVE_SITE_PATH} \
-# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 70 ../${JAR} \
+# --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 70 ../${JAR} \
 # -gaidoutput "${GAID_OUTPUT_PATH}" -gaidinput "${GAID_INPUT_PATH}"  -newoutput "${NEW_OUTPUT_PATH}" \
 # -today ${dt_today}  -dt_30days_ago ${dt_30days_ago}
 #

azkaban/ali/other_single_jobs/etl_com_tencent_news_daily.sh

@@ -18,7 +18,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlComTencentNewsDaily \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 120 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 6g  --executor-cores 6  --num-executors 120 \
  ../../${JAR}  \
  -output ${OUTPUT_PATH} -coalesce 500 \
  -dt_today ${dt_today}  -dt_dash_rec7day  ${dt_dash_rec7day}  -dt_dash_rec15day  ${dt_dash_rec15day}

azkaban/ali/uc_lahuo/uc_imei_lahuo_request.job

 type=command
-dependencies=uc_imei_lahuo_ck,uc_oaid_lahuo_request
+dependencies=uc_imei_lahuo_ck
 command=sh -x uc_imei_lahuo_request.sh

azkaban/ali/uc_lahuo/uc_lahuo_daily.sh

@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCLaHuoDaily \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 80 ../../${JAR} \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 80 ../../${JAR} \
  -imeioutput "${UC_IMEIMD5_OUTPUT_PATH}"  -oaidoutput "${UC_OAIDMD5_OUTPUT_PATH}"  \
  -today ${dt_today}  -last_req_day ${last_req_day}
 

azkaban/ali/uc_lahuo/uc_lahuo_data_to_dmp.sh

@@ -41,7 +41,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCTmpDataToDMP \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -imeiRequestInput ${UC_IMEIMD5_REQUEST_INPUT_PATH}  -oaidRequestInput ${UC_OAIDMD5_REQUEST_INPUT_PATH} \
   -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH}  -oaidResponseInput ${OAIDMD5_RESPONSE_INPUT_PATH}   \
   -imeiOutput ${IMEIMD5_OUTPUT}  -oaidOutput ${OAIDMD5_OUTPUT}   \

azkaban/ali/uc_lahuo/uc_lahuo_df.job

 type=command
-dependencies=uc_imei_lahuo_request
+dependencies=uc_imei_lahuo_request,uc_oaid_lahuo_request
 command=sh -x uc_lahuo_df.sh

azkaban/ali/uc_lahuo/uc_other_data_to_dmp.sh

@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.UCOtherDataToDmp \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 150 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 150 \
  ../../${JAR}  \
  -output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02}  \
  -output03 ${OUTPUT_PATH03} -output04 ${OUTPUT_PATH04}  \

azkaban/ali/uc_lahuo_to_guangdiantong/ali_extract_h_32_from_dsp_req.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlH32FromDmInstallListV2 \
  --conf spark.sql.broadcastTimeout=1200 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 2g  --executor-cores 6  --num-executors 60 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 6  --num-executors 60 \
  ../../${JAR}  \
  -h32_imei ${IMEI_H_32_GUANGDIANTONG_RES_PATH} -h32_imeimd5 ${IMEIMD5_H_32_GUANGDIANTONG_RES_PATH}  \
  -dt_oneday_ago ${dt_oneday_ago}

azkaban/ali/youku_laxin/youku_laxin_daily.sh

@@ -36,7 +36,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.YOUKULaXinDaily \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 80 ../../${JAR} \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 80 ../../${JAR} \
  -imeioutput "${YOUKU_IMEIMD5_OUTPUT_PATH}"  -oaidoutput "${YOUKU_OAIDMD5_OUTPUT_PATH}"  \
  -input_one_day ${INPUT_ONE_DAY}  -input_two_day ${INPUT_TWO_DAY}  -input_three_day ${INPUT_THREE_DAY} \
  -oaid_input_one_day ${OAID_INPUT_ONE_DAY}  -oaid_input_two_day ${OAID_INPUT_TWO_DAY}  -oaid_input_three_day ${OAID_INPUT_THREE_DAY} \

azkaban/ali/youku_laxin/youku_laxin_data_to_dmp.sh

@@ -26,7 +26,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.YoukuTmpDataToDmp \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -Input ${INPUT_PATH}   -Output ${OUTPUT_PATH}   \
   -update ${update}
 
@@ -54,7 +54,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.YoukuLaXinPollingDataDedupli
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../../${JAR}  -dt_today ${dt_today}  -dt_begin_days ${dt_begin_days} \
  -AcquisitionOutput ${ACQUISITIONOUTPUT}
 

azkaban/app_info/package_black_list.sh

@@ -21,7 +21,7 @@ EXPIRE_PATH="${PACKAGE_BLACK_LIST}/$expire_path"
 spark-submit --class mobvista.dmp.main.PackageBlackList \
      --conf spark.sql.shuffle.partitions=10 \
      --files ${HIVE_SITE_PATH} \
-     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 2g  --executor-cores 2  --num-executors 5 \
+     --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 5 \
      ../${JAR} \
      -date "${yes_date}" \
      -iosDailyPath "${TMP_IOS_APP_INFO_PATH}" -adrDailyPath "${TMP_ADR_APP_INFO_PATH}" \

azkaban/appsflyer/appsflyer_total.sh

@@ -64,7 +64,7 @@ $HIVE_CMD	-v  -hivevar  dt_today ${dt_today}     -hivevar update_date ${dt_today
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
   --files ${HIVE_SITE_PATH} \
   --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
-  --master yarn --deploy-mode cluster --name  apps_flyer_total --executor-memory 4g --driver-memory 2g  --executor-cores 3  --num-executors 5 \
+  --master yarn --deploy-mode cluster --name  apps_flyer_total --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 5 \
   ../${JAR}  -outputtotal ${OUTPUT_TOTAL_PATH} -dmpuserinfo ${DMP_USER_INFO_OUTPUT_PATH} \
   -coalesce 20 \
   -today ${dt_today}  -update_date ${dt_today_dash}

azkaban/bundle_match/bundle_match.sh

@@ -23,7 +23,7 @@ if [ $? -eq 0 ];then
   hadoop fs -rm -r $OUTPUT_PATH
 
   # find unmanned
-  spark-submit --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 3g  --executor-cores 2 --num-executors 15 \
+  spark-submit --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2 --num-executors 15 \
   --conf spark.yarn.executor.memoryOverhead=2048M \
   --class mobvista.dmp.main.FindUnmatchBundle ../${JAR} -input $INPUT_PATH  -output $OUTPUT_PATH
   if [ $? -ne 0 ];then

azkaban/clever/adn_clever_daily.sh

@@ -24,7 +24,7 @@ hadoop fs -rm -r "$OUTPUT_PATH/"
 
 spark-submit --class mobvista.dmp.datasource.clever.ParseCleverDaily \
  --conf spark.yarn.executor.memoryOverhead=2048  --conf spark.network.timeout=720s \
- --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 3g  --executor-cores 2  --num-executors 40 \
+ --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 40 \
  ../${JAR} -input $INPUT_PATH -output $OUTPUT_PATH -parallelism 100 -coalesce 20
 if [ $? -ne 0 ];then
   exit 255

azkaban/clever/adn_clever_install.sh

@@ -26,7 +26,7 @@ hadoop fs -rm -r "$OUTPUT_PATH/"
 
 spark-submit --class mobvista.dmp.datasource.clever.CleverInstallList \
  --conf spark.yarn.executor.memoryOverhead=2048  --conf spark.network.timeout=720s --conf spark.app.tag=-1 \
- --master yarn --deploy-mode cluster --executor-memory 5g --driver-memory 3g  --executor-cores 2  --num-executors 20 \
+ --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
  ../${JAR} -input $INPUT_PATH -oldInput $OLD_INPUT_PATH -output $OUTPUT_PATH -date $dt -parallelism 200 -coalesce 20
 if [ $? -ne 0 ];then
   exit 255

azkaban/dm/pseudo_package_to_other_business/Age_Package_Names.sh

@@ -27,7 +27,7 @@ spark-submit --class mobvista.dmp.datasource.dm.AgePackageNames \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 5g --driver-memory 5g  --executor-cores 2 --num-executors 220 \
+ --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 2 --num-executors 220 \
  ../../${JAR} -dt_today ${dt_today}   -update ${update}  -Age_Package_Names ${Age_Package_Names}   -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 
 

azkaban/dm/pseudo_package_to_other_business/Canglan_Package_Names.sh

@@ -27,7 +27,7 @@ spark-submit --class mobvista.dmp.datasource.dm.CanglanPackageNames \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 5g --driver-memory 5g  --executor-cores 2 --num-executors 220 \
+ --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 2 --num-executors 220 \
  ../../${JAR} -dt_today ${dt_today}   -update ${update}  -Canglan_Package_Names ${Canglan_Package_Names}   -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 
 

azkaban/dm/pseudo_package_to_other_business/Three_Kingdoms_Game.sh

@@ -27,7 +27,7 @@ spark-submit --class mobvista.dmp.datasource.dm.ThreeKingdomsGame \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 5g --driver-memory 5g  --executor-cores 2 --num-executors 220 \
+ --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 2 --num-executors 220 \
  ../../${JAR} -dt_today ${dt_today}   -update ${update}  -package_names_input ${Three_Kingdoms_Package_Names}   -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 
 

azkaban/dm/pseudo_package_to_other_business/shinny.sh

@@ -28,7 +28,7 @@ spark-submit --class mobvista.dmp.datasource.dm.ShinnyPackageNames \
  --conf spark.sql.shuffle.partitions=3000 \
  --conf spark.network.timeout=720s \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 5g --driver-memory 5g  --executor-cores 2 --num-executors 220 \
+ --master yarn --deploy-mode cluster  --executor-memory 6g --driver-memory 4g  --executor-cores 2 --num-executors 220 \
  ../../${JAR} -dt_today ${dt_today}   -update ${update}  -Shinny_Package_Names ${Shinny_Package_Names}   -output01 ${OUTPUT_PATH01}  -output02 ${OUTPUT_PATH02} -coalesce 200
 
 

azkaban/dmp_env.sh

@@ -825,7 +825,7 @@ userInfoJob() {
      --conf spark.speculation.multiplier=1 \
      --jars ${JARS} \
      --files ${HIVE_SITE_PATH} \
-     --master yarn --deploy-mode cluster --executor-memory 5g --driver-memory 3g  --executor-cores 2  --num-executors 20 \
+     --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 20 \
      ${jar} -date $LOG_TIME -dailyPath $dailyPath -agePath $agePath -genderPath $genderPath \
      -dailyFormat ${dailyFormat} -dailyDidIndex $dailyDidIndex -dailyDidTypeIndex $dailyDidTypeIndex -dailyPltIndex $dailyPltIndex -dailyCountryIndex $dailyCountryIndex \
      -outputPath $outputPath -parallelism ${parallelism} -coalesce ${coalesce}
@@ -874,7 +874,7 @@ userInfoJob_dsp_req() {
      --conf spark.speculation.multiplier=1 \
      --jars ${JARS} \
      --files ${HIVE_SITE_PATH} \
-     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 3g  --executor-cores 4  --num-executors 80 \
+     --master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g  --executor-cores 4  --num-executors 80 \
      ${jar} -date $LOG_TIME -dailyPath $dailyPath -agePath $agePath -genderPath $genderPath \
      -dailyFormat ${dailyFormat} -dailyDidIndex $dailyDidIndex -dailyDidTypeIndex $dailyDidTypeIndex -dailyPltIndex $dailyPltIndex -dailyCountryIndex $dailyCountryIndex \
      -outputPath $outputPath -parallelism ${parallelism} -coalesce ${coalesce}

azkaban/dsp/mds_dmp_address_daily_dsp.sh

@@ -22,7 +22,7 @@ hadoop fs -rm -r $OUTPUT_PATH
 spark-submit --class mobvista.dmp.datasource.address.AddressInfoTotal \
  --conf spark.yarn.executor.memoryOverhead=1024 --conf spark.network.timeout=720s \
  --conf spark.sql.shuffle.partitions=300 \
- --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 3g  --executor-cores 2  --num-executors 100 \
+ --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g  --executor-cores 2  --num-executors 100 \
  ../${JAR}  -input ${INPUT_PATH} -output ${OUTPUT_PATH} -dailyFormat "text"  -parallelism 200 -coalesce 20 \
  -indices "0,2,3,4,5,6,7"
 

azkaban/event_tag/behavior_thirdparty_datasource_manual_daily.sh

@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.behavior.ThirdPartySourceDaily \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
- --master yarn --deploy-mode cluster --name behavior_from_third_party_daily --executor-memory 2g --driver-memory 2g  --executor-cores 2  --num-executors 2 \
+ --master yarn --deploy-mode cluster --name behavior_from_third_party_daily --executor-memory 2g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
  ../${JAR}  -outputtotal ${OUTPUT_TOTAL_PATH}  -coalesce 10 \
  -today ${dt_today} -yesbef3 ${dt_yes_bef3}
 

azkaban/event_tag/behavior_thirdparty_datasource_total.sh

@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.behavior.ThirdPartySourceTotal \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
  --jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
- --master yarn --deploy-mode cluster --name behavior_from_third_party_total --executor-memory 2g --driver-memory 2g  --executor-cores 2  --num-executors 2 \
+ --master yarn --deploy-mode cluster --name behavior_from_third_party_total --executor-memory 2g --driver-memory 4g  --executor-cores 2  --num-executors 2 \
  ../${JAR}  -outputtotal ${OUTPUT_TOTAL_PATH} -dmpevent ${DMP_EVENT_TAG_PATH} -coalesce 10  \
  -yesterday ${dt_yesterday}
 

azkaban/iqiyi/iqiyi_lahuo_request.sh

@@ -37,7 +37,7 @@ if [[ $? -ne 0 ]]; then
   exit 255
 fi
 
-sleep $((fors * 20))
+sleep $((fors * 25))
 
 shell=" -cp /root/workspace/DMP-1.0.3-jar-with-dependencies.jar mobvista.dmp.datasource.iqiyi.IQiYiRequest"
 

azkaban/iqiyi/iqiyi_tmp_daily_data_to_dmp.sh

@@ -28,7 +28,7 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.IQiYiTmpDataToDMP \
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 30 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 30 \
  ../${JAR}  -input ${INPUT} \
   -output ${OUTPUT}   \
   -update ${update}
@@ -53,7 +53,7 @@ spark-submit --class mobvista.dmp.datasource.iqiyi.IQiYiLaHuoFourDaysDataDedupli
  --conf spark.yarn.executor.memoryOverhead=4096 \
  --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
  --files ${HIVE_SITE_PATH} \
- --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 3g  --executor-cores 4  --num-executors 40 \
+ --master yarn --deploy-mode cluster  --executor-memory 8g --driver-memory 4g  --executor-cores 4  --num-executors 40 \
  ../${JAR}  -dt_today ${dt_today}  -dt_three_days_ago ${dt_three_days_ago} \
   -output ${FOUR_DAYS_OUTPUT}
 

azkaban/setting/appid_package.sh

@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.setting.SettingTotal \
   --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
   --files ${HIVE_SITE_PATH} \
   --jars /data/hadoop-alternative/hive/auxlib/Common-SerDe-1.0-SNAPSHOT.jar \
-  --master yarn --deploy-mode cluster --name  apps_flyer_total --executor-memory 4g --driver-memory 2g  --executor-cores 3  --num-executors 5 \
+  --master yarn --deploy-mode cluster --name  apps_flyer_total --executor-memory 4g --driver-memory 4g  --executor-cores 3  --num-executors 5 \
   ../${JAR}  -outputtotal ${APP_ID_MAPPING_TMP}  \
   -coalesce 30 \
   -today ${LOG_TIME}

report/dsp_and_m/dmp_market_second.sh

@@ -28,7 +28,7 @@ installPath="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dm_install_li
 outputPath="s3://mob-emr-test/feng.liang/report/activeInstall"
 
 spark-submit --master yarn  --executor-cores 2 --executor-memory 4g \
---class mobvista.prd.main.ActiveInstall --driver-memory 3g ../${JAR} \
+--class mobvista.prd.main.ActiveInstall --driver-memory 4g ../${JAR} \
 $date $appTagPath $installPath $outputPath
 
 if [ $? -ne 0 ];then

report/public/weightGame.sh

@@ -12,7 +12,7 @@ while [ $startDay -le $stopDay ];do
   outputPath="s3://mob-emr-test/feng.liang/weightGame/$date_path"
   hadoop fs -rm -r $outputPath
   spark-submit --class mobvista.prd.main.WeightGame \
-  --master yarn  --executor-memory 6g --driver-memory 3g  --executor-cores 2  --num-executors 50 \
+  --master yarn  --executor-memory 6g --driver-memory 4g  --executor-cores 2  --num-executors 50 \
   ../DMP.jar "$inputPath" "$installPath" "$outputPath"
   if [ $? -ne 0 ];then
     echo "$startDay fail"