Skip to content

M
mobvista-dmp
Commit 1d221fa6 by WangJinfeng
Options

update dmp spark-submit

parent a58447ad
Showing with 111 additions and 112 deletions
azkaban/3s/3s_install_device_tag.sh
......@@ -32,7 +32,7 @@ hadoop fs -rmr $store_output_path
spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
--conf spark.sql.shuffle.partitions=20 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 4 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 4 \
../${JAR} \
-date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 20
if [ $? -ne 0 ]; then
......
azkaban/3s/3s_install_device_tag_daily.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.tracking_3s.TrackingTagDaily \
--conf spark.sql.shuffle.partitions=5 \
--conf spark.default.parallelism=5 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 3 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 3 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2
......
azkaban/3s/3s_install_device_tag_v2.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
--conf spark.default.parallelism=8 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 2 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 2 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4
......
azkaban/3s/3s_tracking_install_total_orc.sh
......@@ -40,7 +40,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
--conf spark.default.parallelism=20 \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 10
if [[ $? -ne 0 ]];then
......
azkaban/3s_postback/3s_postback_daily.sh
......@@ -41,7 +41,7 @@ spark-submit --class mobvista.dmp.datasource.postback_3s.PostBackDaily \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 3 --num-executors 100 \
--deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 3 --num-executors 100 \
../${JAR} -output ${OUTPUT_PATH} -coalesce 100 \
-today ${dt_today} -last_sunday ${dt_yesterday}
......
azkaban/3s_postback/3s_postback_device_tag.sh
......@@ -28,7 +28,7 @@ hadoop fs -rmr ${store_output_path}
spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.sql.shuffle.partitions=1000 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 3 --num-executors 40 \
--deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 3 --num-executors 40 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1000
......
azkaban/3s_postback/3s_postback_device_tag_daily.sh
......@@ -28,7 +28,7 @@ spark-submit --class mobvista.dmp.datasource.postback_3s.PostBackTagDaily \
--conf spark.sql.shuffle.partitions=10 \
--conf spark.default.parallelism=10 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 5
......
azkaban/3s_postback/3s_postback_device_tag_v2.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
--conf spark.default.parallelism=100 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 5 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 5 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 40
......
azkaban/3s_postback/3s_postback_install_list.sh
......@@ -24,7 +24,7 @@ spark-submit --class mobvista.dmp.datasource.postback_3s.PostBackInstallList \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.network.timeout=720s \
--conf spark.default.parallelism=1000 \
--master yarn --deploy-mode cluster --name PostBackInstallList --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 20 \
--deploy-mode cluster --name PostBackInstallList --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 20 \
../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 1000 -coalesce 400
......
azkaban/3s_postback/3s_postback_install_total_orc.sh
......@@ -39,9 +39,8 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
--conf spark.sql.shuffle.partitions=1000 \
--conf spark.default.parallelism=1000 \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 25 \
../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 200
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 400
if [[ $? -ne 0 ]];then
exit 255
......
azkaban/ad_server_campaign/merge_campaign_list.sh
......@@ -48,7 +48,7 @@ spark-submit --class mobvista.dmp.datasource.packagelist.MergeCampaignList \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 4 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 4 \
../${JAR} -input_dmp_data_adn ${INPUT_DMP_DATA_ADN} -input_campaign_adn ${INPUT_CAMPAIGN_ADN} -coalesce 20 \
-output ${CAMPAIGN_TAG_OUTPUT} -today ${yester_year}-${yester_month}-${yester_day} || exit 1
......
azkaban/adn/address/mds_dmp_address_daily_adn.sh
......@@ -22,7 +22,7 @@ hadoop fs -rm -r $OUTPUT_PATH
spark-submit --class mobvista.dmp.datasource.address.AddressInfoTotal \
--conf spark.yarn.executor.memoryOverhead=1024 --conf spark.network.timeout=720s \
--conf spark.sql.shuffle.partitions=200 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 40 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 40 \
../../${JAR} -input ${INPUT_PATH} -output ${OUTPUT_PATH} -dailyFormat "rcfile" -parallelism 200 -coalesce 20 \
-indices "0,2,3,5,6"
......
azkaban/adn/adn_device_ids_daily.sh
......@@ -25,15 +25,15 @@ OUTPUT_PATH="$DMP_ADN_REQUEST_DEVICE_IDS/$year/$month/$day/adn_request"
## REDUCE_NUM=$(calculate_reduce_num "$INPUT_ADN_DSP_PATH")
hadoop fs -rm -r ${OUTPUT_PATH}/*
export HADOOP_CLIENT_OPTS="-Xmx2096m $HADOOP_CLIENT_OPTS"
export HADOOP_CLIENT_OPTS="-Xmx4096m $HADOOP_CLIENT_OPTS"
hadoop jar ../${JAR} mobvista.dmp.datasource.adn.mapreduce.AdnDeviceIdsMR \
-Dmapreduce.job.reduces=1000 \
-Dmapred.child.java.opts=-Xmx4096m \
-Dmapreduce.map.memory.mb=8192 \
-Dmapreduce.map.java.opts=-Xmx4096m \
-Dmapreduce.reduce.memory.mb=8192 \
-Dmapreduce.reduce.java.opts=-Xmx4096m \
-Dmapreduce.job.reduces=2000 \
-Dmapred.child.java.opts=-Xmx8192m \
-Dmapreduce.map.memory.mb=10240 \
-Dmapreduce.map.java.opts=-Xmx8192m \
-Dmapreduce.reduce.memory.mb=10240 \
-Dmapreduce.reduce.java.opts=-Xmx8192m \
-Dtask.date="$year-$month-$day" \
-Dmapreduce.fileoutputcommitter.algorithm.version=2 \
"$INPUT_ADN_DSP_PATH" "$OUTPUT_PATH" || exit 1
......
azkaban/adn/device/adn_org_etl_hours_frankfurt.sh
......@@ -24,7 +24,7 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
--conf spark.yarn.executor.memoryOverhead=3072 \
--conf spark.sql.shuffle.partitions=2000 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
--deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 200 -region frankfurt || exit 1
if [[ $? -ne 0 ]]; then
......
azkaban/adn/device/adn_org_etl_hours_seoul.sh
......@@ -24,7 +24,7 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
--conf spark.yarn.executor.memoryOverhead=3072 \
--conf spark.sql.shuffle.partitions=2000 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
--deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 200 -region seoul || exit 1
if [[ $? -ne 0 ]]; then
......
azkaban/adn/device/adn_org_etl_hours_singapore.sh
......@@ -23,7 +23,7 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
--conf spark.yarn.executor.memoryOverhead=3072 \
--conf spark.sql.shuffle.partitions=2000 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 50 \
--deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 50 \
../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 400 -region singapore || exit 1
if [[ $? -ne 0 ]]; then
......
azkaban/adn/device/adn_org_etl_hours_virginia.sh
......@@ -24,7 +24,7 @@ hadoop fs -rm -r $ETL_ADN_REQ_ORG_HOURS_PATH
spark-submit --class mobvista.dmp.datasource.adn.AdnOrgLogEtlHours \
--conf spark.yarn.executor.memoryOverhead=3072 \
--conf spark.sql.shuffle.partitions=2000 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
--deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 4 --num-executors 20 \
../../${JAR} -datetime "$yt$mt$dt$hhpath" -output $ETL_ADN_REQ_ORG_HOURS_PATH -coalesce 200 -region virginia || exit 1
if [[ $? -ne 0 ]]; then
......
azkaban/adn/package/adn_click_daily.sh
......@@ -21,7 +21,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnClickDaily \
--conf spark.default.parallelism=1000 \
--conf spark.kryoserializer.buffer.max=512m \
--conf spark.kryoserializer.buffer=64m \
--master yarn --deploy-mode cluster \
--deploy-mode cluster \
--executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 50 \
../.././DMP.jar \
-datetime ${LOG_TIME} -output ${OUTPUT_ADN_INSTALL_DAILY} -coalesce 200
......
azkaban/adn/package/adn_install_click.sh
......@@ -21,7 +21,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnClickJoinInstallDaily \
--conf spark.default.parallelism=2000 \
--conf spark.kryoserializer.buffer.max=512m \
--conf spark.kryoserializer.buffer=64m \
--master yarn --deploy-mode cluster \
--deploy-mode cluster \
--executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 50 \
../.././DMP.jar \
-input_click ${INPUT_CLICK_PATH} -input_install ${INPUT_ADN_INSTALL_PATH} -output ${OUTPUT_ADN_INSTALL_DAILY} -coalesce 200
......
azkaban/adn/package/adn_install_daily.sh
......@@ -28,7 +28,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnInstallDaily \
--conf spark.default.parallelism=100 \
--conf spark.kryoserializer.buffer.max=512m \
--conf spark.kryoserializer.buffer=64m \
--master yarn --deploy-mode cluster \
--deploy-mode cluster \
--executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
../.././DMP.jar \
-datetime ${LOG_TIME} -output ${TMP_OUTPUT_ADN_INSTALL_DAILY} -coalesce 20
......
azkaban/adn/package/adn_install_device_tag.sh
......@@ -32,7 +32,7 @@ hadoop fs -rmr ${store_output_path}
spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
--conf spark.sql.shuffle.partitions=20 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 5 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 5 \
../../${JAR} \
-date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 20
......
azkaban/adn/package/adn_install_device_tag_daily.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnInstallTagDaily \
--conf spark.sql.shuffle.partitions=50 \
--conf spark.default.parallelism=10 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 4 --num-executors 5 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 4 --num-executors 5 \
../../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 1
......
azkaban/adn/package/adn_install_device_tag_v2.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
--conf spark.default.parallelism=8 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 2 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 2 \
../../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 4
......
azkaban/adn/package/adn_install_total_orc.sh
......@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
--conf spark.sql.shuffle.partitions=1000 \
--conf spark.default.parallelism=1000 \
--conf spark.kryoserializer.buffer.max=256m \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 20 \
--deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 20 \
../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 100
if [[ $? -ne 0 ]];then
......
azkaban/adn/package/adn_install_total_v1.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
--conf spark.kryoserializer.buffer=64m \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
../../${JAR} \
-date ${date} -coalesce 200 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
......
azkaban/adn/package/adn_install_total_v2.sh
......@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
--conf spark.speculation=true \
--conf spark.speculation.quantile=0.8 \
--conf spark.speculation.multiplier=1.2 \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 10 \
--deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 10 \
../../${JAR} \
-output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
'
......
azkaban/adn/package/adn_pre_click_daily.sh
......@@ -21,7 +21,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnPreClickDaily \
--conf spark.default.parallelism=1000 \
--conf spark.kryoserializer.buffer.max=512m \
--conf spark.kryoserializer.buffer=64m \
--master yarn --deploy-mode cluster \
--deploy-mode cluster \
--executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 50 \
../.././DMP.jar \
-datetime ${LOG_TIME} -output ${OUTPUT_ADN_INSTALL_DAILY} -coalesce 200
......
azkaban/adn/package/adn_request_daily_v2.sh
......@@ -32,7 +32,7 @@ spark-submit --class mobvista.dmp.datasource.adn_request_sdk.AdnRequestSdkEtlDai
--conf spark.storage.memoryFraction=0.4 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
--master yarn --deploy-mode cluster --name "AdnRequestSdkEtlDaily.${LOG_TIME}" --executor-memory 10g --driver-memory 6g --executor-cores 5 --num-executors 100 \
--deploy-mode cluster --name "AdnRequestSdkEtlDaily.${LOG_TIME}" --executor-memory 10g --driver-memory 6g --executor-cores 5 --num-executors 100 \
../../${JAR} -appIdMapping ${INPUT_MAPPING_PATH} -manualAppIdMapping ${DIM_MANUAL_MAPPING} -output ${TMP_OUTPUT_PATH} -date ${LOG_TIME} -coalesce 2000
if [[ $? -ne 0 ]]; then
......
azkaban/adn/package/adn_request_device_tag.sh
......@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.sql.shuffle.partitions=4000 \
--conf spark.default.parallelism=4000 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 100 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 100 \
../../${JAR} \
-date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 400
if [ $? -ne 0 ]; then
......
azkaban/adn/package/adn_request_device_tag_daily.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn.AdnRequestSdkTagDaily \
--conf spark.sql.shuffle.partitions=20 \
--conf spark.default.parallelism=5 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 2 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 2 \
../../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 20
......
azkaban/adn/package/adn_request_device_tag_v2.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
--conf spark.default.parallelism=200 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 10 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 10 \
../../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100
......
azkaban/adn/package/adn_request_install_total_orc.sh
......@@ -37,7 +37,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
--conf spark.sql.files.maxPartitionBytes=536870912 \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 3 --num-executors 200 \
--deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 3 --num-executors 200 \
../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 2000
if [[ $? -ne 0 ]];then
......
azkaban/adn/package/adn_request_other_daily.sh
......@@ -32,7 +32,7 @@ spark-submit --class mobvista.dmp.datasource.adn_request_other.EtlAdnRequestOthe
--conf spark.storage.memoryFraction=0.4 \
--conf spark.driver.maxResultSize=5g \
--conf spark.executor.extraJavaOptions=-XX:+UseG1GC \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 40 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 40 \
../../${JAR} -input "${INPUT_PATH}/*/*" -output $OUTPUT_PATH
if [ $? -ne 0 ]; then
exit 255
......
azkaban/adn/package/adn_request_other_device_tag.sh
......@@ -33,7 +33,7 @@ hadoop fs -rmr ${store_output_path}
spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
--conf spark.sql.shuffle.partitions=2000 \
--conf spark.yarn.executor.memoryOverhead=3072 \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 80 \
--deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 80 \
../../${JAR} \
-date $date -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000
......
azkaban/adn/package/adn_request_other_device_tag_daily.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn_request_other.AdnRequestOtherTa
--conf spark.sql.shuffle.partitions=20 \
--conf spark.default.parallelism=10 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
../../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10
......
azkaban/adn/package/adn_request_other_device_tag_v2.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
--conf spark.default.parallelism=200 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 20 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 20 \
../../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100
......
azkaban/adn/package/adn_request_other_install.sh
......@@ -27,7 +27,7 @@ hadoop fs -rm -r "$OUTPUT_PATH"
spark-submit --class mobvista.dmp.datasource.adn_request_other.AdnRequestOtherInstall \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.sql.shuffle.partitions=2000 \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 200 \
--deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 200 \
../../${JAR} \
-input "${INPUT_PATH}" -output $OUTPUT_PATH -date $date -oldInput $OLD_INPUT_PATH -parallelism 2000 -coalesce 2000
if [ $? -ne 0 ]; then
......
azkaban/adn/package/adn_request_other_install_total_orc.sh
......@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
--conf spark.default.parallelism=1000 \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 50 \
--deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 50 \
../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 400
if [[ $? -ne 0 ]];then
......
azkaban/adn/package/adn_request_other_install_v1.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
--conf spark.kryoserializer.buffer=64m \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
../../${JAR} \
-date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
......
azkaban/adn/package/adn_request_other_install_v2.sh
......@@ -38,7 +38,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
--conf spark.speculation=true \
--conf spark.speculation.quantile=0.8 \
--conf spark.speculation.multiplier=1.2 \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 \
--deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 30 \
../../${JAR} \
-output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
'
......
azkaban/adn/package/adn_request_pkg_total_v1.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
--conf spark.kryoserializer.buffer=64m \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
../../${JAR} \
-date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
......
azkaban/adn/package/adn_request_pkg_total_v2.sh
......@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
--conf spark.speculation=true \
--conf spark.speculation.quantile=0.8 \
--conf spark.speculation.multiplier=1.2 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 2 --num-executors 40 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 2 --num-executors 40 \
../../${JAR} \
-output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
'
......
azkaban/adn/package/adn_request_unmatch_install_total_orc.sh
......@@ -32,7 +32,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
--conf spark.default.parallelism=4000 \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC " \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
../../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 2000
if [[ $? -ne 0 ]];then
......
azkaban/adn_adx/adn_tencent_adx_device_tag.sh
......@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.adn_adx.AdnAdxDeviceTag \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--conf spark.sql.hive.verifyPartitionPath=true \
--jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar \
--master yarn --deploy-mode cluster --name AdnAdxDeviceTag --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 32 \
--deploy-mode cluster --name AdnAdxDeviceTag --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 32 \
../${JAR} -outputadxdevtag ${OUTPUT_ADN_ADX_DEVICE_TAG_PATH} \
-coalesce 80 \
-today ${dt_today} -yesterday ${dt_yesterday}
......
azkaban/adn_adx/adn_tencent_adx_package.sh
......@@ -36,7 +36,7 @@ spark-submit --class mobvista.dmp.datasource.adn_adx.AdnTecentAdxDataMidWay \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--conf spark.sql.hive.verifyPartitionPath=true \
--jars s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/hive-hcatalog-core-2.3.3.jar,s3://mob-emr-test/dataplatform/DataWareHouse/offline/myjar/json-serde-1.3.7-jar-with-dependencies.jar \
--master yarn --deploy-mode cluster --name AdnTecentAdxDataMidWay --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
--deploy-mode cluster --name AdnTecentAdxDataMidWay --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 100 \
../${JAR} -outputadxtmp ${OUTPUT_ODS_ADX_TMP_PATH} -dimadxpkg ${OUTPUT_DIM_ADN_ADX_PKG_PATH} \
-coalesce 60 \
-today ${dt_today}
......
azkaban/adn_sdk/adn_sdk_daily.sh
......@@ -30,7 +30,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkDaily \
--conf spark.default.parallelism=2000 \
--conf spark.driver.maxResultSize=8g \
--conf spark.app.coalesce=60000 \
--master yarn --deploy-mode cluster --name adn_sdk_daily --executor-memory 10g --driver-memory 6g --executor-cores 3 --num-executors 300 \
--deploy-mode cluster --name adn_sdk_daily --executor-memory 10g --driver-memory 6g --executor-cores 3 --num-executors 300 \
../${JAR}
if [[ $? -ne 0 ]];then
......
azkaban/adn_sdk/adn_sdk_device_tag.sh
......@@ -34,7 +34,7 @@ hadoop fs -rmr ${store_output_path}
spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
--conf spark.sql.shuffle.partitions=2000 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 3 --num-executors 200 \
--deploy-mode cluster --executor-memory 8g --driver-memory 6g --executor-cores 3 --num-executors 200 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000
......
azkaban/adn_sdk/adn_sdk_device_tag_daily.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkTagDaily \
--conf spark.sql.shuffle.partitions=10 \
--conf spark.default.parallelism=10 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 10
......
azkaban/adn_sdk/adn_sdk_device_tag_v2.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
--conf spark.default.parallelism=1000 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 50 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 50 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100
......
azkaban/adn_sdk/adn_sdk_etl_hour.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkEtlHour \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.driver.maxResultSize=8g \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --name adn_sdk_etl_hour.${date}.${hour} --executor-memory 4g --driver-memory 4g --executor-cores 4 --num-executors 20 \
--deploy-mode cluster --name adn_sdk_etl_hour.${date}.${hour} --executor-memory 4g --driver-memory 4g --executor-cores 4 --num-executors 20 \
../${JAR} -coalesce 4000 -date ${date} -hour ${hour} -input ${INPUT_PATH} -output ${OUTPUT_PATH}
if [[ $? -ne 0 ]];then
......
azkaban/adn_sdk/adn_sdk_install.sh
......@@ -38,7 +38,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkInstallList \
--conf spark.shuffle.memoryFraction=0.4 \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
--master yarn --deploy-mode cluster --name adn_sdk_install --executor-memory 15g --driver-memory 4g --executor-cores 5 --num-executors 60 \
--deploy-mode cluster --name adn_sdk_install --executor-memory 15g --driver-memory 4g --executor-cores 5 --num-executors 60 \
../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 2000 -coalesce 2000
if [[ $? -ne 0 ]];then
......
azkaban/adn_sdk/adn_sdk_install_total_orc.sh
......@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
--conf spark.default.parallelism=4000 \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 100 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 100 \
../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 2000
if [[ $? -ne 0 ]];then
......
azkaban/adn_sdk/adn_sdk_install_v1.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
--conf spark.kryoserializer.buffer=64m \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 80 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 80 \
../${JAR} \
-date ${date} -coalesce 1000 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
......
azkaban/adn_sdk/adn_sdk_install_v2.sh
......@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
--conf spark.speculation=true \
--conf spark.speculation.quantile=0.8 \
--conf spark.speculation.multiplier=1.2 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors 80 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors 80 \
../${JAR} \
-output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
'
......
azkaban/adn_sdk/adn_sdk_v2_device_tag.sh
......@@ -34,7 +34,7 @@ hadoop fs -rmr ${store_output_path}
spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
--conf spark.sql.shuffle.partitions=2000 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 150 \
--deploy-mode cluster --executor-memory 10g --driver-memory 6g --executor-cores 4 --num-executors 150 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 2000
......
azkaban/adn_sdk/adn_sdk_v2_device_tag_v2.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
--conf spark.default.parallelism=100 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 20 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 20 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 100
......
azkaban/adn_sdk/adn_sdk_v2_install.sh
......@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.adn_sdk.AdnSdkInstallList \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=268435456 \
--conf spark.app.version=1 \
--master yarn --deploy-mode cluster --name "adn_sdk_v2_install.${LOG_TIME}" --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 50 \
--deploy-mode cluster --name "adn_sdk_v2_install.${LOG_TIME}" --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 50 \
../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 200 -coalesce 200
if [[ $? -ne 0 ]];then
......
azkaban/adn_sdk/adn_sdk_v2_install_total_orc.sh
......@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
--conf spark.default.parallelism=500 \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 50 \
--deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 50 \
../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 200
if [[ $? -ne 0 ]];then
......
azkaban/adn_sdk/adn_sdk_v2_install_v1.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.dm.FixInstallListRuid \
--conf spark.kryoserializer.buffer=64m \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 40 \
../${JAR} \
-date ${date} -coalesce 200 -output $OUTPUT_PATH -business ${BUSINESS} -input ${INPUT_MAPPING}
......
azkaban/adn_sdk/adn_sdk_v2_install_v2.sh
......@@ -43,7 +43,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
--conf spark.speculation=true \
--conf spark.speculation.quantile=0.8 \
--conf spark.speculation.multiplier=1.2 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors 80 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors 80 \
../${JAR} \
-output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
'
......
azkaban/age/calc_device_age.sh
......@@ -35,7 +35,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.CalcDeviceAge \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.4 \
--conf spark.sql.files.maxPartitionBytes=134217728 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 100 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 100 \
../${JAR} -date ${LOG_TIME} -merge_input ${INPUT_PATH} -dict_input ${FILEDICT} -output ${OUTPUT_PATH} -parallelism 500
if [[ $? -ne 0 ]];then
......
azkaban/age/calc_package_dict_age.sh
......@@ -27,7 +27,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.CalcPackageAge \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.4 \
--conf spark.sql.files.maxPartitionBytes=134217728 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 100 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 100 \
../${JAR} -inputPath ${INPUT_PATH} -outputPath ${OUTPUT_PATH} -parallelism 1 -low ${LOW} -high ${HIGH} -unbelievable ${UNBELIEVABLE}
if [[ $? -ne 0 ]];then
......
azkaban/age/get_dsp_all.sh
......@@ -36,7 +36,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.GetAgeGender \
--conf spark.sql.shuffle.partitions=2000 \
--conf spark.default.parallelism=2000 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 60 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 60 \
../${JAR} -ageOutput ${AGE_OUTPUT_PATH} -genderOutput ${GENDER_OUTPUT_PATH} -date ${LOG_TIME} -business ${business}
if [[ $? -ne 0 ]];then
......
azkaban/age/get_ga_all.sh
......@@ -58,7 +58,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.GetAgeGender \
--conf spark.speculation.quantile=0.9 \
--conf spark.speculation.multiplier=1.5 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 50 \
--deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 2 --num-executors 50 \
../${JAR} -ageOutput ${AGE_OUTPUT_PATH} -genderOutput ${GENDER_OUTPUT_PATH} -date ${GA_TOTAL_DATE} -business ${business}
if [[ $? -ne 0 ]];then
......
azkaban/age/merge_install_age.sh
......@@ -52,7 +52,7 @@ spark-submit --class mobvista.dmp.datasource.age_gender.MergeInstallAge \
--conf spark.sql.files.maxPartitionBytes=536870912 \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=536870912 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 60 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 5 --num-executors 60 \
../${JAR} -date ${LOG_TIME} \
-ga_age_path ${GA_AGE_PATH} -dsp_age_path ${DSP_AGE_PATH} -age_output ${OUTPUT_PATH} -parallelism 2000
......
azkaban/ali/TO/TO_daily.sh
......@@ -27,7 +27,7 @@ spark-submit --class mobvista.dmp.datasource.TO.TODaily \
--conf spark.sql.shuffle.partitions=3000 \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 20 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 20 \
../../${JAR} \
-output ${OUTPUT_PATH} -coalesce 200 -dt_dash_today ${dt_dash_today}
......
azkaban/ali/TO/TO_install_list.sh
......@@ -23,7 +23,7 @@ spark-submit --class mobvista.dmp.datasource.TO.TOInstallList \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.network.timeout=720s \
--conf spark.default.parallelism=100 \
--master yarn --deploy-mode cluster --name PostBackInstallList --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 30 \
--deploy-mode cluster --name PostBackInstallList --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 30 \
../../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 160 -coalesce 160
......
azkaban/ali/ali_ck.sh
......@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.baichuan.BaiChuanJob \
--conf spark.default.parallelism=100 \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g --executor-cores 2 --num-executors 20 \
--deploy-mode cluster --executor-memory 6g --driver-memory 6g --executor-cores 2 --num-executors 20 \
../${JAR} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
if [[ $? -ne 0 ]];then
......
azkaban/ali/ali_daily.sh
......@@ -24,7 +24,7 @@ spark-submit --class mobvista.dmp.datasource.baichuan.AliDaily \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 3 --num-executors 5 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 3 --num-executors 5 \
../${JAR} -date ${LOG_TIME} -partNum 10 -output ${OUTPUT_PATH} -host ${host} -cluster ${cluster} -database ${database} -table ${table}
if [[ $? -ne 0 ]];then
......
azkaban/ali/ali_device_tag.sh
......@@ -36,7 +36,7 @@ hadoop fs -rmr ${store_output_path}
spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTag \
--conf spark.sql.shuffle.partitions=100 \
--master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 6g --executor-cores 2 --num-executors 5 \
--deploy-mode cluster --executor-memory 6g --driver-memory 6g --executor-cores 2 --num-executors 5 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 40
......
azkaban/ali/ali_device_tag_daily.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.baichuan.AliTagDaily \
--conf spark.sql.shuffle.partitions=10 \
--conf spark.default.parallelism=10 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 5
......
azkaban/ali/ali_device_tag_v2.sh
......@@ -31,7 +31,7 @@ spark-submit --class mobvista.dmp.datasource.newtag.MatchInterestTagDailyV2 \
--conf spark.default.parallelism=8 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 2 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 2 \
../${JAR} \
-date ${date} -manualOutput ${output_path} -business ${business} -storeOutput ${store_output_path} -coalesce 8
......
azkaban/ali/ali_install_list.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.baichuan.AliInstallList \
--conf spark.yarn.executor.memoryOverhead=2048 \
--conf spark.network.timeout=720s \
--conf spark.default.parallelism=1000 \
--master yarn --deploy-mode cluster --name ali_install_wangjf --executor-memory 8g --driver-memory 6g --executor-cores 3 --num-executors 50 \
--deploy-mode cluster --name ali_install_wangjf --executor-memory 8g --driver-memory 6g --executor-cores 3 --num-executors 50 \
../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 400 -coalesce 400
......
azkaban/ali/ali_install_list_v2.sh
......@@ -39,7 +39,7 @@ spark-submit --class mobvista.dmp.datasource.dm.DmInstallListOrc \
--conf spark.speculation=true \
--conf spark.speculation.quantile=0.8 \
--conf spark.speculation.multiplier=1.2 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors 80 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 3 --num-executors 80 \
../${JAR} \
-output ${OUTPUT_PATH} -input ${INPUT_PATH} -business ${business}
......
azkaban/ali/ali_install_total_orc.sh
......@@ -36,7 +36,7 @@ spark-submit --class mobvista.dmp.common.InstallListLogic \
--conf spark.default.parallelism=400 \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 50 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 50 \
../${JAR} -date ${LOG_TIME} -business ${business} -output ${OUTPUT} -coalesce 100
if [[ $? -ne 0 ]];then
......
azkaban/ali/ali_userinfo_activation_daily_all_job/ali_ios_userinfo_activation_daily.sh
......@@ -34,7 +34,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliIosActivitionDaily \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 60 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 60 \
../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OUTPUT_DAILY_PATH} -coalesce 500 \
-yesterday ${yesterday} -today ${dt_today} -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \
-request_count_result "${IOS_REQUEST_COUNT_RESULT}/${dt_today}" -last_req_day ${last_req_day}
......
azkaban/ali/ali_userinfo_activation_daily_all_job/ali_oaid_userinfo_activation_daily.sh
......@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliOaidActivitionDaily \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 60 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 60 \
../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OAID_OUTPUT_DAILY_PATH} -coalesce 500 \
-yesterday ${yesterday} -today ${dt_today} -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \
-request_count_result "${OAID_REQUEST_COUNT_RESULT}/${dt_today}" -last_req_day ${last_req_day}
......
azkaban/ali/ali_userinfo_activation_daily_all_job/ali_userinfo_activation_daily.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionDaily \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 60 \
--deploy-mode cluster --executor-memory 10g --driver-memory 4g --executor-cores 4 --num-executors 60 \
../../${JAR} -output ${OUTPUT_PATH} -outputdaily ${ALI_OUTPUT_DAILY_PATH} -coalesce 500 \
-yesterday ${yesterday} -today ${dt_today} -dt_dash_today ${dt_dash_today} -dt_dash_rec14day ${dt_dash_rec14day} \
-request_count_result "${REQUEST_COUNT_RESULT}/${dt_today}" -last_req_day ${last_req_day}
......
azkaban/ali/ali_userinfo_postback_activation_daily.sh
......@@ -84,7 +84,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDail
--conf spark.sql.shuffle.partitions=2000 \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 60 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 3 --num-executors 60 \
../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT} -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 50 \
-today ${dt_today} -update_date ${dt_dash_today} \
-dt_dash_rec15day ${dt_dash_rec15day} -syn_to_3s ${ALI_USER_ACTIVATION_SYS_TO3S_PATH} -syn_3s_day ${syn_3s_day}
......
azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_etl_postback_daily.sh
......@@ -66,7 +66,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlAliActivitionPostBackDail
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--conf spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive=true \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 70 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 70 \
../../${JAR} -output ${OUTPUT_PATH} -iosoutput ${ALI_IOS_OUTPUT} -oaidoutput ${ALI_OAID_OUTPUT} -coalesce 300 \
-today ${dt_today} -update_date ${dt_dash_today} -dt_taobao_postback_day ${dt_taobao_postback_day}\
-dt_dash_rec15day ${dt_dash_rec15day} -syn_to_3s ${ALI_USER_ACTIVATION_SYS_TO3S_PATH} -syn_3s_day ${syn_3s_day}
......
azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_extract_h_18_from_dsp_req.sh
......@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlH18FromDmInstallListV2 \
--conf spark.sql.shuffle.partitions=2000 \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 60 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 60 \
../../${JAR} \
-h18_imei ${IMEI_H_18_GUANGDIANTONG_RES_PATH} -h18_imeimd5 ${IMEIMD5_H_18_GUANGDIANTONG_RES_PATH} \
-dt_oneday_ago ${dt_oneday_ago}
......
azkaban/ali/ali_userinfo_postback_activation_daily_v2/ali_insert_other_data_to_dmp.sh
......@@ -43,7 +43,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.EtlOtherDataFromPostBackDail
--conf spark.sql.shuffle.partitions=3000 \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 150 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 150 \
../../${JAR} \
-output ${OUTPUT_PATH} \
-dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago}
......
azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck.sh
......@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--conf hive.exec.orc.default.stripe.size=268435456 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 10 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 10 \
../../${JAR} -imeiInput ${IMEIMD5_INPUT} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
if [[ $? -ne 0 ]];then
......
azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck_02.sh
......@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--conf hive.exec.orc.default.stripe.size=268435456 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 10 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 10 \
../../${JAR} -imeiInput ${IMEIMD5_INPUT} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
if [[ $? -ne 0 ]];then
......
azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck_03.sh
......@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--conf hive.exec.orc.default.stripe.size=268435456 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 10 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 10 \
../../${JAR} -imeiInput ${IMEIMD5_INPUT} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
if [[ $? -ne 0 ]];then
......
azkaban/ali/alipay_lahuo_laxin/alipay_imei_lahuo_ck_04.sh
......@@ -25,7 +25,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayImeiLaHuoCK \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--conf hive.exec.orc.default.stripe.size=268435456 \
--conf spark.sql.files.maxPartitionBytes=268435456 \
--master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 10 \
--deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 10 \
../../${JAR} -imeiInput ${IMEIMD5_INPUT} -date ${LOG_TIME} -host ${host} -cluster ${cluster} -database ${database} -table ${table} -hour ${hour}
if [[ $? -ne 0 ]];then
......
azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_daily.sh
......@@ -37,7 +37,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDaily \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 120 ../../${JAR} \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 120 ../../${JAR} \
-imeioutput "${ALIPAY_IMEIMD5_OUTPUT_PATH}" \
-today ${dt_today} -last_req_day ${last_req_day} -dt_after_one_day ${dt_after_one_day}\
-input_one_day ${INPUT_ONE_DAY} -input_two_day ${INPUT_TWO_DAY} -input_three_day ${INPUT_THREE_DAY} \
......
azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
../../${JAR} -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH} -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
-output01 ${OUTPUT01} -output02 ${OUTPUT02}
......@@ -62,7 +62,7 @@ fi
# --conf spark.sql.broadcastTimeout=1200 \
# --conf spark.yarn.executor.memoryOverhead=4096 \
# --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
# --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
# --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
# ../../${JAR} -dt_today ${dt_today} -dt_three_days_ago ${dt_three_days_ago} \
# -ActivationOutput ${ACTIVATIONOUTPUT} -AcquisitionOutput ${ACQUISITIONOUTPUT}
#
......
azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_02.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
../../${JAR} -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH} -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
-output01 ${OUTPUT01} -output02 ${OUTPUT02}
......@@ -62,7 +62,7 @@ fi
# --conf spark.sql.broadcastTimeout=1200 \
# --conf spark.yarn.executor.memoryOverhead=4096 \
# --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
# --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
# --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
# ../../${JAR} -dt_today ${dt_today} -dt_three_days_ago ${dt_three_days_ago} \
# -ActivationOutput ${ACTIVATIONOUTPUT} -AcquisitionOutput ${ACQUISITIONOUTPUT}
#
......
azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_03.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
../../${JAR} -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH} -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
-output01 ${OUTPUT01} -output02 ${OUTPUT02}
......@@ -62,7 +62,7 @@ fi
# --conf spark.sql.broadcastTimeout=1200 \
# --conf spark.yarn.executor.memoryOverhead=4096 \
# --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
# --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
# --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
# ../../${JAR} -dt_today ${dt_today} -dt_three_days_ago ${dt_three_days_ago} \
# -ActivationOutput ${ACTIVATIONOUTPUT} -AcquisitionOutput ${ACQUISITIONOUTPUT}
#
......
azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_data_to_dmp_04.sh
......@@ -33,7 +33,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayTmpDataToDmp \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.yarn.executor.memoryOverhead=4096 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
../../${JAR} -imeiRequestInput ${IMEIMD5_REQUEST_INPUT_PATH} -imeiResponseInput ${IMEIMD5_RESPONSE_INPUT_PATH} \
-output01 ${OUTPUT01} -output02 ${OUTPUT02}
......@@ -62,7 +62,7 @@ fi
# --conf spark.sql.broadcastTimeout=1200 \
# --conf spark.yarn.executor.memoryOverhead=4096 \
# --conf spark.sql.autoBroadcastJoinThreshold=31457280 \
# --master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
# --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 4 --num-executors 40 \
# ../../${JAR} -dt_today ${dt_today} -dt_three_days_ago ${dt_three_days_ago} \
# -ActivationOutput ${ACTIVATIONOUTPUT} -AcquisitionOutput ${ACQUISITIONOUTPUT}
#
......
azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df.sh
......@@ -20,7 +20,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 10 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 10 \
../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH} -cluster 'cluster_1st' -hour ${hour}
if [[ $? -ne 0 ]];then
......
azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_02.sh
......@@ -19,7 +19,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 10 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 10 \
../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH} -cluster 'cluster_1st' -hour ${hour}
if [[ $? -ne 0 ]];then
......
azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_03.sh
......@@ -19,7 +19,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 10 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 10 \
../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH} -cluster 'cluster_1st' -hour ${hour}
if [[ $? -ne 0 ]];then
......
azkaban/ali/alipay_lahuo_laxin/alipay_lahuo_df_04.sh
......@@ -19,7 +19,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayLaHuoDF \
--conf spark.kryoserializer.buffer.max=256m \
--conf spark.driver.extraJavaOptions="-XX:+UseG1GC" \
--conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 10 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 10 \
../../${JAR} -date ${LOG_TIME} -dt_end_days ${dt_end_days} -partNum 60 -imeiOutput ${IMEIMD5_OUTPUT_PATH} -cluster 'cluster_1st' -hour ${hour}
if [[ $? -ne 0 ]];then
......
azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp.sh
......@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
--conf spark.sql.shuffle.partitions=3000 \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 140 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 140 \
../../${JAR} \
-output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02} \
-dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago} -hour ${hour}
......
azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_02.sh
......@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
--conf spark.sql.shuffle.partitions=3000 \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 140 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 140 \
../../${JAR} \
-output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02} \
-dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago} -hour ${hour}
......
azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_03.sh
......@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
--conf spark.sql.shuffle.partitions=3000 \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 140 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 140 \
../../${JAR} \
-output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02} \
-dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago} -hour ${hour}
......
azkaban/ali/alipay_lahuo_laxin/alipay_other_data_to_dmp_04.sh
......@@ -29,7 +29,7 @@ spark-submit --class mobvista.dmp.datasource.taobao.AlipayOtherDataToDmp \
--conf spark.sql.shuffle.partitions=3000 \
--conf spark.sql.broadcastTimeout=1200 \
--conf spark.sql.autoBroadcastJoinThreshold=31457280 \
--master yarn --deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 140 \
--deploy-mode cluster --executor-memory 8g --driver-memory 4g --executor-cores 5 --num-executors 140 \
../../${JAR} \
-output01 ${OUTPUT_PATH01} -output02 ${OUTPUT_PATH02} \
-dt_today ${dt_today} -dt_oneday_ago ${dt_oneday_ago} -hour ${hour}
......
azkaban/ali/alipay_lahuo_to_dmp/alipay_activation_install_list.sh
......@@ -24,7 +24,7 @@ spark-submit --class mobvista.dmp.datasource.TO.TOInstallList \
--conf spark.network.timeout=720s \
--conf spark.default.parallelism=100 \
--conf spark.read.textfile.tag=1 \
--master yarn --deploy-mode cluster --name AlipayActivationInstallList --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 30 \
--deploy-mode cluster --name AlipayActivationInstallList --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 30 \
../../${JAR} -input ${INPUT_PATH} -oldInput ${OLD_INPUT_PATH} -output ${OUTPUT_PATH} -date ${dt} -parallelism 160 -coalesce 160
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment