update rtdmp add lazada

azkaban/rtdmp/lazada/etl_event.job

+type=command
+command=bash -x etl_job.sh 'event'
\ No newline at end of file

azkaban/rtdmp/lazada/etl_install.job

+type=command
+command=bash -x etl_job.sh 'install'
\ No newline at end of file

azkaban/rtdmp/lazada/etl_job.sh

+#!/bin/bash
+
+source ../../dmp_env.sh
+
+tb_type=$1
+
+today=${ScheduleTime}
+
+date=$(date +"%Y%m%d" -d "-1 day $today")
+
+date_path=$(date +"%Y/%m/%d/" -d "-1 day $today")
+
+INPUT="s3://mob-ad/adn/tracking-v3"
+
+hours="23"
+res="beijing virginia"
+
+for hh in ${hours}; do
+  for re in ${res}; do
+    check_await "${INPUT}/${tb_type}/${date_path}/${re}/${hh}/_SUCCESS"
+  done
+done
+
+OUTPUT="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/etl_adn_tracking_${tb_type}/${date_path}"
+
+spark-submit --class mobvista.dmp.datasource.rtdmp.lazada.ETLJob \
+  --name "ETLJob.${tb_type}.${date}" \
+  --conf spark.sql.shuffle.partitions=100 \
+  --conf spark.default.parallelism=100 \
+  --conf spark.kryoserializer.buffer.max=256m \
+  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
+  --files ${HIVE_SITE_PATH} \
+  --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 4 --num-executors 5 \
+  ../../${JAR} -dt ${date} -output ${OUTPUT} -tb_type ${tb_type}
+
+if [[ $? -ne 0 ]]; then
+  exit 255
+fi
+
+mount_partition "etl_adn_tracking_${tb_type}" "dt='${date}'" "$OUTPUT"

azkaban/rtdmp/lazada/merge_install.job

+type=command
+dependencies=etl_install,etl_event
+command=bash -x merge_install.sh
\ No newline at end of file

azkaban/rtdmp/lazada/merge_install.sh

+#!/bin/bash
+
+source ../../dmp_env.sh
+
+today=${ScheduleTime}
+
+date=$(date +"%Y%m%d" -d "-1 day $today")
+
+date_path=$(date +"%Y/%m/%d/" -d "-1 day $today")
+
+OUTPUT="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/merge_adn_tracking_install/${date_path}"
+
+old_date_path=$(date +"%Y/%m/%d/" -d "-2 day $today")
+
+OLD_OUTPUT="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/merge_adn_tracking_install/${old_date_path}"
+
+# check_await "$OLD_OUTPUT/_SUCCESS"
+
+spark-submit --class mobvista.dmp.datasource.rtdmp.lazada.MergeInstallJob \
+  --name "MergeInstallJob.${date}" \
+  --conf spark.sql.shuffle.partitions=4000 \
+  --conf spark.default.parallelism=4000 \
+  --conf spark.kryoserializer.buffer.max=256m \
+  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
+  --files ${HIVE_SITE_PATH} \
+  --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 4 --num-executors 5 \
+  ../../${JAR} -dt ${date} -output ${OUTPUT}
+
+if [[ $? -ne 0 ]]; then
+  exit 255
+fi
+
+mount_partition "merge_adn_tracking_install" "dt='${date}'" "$OUTPUT"

azkaban/rtdmp/lazada/process_rtdmp.job

+type=command
+dependencies=merge_install
+command=bash -x process_rtdmp.sh
\ No newline at end of file

azkaban/rtdmp/lazada/process_rtdmp.sh

+#!/bin/bash
+
+source ../../dmp_env.sh
+
+today=${ScheduleTime}
+
+date=$(date +"%Y%m%d" -d "-1 day $today")
+
+date_path=$(date +"%Y/%m/%d/" -d "-1 day $today")
+
+OUTPUT="s3://mob-emr-test/dataplatform/rtdmp_request/${date_path}/lazada"
+
+spark-submit --class mobvista.dmp.datasource.rtdmp.lazada.ProcessRTJob \
+  --name "ProcessRTJob.${date}" \
+  --conf spark.sql.shuffle.partitions=4000 \
+  --conf spark.default.parallelism=4000 \
+  --conf spark.kryoserializer.buffer.max=256m \
+  --conf spark.executor.extraJavaOptions="-XX:+UseG1GC" \
+  --files ${HIVE_SITE_PATH} \
+  --master yarn --deploy-mode cluster --executor-memory 6g --driver-memory 4g --executor-cores 4 --num-executors 5 \
+  ../../${JAR} -dt ${date} -output ${OUTPUT}
+
+if [[ $? -ne 0 ]]; then
+  exit 255
+fi