all_incremental.sh 1.01 KB
#!/usr/bin/env bash

# # # # # # # # # # # # # # # # # # # # # #
# @file    :all_cross.sh
# @author  :liushuai
# @revision:2017-05-17 20:09
# @desc    :dmp 4个数据源纯新增
# # # # # # # # # # # # # # # # # # # # # #


date_path=$(date +%Y/%m/%d -d "-2 day $ScheduleTime")
old_date_path=$(date +%Y/%m/%d -d "-3 day $ScheduleTime")

source ../prd_env.sh


INPUT_PATH="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dm_install_list/${date_path}/{ga,3s,dsp_req,adn_install,adn_request_sdk}/"
INPUT_ALL="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dm_install_list/${old_date_path}/"
TEMP_OUTPUT="s3://mob-emr-test/liushuai/prd/${date_path}/tmp_all_merge"
OUTPUT="s3://mob-emr-test/liushuai/prd/${date_path}/new_all_merge/"

hadoop fs -rmr "${OUTPUT}" "${TEMP_OUTPUT}"

hadoop jar ../${JAR} mobvista.prd.datasource.newall.NewAllMergeMR \
  -Dmapreduce.fileoutputcommitter.algorithm.version=2 \
  -Dday=${date_path} -DoldDay=${old_date_path} \
  "${INPUT_PATH}" "${INPUT_ALL}" "${TEMP_OUTPUT}" "${OUTPUT}" "100,10"