dmp_age.sh 1.45 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#! /bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @file    :dmp_age.sh
# @author  :wangjf
# @revision:2018-09-26 18:22:29
# @description: 重构
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

LOG_TIME=$(date +%Y%m%d -d "-1 day ${ScheduleTime}")

date_path=$(date +"%Y/%m/%d" -d "-1 day ${ScheduleTime}")

before_date_path=$(date +"%Y/%m/%d" -d "-4 day ${ScheduleTime}")

INPUT_PATH="${AGE_CALC_DEVICE}/${date_path}/"
OUTPUT_PATH="${AGE_VALIDATE}/${date_path}/"

hadoop fs -rm -r ${OUTPUT_PATH}
hadoop jar ../${JAR} mobvista.dmp.datasource.age.mapreduce.DmpYearMRV2 \
    -Dmapreduce.job.inputformat.class=com.mobvista.ai.mrdemo.mapreduce.CombineOrcFileInputFormat \
    -libjars /data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-orcinputwriter-1.0-SNAPSHOT.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-core-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-mapreduce-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/aircompressor-0.10.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/hive-storage-api-2.6.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-shims-1.5.0.jar \
    "${INPUT_PATH}" "${OUTPUT_PATH}"

if [[ $? -ne 0 ]]; then
    exit 255
fi

hadoop fs -rm -r ${AGE_GET_DSP_PATH}/${before_date_path}
hadoop fs -rm -r ${AGE_GET_GA_PATH}/${before_date_path}
hadoop fs -rm -r ${AGE_MERGE_INSTALL}/${before_date_path}
hadoop fs -rm -r ${AGE_CALC_PACKAGE_DICT}/${before_date_path}