validate.sh 1.52 KB
#! /bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @file    :extract_device.sh
# @author  :liushuai
# @revision:2017-01-18 17:01
# # # # # # # # # # # # # # # # # # # # # #

source ../dmp_env.sh

LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime")

date_path=$(date +"%Y/%m/%d" -d "-1 day ${ScheduleTime}")

before_date_path=$(date +"%Y/%m/%d" -d "-4 day ${ScheduleTime}")

INPUT_PATH="${GENDER_CALC_DEVICE}/${date_path}/"
OUTPUT_PATH="${GENDER_VALIDATE}/${date_path}/"

hadoop fs -rm -r ${OUTPUT_PATH}

hadoop jar ../${JAR} mobvista.dmp.datasource.gender.ValidateMRV2 \
    -Dmapreduce.job.inputformat.class=com.mobvista.ai.mrdemo.mapreduce.CombineOrcFileInputFormat \
    -libjars /data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-orcinputwriter-1.0-SNAPSHOT.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-core-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-mapreduce-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/aircompressor-0.10.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/hive-storage-api-2.6.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-shims-1.5.0.jar \
    -Dmapreduce.fileoutputcommitter.algorithm.version=2 \
    "${INPUT_PATH}" "${OUTPUT_PATH}"

if [[ $? -ne 0 ]]; then
    exit 255
fi

# 删除临时结果
hadoop fs -rm -r ${GENDER_GET_DSP_PATH}/${before_date_path}
hadoop fs -rm -r ${GENDER_GET_GA_PATH}/${before_date_path}
hadoop fs -rm -r ${GENDER_MERGE_INSTALL}/${before_date_path}
hadoop fs -rm -r ${GENDER_CALC_PACKAGE_DICT}/${before_date_path}