#!/bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file : collect_package_name.sh # @author: houying # @date : 16-11-14 # @desc : 收集需要抓取的package列表 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh LOG_DATE=$(date -d "$ScheduleTime 1 days ago" "+%Y%m%d") date_path=$(date -d "$ScheduleTime 1 days ago" +"%Y/%m/%d") yes_date=$(date -d "$ScheduleTime 2 days ago" +"%Y%m%d") expire_date=$(date -d "$ScheduleTime 4 days ago" +"%Y%m%d") expire_path=$(date -d "$ScheduleTime 4 days ago" +"%Y/%m/%d") OUTPUT_PATH="${PACKAGE_BLACK_LIST}/$date_path" EXPIRE_PATH="${PACKAGE_BLACK_LIST}/$expire_path" spark-submit --class mobvista.dmp.main.PackageBlackList \ --conf spark.sql.shuffle.partitions=10 \ --master yarn --deploy-mode cluster --executor-memory 4g --driver-memory 4g --executor-cores 2 --num-executors 5 \ ../${JAR} \ -date "${yes_date}" \ -iosDailyPath "${TMP_IOS_APP_INFO_PATH}" -adrDailyPath "${TMP_ADR_APP_INFO_PATH}" \ -toCrawlerPath "${TMP_CRAWLER_INFO_PATH}" -output $OUTPUT_PATH if [ $? -ne 0 ];then exit 255 fi hive_cmd " use dev; alter table dm_package_black_list add if not exists partition(dt='${LOG_DATE}') location '${OUTPUT_PATH}'; alter table dm_package_black_list drop if exists partition(dt='${expire_date}'); " if [ $? -ne 0 ];then exit 255 fi hadoop fs -rmr $EXPIRE_PATH echo "[Finish]"