#!/bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file :3s_tracking_install_daily.sh # @author :liushuai # @revision:2017-03-20 20:09 # # # # # # # # # # # # # # # # # # # # # # source ../dmp_env.sh LOG_TIME=$(date +%Y%m%d -d "-1 day $ScheduleTime") dt_yesterday=$(date +%Y%m%d -d "-2 day $ScheduleTime") year=${LOG_TIME:0:4} month=${LOG_TIME:4:2} day=${LOG_TIME:6:2} dt_today_dash=$(date +%Y-%m-%d -d "-2 day $ScheduleTime") date_path="${year}/${month}/${day}" old_path=$(date -d"$LOG_TIME -1 day " +"%Y/%m/%d") CAMPAIGN_INPUT_PATH="${DIM_CAMPAIGN_PACKAGE_3S}/${year}/${month}-${day}/" INPUT_PATH_3S="${PATH_3S}/$date_path/" TMP_OUT_PATH="${TMP_INSTALL_DAILY_3S}/$date_path" OUT_PATH="${INSTALL_DAILY_3S}/$date_path/" check_await "${INPUT_PATH_3S}/23/45/*" hadoop fs -rm -r ${TMP_OUT_PATH} hadoop jar ../${JAR} mobvista.dmp.datasource.tracking.mapreduce.TrackingInstallDailyMR \ -Dtask.date=${year}-${month}-${day} \ -Dmapreduce.fileoutputcommitter.algorithm.version=2 \ "${CAMPAIGN_INPUT_PATH}" "${INPUT_PATH_3S}" "${TMP_OUT_PATH}" if [ $? -ne 0 ];then exit 255 fi # 为bundleId 匹配 packageName matchBundlePackage "$date_path" "$old_path" "2" "3" "$TMP_OUT_PATH" "$OUT_PATH" "3s" "../${JAR}" mount_partition "etl_3s_install_daily" "\`date\`='$LOG_TIME'" "$OUT_PATH" || exit 1 # 国内大媒体入库dmp系统 hive_cmd "set mapred.reduce.tasks = 15; set hive.exec.compress.output=true; set mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec; insert overwrite table dwh.etl_3s_install_daily_tmp select lower(device_id) device_id,device_type,platform,package_name,update_date,country from dwh.etl_3s_install_daily where \`date\` = '${LOG_TIME}' union select lower(device_id) device_id,device_type,platform,package_name, '${dt_today_dash}' update_date,'CN' country from dwh.ods_bigmedia_domestic_daily where dt = '${dt_yesterday}' and package_name !='' " hadoop fs -rmr $OUT_PATH* if [ $? -eq 0 ];then hadoop fs -cp s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/tmp/etl_3s_install_daily_tmp/* $OUT_PATH fi hadoop fs -touchz ${OUT_PATH}_SUCCESS