1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#!/bin/bash
source ./common/tools.sh
source ../dmp_env.sh
export HIVE_CONF_DIR=/data/azkaban-hadoop/command-home/hive-offline/conf
today=${ScheduleTime:-$1}
echo "ScheduleTime=$ScheduleTime"
echo "toady=$today"
date_path=$(date +%Y/%m/%d -d "-1 day $today")
date_path_old=$(date +%Y%m%d -d "-2 day $today")
date_str_undline=$(date +%Y_%m_%d -d "-1 day $today")
date_str_midline=$(date +%Y-%m-%d -d "-1 day $today")
## check_await "s3://live-ga-exchange-rates/${date_path_old}/rates.json"
get_recently_slash_date() {
local path="$1"
local expect_date="$2"
local sub_partition="$3"
while :; do
local year=${expect_date:0:4}
local month=${expect_date:4:2}
local day=${expect_date:6:2}
local check_path="$path/$year/$month/$day/$sub_partition"
if hadoop fs -test -e "${check_path}"; then
echo "${year}/${month}/${day}"
break
fi
local expect_date=$(date -d "$expect_date 1 days ago" "+%Y%m%d")
done
}
recent_date_path_old=$(get_recently_slash_date "s3://live-ga-exchange-rates" "${date_path_old}" "rates.json")
HIVE_CMD=$(hive_func)
$HIVE_CMD -v -hivevar old_prefix=$recent_date_path_old -hivevar prefix=$date_path -hivevar date_str_undline=$date_str_undline -hivevar date_str_midline=$date_str_midline -f payment_checkpoint.sql