#!/bin/bash source ./common/tools.sh today=${ScheduleTime:-$1} echo "ScheduleTime=$ScheduleTime" echo "toady=$today" cur_date=$(date '+%Y-%m-%d') part_pre1_path=$(date +%Y%m%d -d "-1 day $today") part_pre1_dash=$(date +%Y-%m-%d -d "-1 day $today") part_pre2_path=$(date +%Y%m%d -d "-2 day $today") pre1_dateslash=$(date +%Y/%m/%d -d "-1 day $today") part_pre8_path=$(date +%Y%m%d -d "-8 day $today") part_pre31_path=$(date +%Y%m%d -d "-31 day $today") date_str_undline=$(date +%Y_%m_%d -d "-1 day $today") HIVE_CMD=$(hive_func) hadoop fs -text s3://ga-gameid/$pre1_dateslash/gameidkeys_new.gz | jq -r -c ' .[] | {store_id: .store_id, game_id: .game_id}' >redshift-compatible.json hadoop fs -put -f redshift-compatible.json s3://mob-emr-test/dataplatform/emr/store_game/ hadoop fs -test -e s3://mob-emr-test/dataplatform/datawarehourse/ga/player_checkpoint_ga/${part_pre1_path} if [ $? -ne 0 ]; then hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/datawarehourse/ga/player_checkpoint_ga/${part_pre1_path} fi hadoop fs -test -e s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/$pre1_dateslash/dau-device-data-export if [ $? -ne 0 ]; then hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/$pre1_dateslash/dau-device-data-export fi hadoop fs -test -e s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/device-metrics if [ $? -ne 0 ]; then hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/device-metrics fi hadoop fs -test -e s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/installation-list if [ $? -ne 0 ]; then hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/installation-list fi $HIVE_CMD -v -hivevar part_pre8_path=$part_pre8_path -hivevar part_pre31_path=$part_pre31_path -hivevar date_str_undline=$date_str_undline -hivevar pre1_dateslash=$pre1_dateslash -hivevar part_pre1_path=$part_pre1_path -hivevar part_pre2_path=$part_pre2_path -hivevar cur_date=$cur_date -hivevar part_pre1_dash=$part_pre1_dash -f player_checkpoint_query_cluster.sql hadoop fs -touchz s3://mob-emr-test/dataplatform/datawarehourse/ga/${part_pre1_path}/_SUCCESS hadoop fs -touchz s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/$pre1_dateslash/dau-device-data-export/_SUCCESS hadoop fs -touchz s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/device-metrics/_SUCCESS hadoop fs -touchz s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/installation-list/_SUCCESS