player_checkpoint_query_cluster.sh 2.59 KB
Newer Older
wang-jinfeng committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48
#!/bin/bash
source ./common/tools.sh

today=${ScheduleTime:-$1}

echo "ScheduleTime=$ScheduleTime"
echo "toady=$today"

cur_date=$(date '+%Y-%m-%d')
part_pre1_path=$(date +%Y%m%d -d "-1 day $today")
part_pre1_dash=$(date +%Y-%m-%d -d "-1 day $today")
part_pre2_path=$(date +%Y%m%d -d "-2 day $today")
pre1_dateslash=$(date +%Y/%m/%d -d "-1 day $today")
part_pre8_path=$(date +%Y%m%d -d "-8 day $today")
part_pre31_path=$(date +%Y%m%d -d "-31 day $today")
date_str_undline=$(date +%Y_%m_%d -d "-1 day $today")

HIVE_CMD=$(hive_func)

hadoop fs -text s3://ga-gameid/$pre1_dateslash/gameidkeys_new.gz | jq -r -c ' .[] | {store_id: .store_id, game_id: .game_id}' >redshift-compatible.json
hadoop fs -put -f redshift-compatible.json s3://mob-emr-test/dataplatform/emr/store_game/

hadoop fs -test -e s3://mob-emr-test/dataplatform/datawarehourse/ga/player_checkpoint_ga/${part_pre1_path}
if [ $? -ne 0 ]; then
  hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/datawarehourse/ga/player_checkpoint_ga/${part_pre1_path}
fi

hadoop fs -test -e s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/$pre1_dateslash/dau-device-data-export
if [ $? -ne 0 ]; then
  hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/$pre1_dateslash/dau-device-data-export
fi

hadoop fs -test -e s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/device-metrics
if [ $? -ne 0 ]; then
  hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/device-metrics
fi

hadoop fs -test -e s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/installation-list
if [ $? -ne 0 ]; then
  hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/installation-list
fi

$HIVE_CMD -v -hivevar part_pre8_path=$part_pre8_path -hivevar part_pre31_path=$part_pre31_path -hivevar date_str_undline=$date_str_undline -hivevar pre1_dateslash=$pre1_dateslash -hivevar part_pre1_path=$part_pre1_path -hivevar part_pre2_path=$part_pre2_path -hivevar cur_date=$cur_date -hivevar part_pre1_dash=$part_pre1_dash -f player_checkpoint_query_cluster.sql

hadoop fs -touchz s3://mob-emr-test/dataplatform/datawarehourse/ga/${part_pre1_path}/_SUCCESS
hadoop fs -touchz s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/$pre1_dateslash/dau-device-data-export/_SUCCESS
hadoop fs -touchz s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/device-metrics/_SUCCESS
hadoop fs -touchz s3://mob-emr-test/dataplatform/datawarehourse/dmp/daily_export/${pre1_dateslash}/installation-list/_SUCCESS