ftp_put_hdfs.sh 7.78 KB
#! /bin/bash

# # # # # # # # # # # # # # # # # # # # # #
# @file    :ftp_put_hdfs.sh
# @author  :liushuai
# @revision:2017-02-06 18:48:26
# # # # # # # # # # # # # # # # # # # # # #


LOG_TIME=$(date +%Y%m%d -d "$ScheduleTime")

year=${LOG_TIME:0:4}
month=${LOG_TIME:4:2}
day=${LOG_TIME:6:2}

export LC_ALL=en_US.UTF-8
export LANGUAGE=en_US:en

TIME=$(date +%b%d -d "$ScheduleTime")
exit 0
num=0
while :
do
    if [ ${num} -eq 13 ]
    then
        exit 1
        break
    fi
    num=`expr ${num} + 1`
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls android_device_id_exclusion_sale.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ ${TIME} != ${ftp_time} ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls android_device_id_install_no_transaction_0d_to_7d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls android_device_id_install_no_transaction_7d_to_14d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls android_device_id_install_no_transaction_15d_to_29d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls android_device_id_install_no_transaction_30d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls android_device_id_install_no_transaction_15d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls ios_device_id_exclusion_sale.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls ios_device_id_install_no_transaction_0d_to_7d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls ios_device_id_install_no_transaction_7d_to_14d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls ios_device_id_install_no_transaction_15d_to_29d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls ios_device_id_install_no_transaction_30d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        ls ios_device_id_install_no_transaction_15d.csv tmp
        bye
EOF
    ftp_time=`awk '{print $6 $7}' ./tmp`
    if [ $TIME != $ftp_time ]
    then
        sleep 1h
        continue
    fi
    ftp -n <<- EOF
        open sftp.cloudgates.net
        user gate-qqyizx WDGGPBeXumSa
        cd /
        bin
        get android_device_id_exclusion_sale.csv
        get android_device_id_install_no_transaction_0d_to_7d.csv
        get android_device_id_install_no_transaction_7d_to_14d.csv
        get android_device_id_install_no_transaction_15d_to_29d.csv
        get android_device_id_install_no_transaction_30d.csv
        get android_device_id_install_no_transaction_15d.csv
        get ios_device_id_exclusion_sale.csv
        get ios_device_id_install_no_transaction_0d_to_7d.csv
        get ios_device_id_install_no_transaction_7d_to_14d.csv
        get ios_device_id_install_no_transaction_15d_to_29d.csv
        get ios_device_id_install_no_transaction_30d.csv
        get ios_device_id_install_no_transaction_15d.csv
        bye
EOF
    ##修改文件名
    mv android_device_id_exclusion_sale.csv android.csv
    mv android_device_id_install_no_transaction_0d_to_7d.csv android_0d_to_7d.csv
    mv android_device_id_install_no_transaction_7d_to_14d.csv android_7d_to_14d.csv
    mv android_device_id_install_no_transaction_15d_to_29d.csv android_15d_to_29d.csv
    mv android_device_id_install_no_transaction_30d.csv android_30d.csv
    mv android_device_id_install_no_transaction_15d.csv android_15d.csv
    mv ios_device_id_exclusion_sale.csv ios.csv
    mv ios_device_id_install_no_transaction_0d_to_7d.csv ios_0d_to_7d.csv
    mv ios_device_id_install_no_transaction_7d_to_14d.csv ios_7d_to_14d.csv
    mv ios_device_id_install_no_transaction_15d_to_29d.csv ios_15d_to_29d.csv
    mv ios_device_id_install_no_transaction_30d.csv ios_30d.csv
    mv ios_device_id_install_no_transaction_15d.csv ios_15d.csv

    #创建hdfs目录
    hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/
    hadoop fs -rm -r s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/transactions/
    hadoop fs -rm -r s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/untransactions/
    hadoop fs -mkdir s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/transactions/
    hadoop fs -mkdir s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/untransactions/

    hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/
    hadoop fs -rm -r s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/transactions/
    hadoop fs -rm -r s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/untransactions/
    hadoop fs -mkdir s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/transactions/
    hadoop fs -mkdir s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/untransactions/

    #上传文件
    hadoop fs -put android.csv s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/transactions/
    hadoop fs -put ios.csv s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/transactions/

    hadoop fs -put android_* s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/untransactions/
    hadoop fs -put ios_* s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/untransactions/

    #删除本地文件
    rm -rf android*
    rm -rf ios*
    break
done