#! /bin/bash # # # # # # # # # # # # # # # # # # # # # # # @file :ftp_put_hdfs.sh # @author :liushuai # @revision:2017-02-06 18:48:26 # # # # # # # # # # # # # # # # # # # # # # LOG_TIME=$(date +%Y%m%d -d "$ScheduleTime") year=${LOG_TIME:0:4} month=${LOG_TIME:4:2} day=${LOG_TIME:6:2} export LC_ALL=en_US.UTF-8 export LANGUAGE=en_US:en TIME=$(date +%b%d -d "$ScheduleTime") exit 0 num=0 while : do if [ ${num} -eq 13 ] then exit 1 break fi num=`expr ${num} + 1` ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls android_device_id_exclusion_sale.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ ${TIME} != ${ftp_time} ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls android_device_id_install_no_transaction_0d_to_7d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls android_device_id_install_no_transaction_7d_to_14d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls android_device_id_install_no_transaction_15d_to_29d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls android_device_id_install_no_transaction_30d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls android_device_id_install_no_transaction_15d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls ios_device_id_exclusion_sale.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls ios_device_id_install_no_transaction_0d_to_7d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls ios_device_id_install_no_transaction_7d_to_14d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls ios_device_id_install_no_transaction_15d_to_29d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls ios_device_id_install_no_transaction_30d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin ls ios_device_id_install_no_transaction_15d.csv tmp bye EOF ftp_time=`awk '{print $6 $7}' ./tmp` if [ $TIME != $ftp_time ] then sleep 1h continue fi ftp -n <<- EOF open sftp.cloudgates.net user gate-qqyizx WDGGPBeXumSa cd / bin get android_device_id_exclusion_sale.csv get android_device_id_install_no_transaction_0d_to_7d.csv get android_device_id_install_no_transaction_7d_to_14d.csv get android_device_id_install_no_transaction_15d_to_29d.csv get android_device_id_install_no_transaction_30d.csv get android_device_id_install_no_transaction_15d.csv get ios_device_id_exclusion_sale.csv get ios_device_id_install_no_transaction_0d_to_7d.csv get ios_device_id_install_no_transaction_7d_to_14d.csv get ios_device_id_install_no_transaction_15d_to_29d.csv get ios_device_id_install_no_transaction_30d.csv get ios_device_id_install_no_transaction_15d.csv bye EOF ##修改文件名 mv android_device_id_exclusion_sale.csv android.csv mv android_device_id_install_no_transaction_0d_to_7d.csv android_0d_to_7d.csv mv android_device_id_install_no_transaction_7d_to_14d.csv android_7d_to_14d.csv mv android_device_id_install_no_transaction_15d_to_29d.csv android_15d_to_29d.csv mv android_device_id_install_no_transaction_30d.csv android_30d.csv mv android_device_id_install_no_transaction_15d.csv android_15d.csv mv ios_device_id_exclusion_sale.csv ios.csv mv ios_device_id_install_no_transaction_0d_to_7d.csv ios_0d_to_7d.csv mv ios_device_id_install_no_transaction_7d_to_14d.csv ios_7d_to_14d.csv mv ios_device_id_install_no_transaction_15d_to_29d.csv ios_15d_to_29d.csv mv ios_device_id_install_no_transaction_30d.csv ios_30d.csv mv ios_device_id_install_no_transaction_15d.csv ios_15d.csv #创建hdfs目录 hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/ hadoop fs -rm -r s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/transactions/ hadoop fs -rm -r s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/untransactions/ hadoop fs -mkdir s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/transactions/ hadoop fs -mkdir s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/untransactions/ hadoop fs -mkdir -p s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/ hadoop fs -rm -r s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/transactions/ hadoop fs -rm -r s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/untransactions/ hadoop fs -mkdir s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/transactions/ hadoop fs -mkdir s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/untransactions/ #上传文件 hadoop fs -put android.csv s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/transactions/ hadoop fs -put ios.csv s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/transactions/ hadoop fs -put android_* s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/android/${year}/${month}/${day}/untransactions/ hadoop fs -put ios_* s3://mob-emr-test/dataplatform/DataWareHouse/data/import/zarola/ios/${year}/${month}/${day}/untransactions/ #删除本地文件 rm -rf android* rm -rf ios* break done