fix

azkaban/mysql2hive.sh

-#!/usr/sh
-
+#!/usr/bash
 
 today=${ScheduleTime:-$1}
 date_path=$(date +%Y/%m/%d -d "-1 day $today")
 
+java -jar -Dloader.main=com.mobvista.apptag.utils.StorageTag apptag.jar
+if [[ $? -ne 0 ]]; then
+    exit 255
+fi
+
 # sqoop import "-Dorg.apache.sqoop.splitter.allow_text_splitter=true" --connect jdbc:mysql://dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com:3306/app_tag  \
 #     --username apptag_rw --password 7gyLEVtkER3u8c9 --table tag_result --columns 'package_name,app_name,platform,feat_id' -m 1 --null-string '\\N' --null-non-string '\\N' \
 #     --target-dir 's3://mob-emr-test/wangjf/data/dwh/app_tag' --fields-terminated-by '\001' --delete-target-dir
-
+: '
 sqoop import "-Dorg.apache.sqoop.splitter.allow_text_splitter=true" --connect jdbc:mysql://dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com:3306/app_tag  \
     --username apptag_rw --password 7gyLEVtkER3u8c9 --query 'select package_name,app_name,lower(platform),json_keys(feat_id) from tag_result where $CONDITIONS and feat_id is not null' -m 1 --null-string '\\N' --null-non-string '\\N' \
     --target-dir 's3://mob-emr-test/wangjf/data/dwh/app_tag' --fields-terminated-by '\001' --delete-target-dir
+'
+
+sqoop import "-Dorg.apache.sqoop.splitter.allow_text_splitter=true" --connect jdbc:mysql://dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com:3306/app_tag  \
+    --username apptag_rw --password 7gyLEVtkER3u8c9 --query 'select package_name,app_name,lower(platform),json_keys(tag) from storage where $CONDITIONS and tag is not null' -m 1 --null-string '\\N' --null-non-string '\\N' \
+    --target-dir 's3://mob-emr-test/wangjf/data/dwh/app_tag' --fields-terminated-by '\001' --delete-target-dir
+
+if [[ $? -ne 0 ]]; then
+    exit 255
+fi
 
 base_dir="s3://mob-emr-test/dataplatform/DataWareHouse/data/dwh/dim_package_tags"
 
@@ -30,7 +43,7 @@ sql="INSERT OVERWRITE DIRECTORY '${base_dir}/${date_path}'
     "
 
 hive -e "${sql}"
-if [ $? -ne 0 ]; then
+if [[ $? -ne 0 ]]; then
     exit 255
 fi
 

package.sh

+#!/bin/bash
+
+mvn clean compile package || exit 1
+
+mkdir target/zip
+cp -r azkaban/* target/zip/
+cp target/apptag.jar target/zip/apptag.jar
+cd target/zip
+zip -r ../AppTag.zip ./*
\ No newline at end of file

src/main/java/com/mobvista/apptag/utils/StorageTag.java

@@ -19,13 +19,13 @@ import java.util.*;
  * @phone: 152-1062-7698
  */
 public class StorageTag {
-    static final String CONNECTION_URL = "jdbc:mysql://localhost:3306/app_tag?useUnicode=true&characterEncoding=utf8&useSSL=false";
-    //  static final String CONNECTION_URL = "jdbc:mysql://dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com:3306/app_tag?useUnicode=true&characterEncoding=utf8&useSSL=false";
+    //  static final String CONNECTION_URL = "jdbc:mysql://localhost:3306/app_tag?useUnicode=true&characterEncoding=utf8&useSSL=false";
+    static final String CONNECTION_URL = "jdbc:mysql://dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com:3306/app_tag?useUnicode=true&characterEncoding=utf8&useSSL=false";
     static final String DRIVER = "com.mysql.jdbc.Driver";
-    static final String USER = "root";
-    //  static final String USER = "apptag_rw";
-    static final String PASSWORD = "19920627";
-    //  static final String PASSWORD = "7gyLEVtkER3u8c9";
+    //  static final String USER = "root";
+    static final String USER = "apptag_rw";
+    //  static final String PASSWORD = "19920627";
+    static final String PASSWORD = "7gyLEVtkER3u8c9";
     static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
     static ObjectMapper mapper = new ObjectMapper();
@@ -83,23 +83,26 @@ public class StorageTag {
         }
     }
 
+    //  更新已经入库的标签
     private static void delete(Set<String> set) {
         try {
             Class.forName("com.mysql.jdbc.Driver");
             Connection con = DriverManager.getConnection(CONNECTION_URL, USER, PASSWORD);
-            Statement stmt = con.createStatement();
-            String deleteSql = "DELETE FROM storage_result WHERE package_name IN ";
+            Statement stmt1 = con.createStatement();
+            Statement stmt2 = con.createStatement();
+            String deleteSql1 = "DELETE FROM storage_result WHERE package_name IN ";
+            String deleteSql2 = "DELETE FROM storage WHERE package_name IN ";
             Iterator<String> iterator = set.iterator();
             StringBuffer sql = new StringBuffer();
             while (iterator.hasNext()) {
                 String packageName = iterator.next();
                 sql.append("'").append(packageName).append("',");
             }
-            //  System.out.println(deleteSql + "(" + sql.substring(0, sql.length() - 1) + ")");
-            stmt.execute(deleteSql + "(" + sql.substring(0, sql.length() - 1) + ")");
+            stmt1.execute(deleteSql1 + "(" + sql.substring(0, sql.length() - 1) + ")");
+            stmt2.execute(deleteSql2 + "(" + sql.substring(0, sql.length() - 1) + ")");
             con.close();
         } catch (Exception e) {
-            System.out.println(e);
+            e.printStackTrace();
         }
     }
 
@@ -162,7 +165,7 @@ public class StorageTag {
             stmt1.executeQuery(insert_sql2 + insertSql_2.substring(0, insertSql_2.length() - 1));
             con.close();
         } catch (Exception e) {
-            System.out.println(e);
+            e.printStackTrace();
         }
     }