fix

azkaban/mysql2hive.sh

@@ -23,7 +23,7 @@ sql="INSERT OVERWRITE DIRECTORY '${base_dir}/${date_path}'
                     LATERAL VIEW EXPLODE(SPLIT(REGEXP_REPLACE(REGEXP_REPLACE(SUBSTRING(category,2,LENGTH(category) - 2),'\"',''),' ',''),',')) featTable AS feat_id) a
                 INNER JOIN dwh.dm_old2new_tag b ON a.feat_id = b.new_second_id
             UNION ALL
-            SELECT c.package_name,c.platform,c.tag_type,c.first_tag,c.second_tag,c,comment
+            SELECT c.package_name,c.platform,c.tag_type,c.first_tag,c.second_tag,c.comment
             FROM dwh.dim_package_tags c LEFT JOIN dev.app_tag d ON LOWER(c.package_name) = LOWER(d.package_name)
             WHERE d.package_name IS NULL
         ) package_tags

pom.xml

@@ -93,11 +93,13 @@
             <artifactId>spring-aspects</artifactId>
             <version>5.1.2.RELEASE</version>
         </dependency>
+        <!--
         <dependency>
             <groupId>org.mongodb</groupId>
             <artifactId>mongo-java-driver</artifactId>
             <version>3.10.2</version>
         </dependency>
+        -->
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>

src/main/java/com/mobvista/apptag/utils/GetPackageFromMongo.java

-package com.mobvista.apptag.utils;
-
-import com.mongodb.MongoClientSettings;
-import com.mongodb.ReadPreference;
-import com.mongodb.ServerAddress;
-import com.mongodb.client.*;
-import com.mongodb.client.model.Filters;
-import org.apache.commons.lang3.StringUtils;
-import org.bson.Document;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @package: com.mobvista.apptag.utils
- * @author: wangjf
- * @date: 2019-07-17
- * @time: 11:38
- * @email: jinfeng.wang@mobvista.com
- * @phone: 152-1062-7698
- */
-public class GetPackageFromMongo {
-    public static void main(String[] args) {
-        try {
-            String mongoUrl = "internal-beijing-adServerMongo-virginia-1390640500.us-east-1.elb.amazonaws.com";
-
-            Date date = new Date();
-            SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
-            Date dt = sdf.parse(sdf.format(date));
-            Calendar rightNow = Calendar.getInstance();
-            rightNow.setTime(dt);
-            rightNow.add(Calendar.DAY_OF_YEAR, -30);
-            //  MongoClientOptions.Builder builder = new MongoClientOptions.Builder()
-            MongoClientSettings settings = MongoClientSettings.builder()
-                    .retryWrites(true)
-                    .applyToConnectionPoolSettings(builder ->
-                            builder.maxConnectionIdleTime(6000, TimeUnit.MILLISECONDS))
-                    .applyToClusterSettings(builder -> {
-                        builder.hosts(Arrays.asList(new ServerAddress(mongoUrl, 27017)));
-                    }).build();
-
-            //  MongoClient mongoClient = new MongoClient(new ServerAddress("server ip",27017), Arrays.asList(credential2),opts);
-            // 连接到 mongodb 服务
-
-            MongoClient mongoClient = MongoClients.create(settings);
-
-            // 连接到数据库
-            MongoDatabase mongoDatabase = mongoClient.getDatabase("new_adn");
-            mongoDatabase.withReadPreference(ReadPreference.secondary());
-
-            System.out.println("Connect to database successfully");
-
-            MongoCollection<Document> collection = mongoDatabase.getCollection("campaign");
-            MongoCursor<Document> findIterable = collection.find(Filters.and(Filters.gt("startTime", rightNow.getTime().getTime() / 1000), Filters.eq("advertiserId", 903))).iterator();
-
-            Set<String> set = new HashSet<>();
-
-            while (findIterable.hasNext()) {
-                String json = findIterable.next().toJson();
-                if (GsonUtil.String2JsonObject(json).has("packageName")) {
-                    String packageName = GsonUtil.String2JsonObject(json).get("packageName").getAsString();
-                    if (StringUtils.isNotBlank(packageName)) {
-                        set.add(packageName);
-                    }
-                }
-            }
-            System.out.println("packageNames.size ===>>> " + set.size());
-            MySqlUtil mySqlUtil = new MySqlUtil();
-            mySqlUtil.insert(set);
-        } catch (Exception e) {
-            System.err.println(e.getClass().getName() + ": " + e.getMessage());
-        }
-    }
-}