update AppTag support oldTag upload

pom.xml

@@ -103,6 +103,16 @@
             <artifactId>aws-java-sdk-s3</artifactId>
             <version>1.11.437</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi</artifactId>
+            <version>3.17</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml</artifactId>
+            <version>3.17</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/java/com/mobvista/apptag/controller/StorageController.java

@@ -2,6 +2,7 @@ package com.mobvista.apptag.controller;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.JsonObject;
 import com.mobvista.apptag.config.WebSecurityConfig;
 import com.mobvista.apptag.entity.Feat;
 import com.mobvista.apptag.entity.Storage;
@@ -13,19 +14,31 @@ import com.mobvista.apptag.mapper.TagDao;
 import com.mobvista.apptag.mapper.UserDao;
 import com.mobvista.apptag.service.FeatService;
 import com.mobvista.apptag.service.StorageService;
-import com.mobvista.apptag.utils.PageUtil;
-import com.mobvista.apptag.utils.Query;
-import com.mobvista.apptag.utils.R;
+import com.mobvista.apptag.utils.*;
 import net.sf.json.JSONArray;
 import net.sf.json.JSONObject;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.Model;
 import org.springframework.web.bind.annotation.*;
-
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -155,4 +168,82 @@ public class StorageController {
         model.addAttribute("userRole", user.getRole());
         return "storage/list";
     }
+
+    @PostMapping("/upload")
+    @ResponseBody
+    public String upload(@RequestParam("file") MultipartFile multipartFile) {
+
+        String rootPath = PropertyUtil.getProperty("app_tag.path");
+
+        if (multipartFile.isEmpty()) {
+            return "上传失败，请选择文件!";
+        }
+        String fileName = multipartFile.getOriginalFilename();
+        File dest = new File(rootPath + "/" + fileName);
+        if (dest.exists()) {
+            dest.delete();
+        }
+        try {
+            multipartFile.transferTo(dest);
+            logger.info("上传成功");
+            if (storageService.writeToMysql(dest)) {
+                String execId = execAZ();
+                if (StringUtils.isNotBlank(execId)) {
+                    return "上传成功,execId == >> " + execId;
+                } else {
+                    return "上传失败！";
+                }
+            }
+        } catch (IOException | URISyntaxException e) {
+            logger.error(e.toString(), e);
+        }
+        return "上传失败！";
+    }
+
+    public static String execAZ() throws URISyntaxException {
+
+        CloseableHttpClient client = HttpClients.createDefault();
+        List<BasicNameValuePair> formparams = new ArrayList<>();
+
+        String AZ_URL = PropertyUtil.getProperty("azkaban.server.http");
+        final String serverUrl = AZ_URL + "azkaban/execute";
+        URIBuilder uri = new URIBuilder();
+        try {
+            uri = new URIBuilder(serverUrl)
+                    .addParameter("cluster", "offline")
+                    .addParameter("projectName", "app_tag")
+                    .addParameter("flowName", "mysql2hive");
+        } catch (URISyntaxException e) {
+            e.printStackTrace();
+        }
+
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(1000).setConnectionRequestTimeout(1000)
+                .setSocketTimeout(1000).build();
+
+        final HttpGet httpGet = new HttpGet(uri.build());
+
+        String execId = "";
+        CloseableHttpResponse response;
+        try {
+            response = client.execute(httpGet);
+            BufferedReader rd = new BufferedReader(
+                    new InputStreamReader(response.getEntity().getContent()));
+            StringBuilder result = new StringBuilder();
+            String line;
+            while ((line = rd.readLine()) != null) {
+                result.append(line);
+            }
+            JsonObject jsonObject = GsonUtil.String2JsonObject(result.toString());
+            if (jsonObject.get("code").getAsInt() == 200) {
+                execId = jsonObject.get("data").getAsJsonObject().get("execId").getAsString();
+            }
+        } catch (IOException e) {
+            logger.info("Execute Failed!!!");
+        } finally {
+            httpGet.abort();
+        }
+
+        return execId;
+    }
 }
\ No newline at end of file

src/main/java/com/mobvista/apptag/controller/TreeTagController.java

@@ -100,7 +100,9 @@ public class TreeTagController {
         final String serverUrl = AZ_URL + "azkaban/execute";
         URIBuilder uri = new URIBuilder();
         try {
-            uri = new URIBuilder(serverUrl).addParameter("projectName", "dmp_on_k8s")
+            uri = new URIBuilder(serverUrl)
+                    .addParameter("cluster", "online")
+                    .addParameter("projectName", "dmp_on_k8s")
                     .addParameter("flowName", "app_tree_tag");
         } catch (URISyntaxException e) {
             e.printStackTrace();

src/main/java/com/mobvista/apptag/service/StorageService.java

@@ -3,6 +3,8 @@ package com.mobvista.apptag.service;
 import com.mobvista.apptag.entity.Storage;
 import com.mobvista.apptag.utils.Query;
 
+import java.io.File;
+import java.io.IOException;
 import java.util.List;
 
 /**
@@ -10,6 +12,7 @@ import java.util.List;
  */
 
 public interface StorageService {
+
     boolean update(String packageName, String userName, String json);
 
     Storage find(String packageName);
@@ -17,4 +20,6 @@ public interface StorageService {
     List<Storage> list(Query query);
 
     int count(String search);
+
+    boolean writeToMysql(File file);
 }
\ No newline at end of file

src/main/java/com/mobvista/apptag/service/impl/StorageServiceImpl.java

@@ -5,17 +5,18 @@ import com.github.pagehelper.PageHelper;
 import com.mobvista.apptag.entity.Storage;
 import com.mobvista.apptag.entity.Tag;
 import com.mobvista.apptag.entity.User;
+import com.mobvista.apptag.main.AppTagClass;
 import com.mobvista.apptag.mapper.StorageDao;
 import com.mobvista.apptag.mapper.TagDao;
 import com.mobvista.apptag.mapper.UserDao;
 import com.mobvista.apptag.service.StorageService;
 import com.mobvista.apptag.utils.Query;
+import com.mobvista.apptag.utils.UploadUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
+import java.io.File;
+import java.util.*;
 
 /**
  * @package: com.mobvista.apptag.service.impl
@@ -73,4 +74,39 @@ public class StorageServiceImpl implements StorageService {
     public int count(String search) {
         return storageDao.count(search);
     }
+
+    @Override
+    public boolean writeToMysql(File file) {
+        try {
+            List<AppTagClass> list = new ArrayList<>(UploadUtil.readTxtFileIntoStringSet(file).values());
+            if (list.size() % 1000 == 0) {
+                for (int i = 0; i < list.size() / 1000; i++) {
+                    int fromIndex = i * 1000;
+                    int toIndex;
+                    if (i == list.size() / 1000) {
+                        toIndex = list.size();
+                    } else {
+                        toIndex = (i + 1) * 1000;
+                    }
+                    Set<AppTagClass> set = new HashSet<>(list.subList(fromIndex, toIndex));
+                    UploadUtil.insert(set);
+                }
+            } else {
+                for (int i = 0; i < list.size() / 1000 + 1; i++) {
+                    int fromIndex = i * 1000;
+                    int toIndex;
+                    if (i == list.size() / 1000) {
+                        toIndex = list.size();
+                    } else {
+                        toIndex = (i + 1) * 1000;
+                    }
+                    Set<AppTagClass> set = new HashSet<>(list.subList(fromIndex, toIndex));
+                    UploadUtil.insert(set);
+                }
+            }
+            return true;
+        } catch (Exception e) {
+            return false;
+        }
+    }
 }

src/main/java/com/mobvista/apptag/utils/UploadUtil.java

+package com.mobvista.apptag.utils;
+
+import com.alibaba.fastjson.JSONObject;
+import com.mobvista.apptag.main.AppTagClass;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.ss.usermodel.*;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.regex.Pattern;
+
+/**
+ * @package: com.mobvista.apptag.utils
+ * @author: wangjf
+ * @date: 2021/6/10
+ * @time: 2:33 下午
+ * @email: jinfeng.wang@mobvista.com
+ */
+public class UploadUtil {
+
+    private static Logger logger = LoggerFactory.getLogger(UploadUtil.class);
+
+    private static final String CONNECTION_URL = "jdbc:mysql://dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com:3306/app_tag?useUnicode=true&characterEncoding=utf8&useSSL=false";
+    private static final String DRIVER = "com.mysql.jdbc.Driver";
+    private static final String USER = "apptag_rw";
+    private static final String PASSWORD = "7gyLEVtkER3u8c9";
+
+    private static final SimpleDateFormat SDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+    public static Map<String, AppTagClass> readTxtFileIntoStringSet(File file) {
+        Map<String, String> tmap = getMap();
+        Map<String, AppTagClass> map = new HashMap<>();
+        try {
+            if (file.isFile() && file.exists()) {
+                InputStream read = new FileInputStream(file);
+                String fileName = file.getName();
+                Workbook workbook = null;
+                if (fileName.toLowerCase().endsWith("xlsx")) {
+                    workbook = new XSSFWorkbook(read);
+                } else if (fileName.toLowerCase().endsWith("xls")) {
+                    workbook = new HSSFWorkbook(read);
+                }
+                Sheet sheet = workbook.getSheetAt(0);
+                List<List<String>> list = new ArrayList<>();
+                Iterator<Row> rowIterator = sheet.iterator();
+                while (rowIterator.hasNext()) {
+                    AppTagClass appTagClass = new AppTagClass();
+                    Row row = rowIterator.next();
+                    List<String> rowList = new ArrayList<>();
+                    Iterator<Cell> cellIterator = row.cellIterator();
+
+                    String packageName = getValue(row.getCell(0));
+                    String platform = getValue(row.getCell(1));
+                    String tag = getValue(row.getCell(4));
+                    if (StringUtils.isNotBlank(getValue(row.getCell(5)))) {
+                        tag = getValue(row.getCell(5));
+                    }
+                    if (map.containsKey(packageName)) {
+                        appTagClass = map.get(packageName);
+                        Set<String> set = appTagClass.getTag_list();
+                        set.add(tag);
+                        appTagClass.setTag_list(set);
+                    } else {
+                        appTagClass.setPackage_name(packageName);
+                        appTagClass.setApp_name("");
+                        appTagClass.setPlatform(platform);
+                        Set<String> set = new HashSet<>();
+                        set.add(tag);
+                        appTagClass.setTag_list(set);
+                    }
+
+                    String tagType = getValue(row.getCell(6));
+                    String firstTag = getValue(row.getCell(7));
+                    String secondTag = "";
+                    if (StringUtils.isNotBlank(getValue(row.getCell(8)))) {
+                        secondTag = getValue(row.getCell(8));
+                    }
+                    String tagId = "";
+                    if (StringUtils.isNotBlank(secondTag)) {
+                        if (tmap.containsKey((tagType + "-" + firstTag + "-" + secondTag).toUpperCase())) {
+                            tagId = tmap.get((tagType + "-" + firstTag + "-" + secondTag).toUpperCase());
+                        }
+                    } else {
+                        if (tmap.containsKey((tagType + "-" + firstTag).toUpperCase())) {
+                            tagId = tmap.get((tagType + "-" + firstTag).toUpperCase());
+                        }
+                    }
+
+                    if (StringUtils.isBlank(tagId)) {
+                        continue;
+                    }
+                    String[] tags = tagId.split("#");
+                    if (map.containsKey(packageName)) {
+                        appTagClass = map.get(packageName);
+                        JSONObject jsonObject = appTagClass.getTag_result();
+                        jsonObject.put(tags[0], tags[1]);
+                        appTagClass.setTag_result(jsonObject);
+                    } else {
+                        appTagClass.setPackage_name(packageName);
+                        appTagClass.setApp_name("");
+                        appTagClass.setPlatform(platform);
+                        JSONObject jsonObject = new JSONObject();
+                        jsonObject.put(tags[0], tags[1]);
+                        appTagClass.setTag_result(jsonObject);
+                    }
+                    map.put(packageName, appTagClass);
+                }
+                read.close();
+            } else {
+                logger.info("找不到指定的文件");
+            }
+        } catch (Exception e) {
+            logger.info("读取文件内容出错");
+            e.printStackTrace();
+        }
+        return map;
+    }
+
+    public static String getValue(Cell oneCell) {
+        String value = "";
+        if (oneCell == null || oneCell.getCellTypeEnum() == CellType.BLANK
+                || (oneCell.getCellTypeEnum() == CellType.STRING && oneCell.getStringCellValue().length() == 0)) {
+            if (oneCell == null) {
+            } else if (oneCell.getCellTypeEnum() == CellType.BLANK) {
+            } else if (oneCell.getCellTypeEnum() == CellType.STRING && oneCell.getStringCellValue().length() == 0) {
+            }
+        } else {
+            value = oneCell.getStringCellValue();
+        }
+        return value;
+    }
+
+    public static void insert(Set<AppTagClass> set) {
+        try {
+            Class.forName(DRIVER);
+            Connection con = DriverManager.getConnection(CONNECTION_URL, USER, PASSWORD);
+            // ON DUPLICATE KEY UPDATE tag = tag
+
+            String insertSql = "REPLACE INTO tag_list(package_name, app_name, platform, category, tag, flag, install) VALUES @VALUES";
+            StringBuilder stringBuilder = new StringBuilder();
+            Iterator<AppTagClass> iset = set.iterator();
+            while (iset.hasNext()) {
+                AppTagClass appTagClass = iset.next();
+                String platform;
+                String packageName = appTagClass.package_name;
+                String appName = appTagClass.app_name;
+                String category = String.join(",", appTagClass.tag_list);
+                Pattern iosPkgPtn = Pattern.compile("^id\\d+$");
+                Pattern adrPkgPtn = Pattern.compile("^[0-9a-zA-Z\\.\\_\\-]+$");
+                // Pattern otherAdrPkgPtn = Pattern.compile("^[\\_]+$");
+                if (iosPkgPtn.matcher(packageName).matches() || adrPkgPtn.matcher(packageName).matches()) {
+                    if (iosPkgPtn.matcher(packageName).matches()) {
+                        platform = "IOS";
+                        if (packageName != null && packageName.matches("^id\\d+$")) {
+                            packageName = packageName.replace("id", "");
+                        }
+                    } else {
+                        platform = "Android";
+                    }
+                    stringBuilder.append("('").append(packageName).append("','").append(appName).append("','")
+                            .append(platform).append("','").append(category).append("','").append(1).append("','")
+                            .append(1).append("',").append("30000000000),");
+                }
+            }
+
+            String instring = insertSql.replace("@VALUES", stringBuilder.substring(0, stringBuilder.length() - 1));
+            logger.info(instring);
+
+            PreparedStatement stmt = con.prepareStatement(instring);
+            stmt.execute();
+
+            insertSql = "REPLACE INTO storage(package_name, app_name, platform, tag, update_time) VALUES @VALUES";
+            stringBuilder = new StringBuilder();
+            iset = set.iterator();
+            logger.info("insert.size ==>> " + set.size());
+            while (iset.hasNext()) {
+                AppTagClass appTagClass = iset.next();
+                String platform;
+                String packageName = appTagClass.package_name;
+                String appName = appTagClass.app_name;
+                String tag = appTagClass.getTag_result().toJSONString();
+                Pattern iosPkgPtn = Pattern.compile("^id\\d+$");
+                Pattern adrPkgPtn = Pattern.compile("^[0-9a-zA-Z\\.\\_\\-]+$");
+                // Pattern adrPkgPtn = Pattern.compile("^\\w+\\..*$");
+                if (iosPkgPtn.matcher(packageName).matches() || adrPkgPtn.matcher(packageName).matches()) {
+                    if (iosPkgPtn.matcher(packageName).matches()) {
+                        platform = "IOS";
+                        if (packageName != null && packageName.matches("^id\\d+$")) {
+                            packageName = packageName.replace("id", "");
+                        }
+                    } else {
+                        platform = "Android";
+                    }
+                    stringBuilder.append("('").append(packageName).append("','").append(appName).append("','")
+                            .append(platform).append("','").append(tag).append("','").append(SDF.format(new Date())).append("'),");
+                }
+            }
+
+            instring = insertSql.replace("@VALUES", stringBuilder.substring(0, stringBuilder.length() - 1));
+            logger.info(instring);
+
+            PreparedStatement stmt0 = con.prepareStatement(instring);
+            stmt0.execute();
+            con.close();
+        } catch (Exception e) {
+            logger.info(e.getMessage());
+        }
+    }
+
+    public static Map<String, String> getMap() {
+        String filePath = "/home/mobdev/workspace/AppTag/src/main/resources/datas/apptagId.txt";
+        Map<String, String> map = new HashMap<>();
+        try {
+            String encoding = "UTF8";
+            File file = new File(filePath);
+            // 判断文件是否存在
+            if (file.isFile() && file.exists()) {
+                // 考虑到编码格式
+                InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);
+                BufferedReader bufferedReader = new BufferedReader(read);
+                String lineTxt;
+                while ((lineTxt = bufferedReader.readLine()) != null) {
+                    String[] appTags = lineTxt.split(";", -1);
+                    String tagType = appTags[0];
+                    String firstTag = appTags[1];
+                    String secondTag = appTags[2];
+                    String tagId;
+                    String tagName = appTags[3];
+
+                    if (StringUtils.isNotBlank(secondTag)) {
+                        tagId = appTags[9];
+                        map.put((tagType + "-" + firstTag + "-" + secondTag).toUpperCase(), tagId + "#" + tagName);
+                    }
+                    tagId = appTags[8];
+                    map.put((tagType + "-" + firstTag).toUpperCase(), tagId + "#" + tagName);
+                }
+                bufferedReader.close();
+                read.close();
+            } else {
+                logger.info("找不到指定的文件");
+            }
+        } catch (Exception e) {
+            logger.info("读取文件内容出错");
+            e.printStackTrace();
+        }
+        return map;
+    }
+}

src/main/resources/config.properties

 app_tag.path=/home/mobdev/workspace/AppTag/output/app_tag
 
-azkaban.server.http=http://ip-172-31-29-117:8688/
+# app_tag.path=/Users/wangjf/Workspace/AppTag/src/main/resources/datas
\ No newline at end of file
+
+azkaban.server.http=http://ip-172-31-29-117:8688/
+
+# azkaban.server.http=http://107.21.162.31:8688/
\ No newline at end of file

src/main/resources/templates/storage/list.html

@@ -12,6 +12,14 @@
         <div class="ibox">
             <div class="ibox-body">
                 <div class="fixed-table-toolbar">
+                    <form method="post" action="/storage/upload" enctype="multipart/form-data">
+                        <div class="columns pull-left col-md-3 nopadding">
+                            <input id="file" name="file" type="file" class="form-control">
+                        </div>
+                        <div class="columns pull-left">
+                            <button class="btn btn-success" type="submit">提交</button>
+                        </div>
+                    </form>
                     <div class="columns pull-right">
                         <button class="btn btn-success" onclick="reLoad(1)">查询</button>
                     </div>