update AppTag support oldTag upload

pom.xml

@@ -103,6 +103,16 @@
             <artifactId>aws-java-sdk-s3</artifactId>
             <version>1.11.437</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi</artifactId>
+            <version>3.17</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml</artifactId>
+            <version>3.17</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/java/com/mobvista/apptag/controller/StorageController.java

@@ -2,6 +2,7 @@ package com.mobvista.apptag.controller;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.JsonObject;
 import com.mobvista.apptag.config.WebSecurityConfig;
 import com.mobvista.apptag.entity.Feat;
 import com.mobvista.apptag.entity.Storage;
@@ -13,19 +14,31 @@ import com.mobvista.apptag.mapper.TagDao;
 import com.mobvista.apptag.mapper.UserDao;
 import com.mobvista.apptag.service.FeatService;
 import com.mobvista.apptag.service.StorageService;
-import com.mobvista.apptag.utils.PageUtil;
-import com.mobvista.apptag.utils.Query;
-import com.mobvista.apptag.utils.R;
+import com.mobvista.apptag.utils.*;
 import net.sf.json.JSONArray;
 import net.sf.json.JSONObject;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.Model;
 import org.springframework.web.bind.annotation.*;
-
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -155,4 +168,82 @@ public class StorageController {
         model.addAttribute("userRole", user.getRole());
         return "storage/list";
     }
+
+    @PostMapping("/upload")
+    @ResponseBody
+    public String upload(@RequestParam("file") MultipartFile multipartFile) {
+
+        String rootPath = PropertyUtil.getProperty("app_tag.path");
+
+        if (multipartFile.isEmpty()) {
+            return "上传失败，请选择文件!";
+        }
+        String fileName = multipartFile.getOriginalFilename();
+        File dest = new File(rootPath + "/" + fileName);
+        if (dest.exists()) {
+            dest.delete();
+        }
+        try {
+            multipartFile.transferTo(dest);
+            logger.info("上传成功");
+            if (storageService.writeToMysql(dest)) {
+                String execId = execAZ();
+                if (StringUtils.isNotBlank(execId)) {
+                    return "上传成功,execId == >> " + execId;
+                } else {
+                    return "上传失败！";
+                }
+            }
+        } catch (IOException | URISyntaxException e) {
+            logger.error(e.toString(), e);
+        }
+        return "上传失败！";
+    }
+
+    public static String execAZ() throws URISyntaxException {
+
+        CloseableHttpClient client = HttpClients.createDefault();
+        List<BasicNameValuePair> formparams = new ArrayList<>();
+
+        String AZ_URL = PropertyUtil.getProperty("azkaban.server.http");
+        final String serverUrl = AZ_URL + "azkaban/execute";
+        URIBuilder uri = new URIBuilder();
+        try {
+            uri = new URIBuilder(serverUrl)
+                    .addParameter("cluster", "offline")
+                    .addParameter("projectName", "app_tag")
+                    .addParameter("flowName", "mysql2hive");
+        } catch (URISyntaxException e) {
+            e.printStackTrace();
+        }
+
+        RequestConfig requestConfig = RequestConfig.custom()
+                .setConnectTimeout(1000).setConnectionRequestTimeout(1000)
+                .setSocketTimeout(1000).build();
+
+        final HttpGet httpGet = new HttpGet(uri.build());
+
+        String execId = "";
+        CloseableHttpResponse response;
+        try {
+            response = client.execute(httpGet);
+            BufferedReader rd = new BufferedReader(
+                    new InputStreamReader(response.getEntity().getContent()));
+            StringBuilder result = new StringBuilder();
+            String line;
+            while ((line = rd.readLine()) != null) {
+                result.append(line);
+            }
+            JsonObject jsonObject = GsonUtil.String2JsonObject(result.toString());
+            if (jsonObject.get("code").getAsInt() == 200) {
+                execId = jsonObject.get("data").getAsJsonObject().get("execId").getAsString();
+            }
+        } catch (IOException e) {
+            logger.info("Execute Failed!!!");
+        } finally {
+            httpGet.abort();
+        }
+
+        return execId;
+    }
 }
\ No newline at end of file

src/main/java/com/mobvista/apptag/controller/TreeTagController.java

@@ -100,7 +100,9 @@ public class TreeTagController {
         final String serverUrl = AZ_URL + "azkaban/execute";
         URIBuilder uri = new URIBuilder();
         try {
-            uri = new URIBuilder(serverUrl).addParameter("projectName", "dmp_on_k8s")
+            uri = new URIBuilder(serverUrl)
+                    .addParameter("cluster", "online")
+                    .addParameter("projectName", "dmp_on_k8s")
                     .addParameter("flowName", "app_tree_tag");
         } catch (URISyntaxException e) {
             e.printStackTrace();

src/main/java/com/mobvista/apptag/service/StorageService.java

@@ -3,6 +3,8 @@ package com.mobvista.apptag.service;
 import com.mobvista.apptag.entity.Storage;
 import com.mobvista.apptag.utils.Query;
 
+import java.io.File;
+import java.io.IOException;
 import java.util.List;
 
 /**
@@ -10,6 +12,7 @@ import java.util.List;
  */
 
 public interface StorageService {
+
     boolean update(String packageName, String userName, String json);
 
     Storage find(String packageName);
@@ -17,4 +20,6 @@ public interface StorageService {
     List<Storage> list(Query query);
 
     int count(String search);
+
+    boolean writeToMysql(File file);
 }
\ No newline at end of file

src/main/java/com/mobvista/apptag/service/impl/StorageServiceImpl.java

@@ -5,17 +5,18 @@ import com.github.pagehelper.PageHelper;
 import com.mobvista.apptag.entity.Storage;
 import com.mobvista.apptag.entity.Tag;
 import com.mobvista.apptag.entity.User;
+import com.mobvista.apptag.main.AppTagClass;
 import com.mobvista.apptag.mapper.StorageDao;
 import com.mobvista.apptag.mapper.TagDao;
 import com.mobvista.apptag.mapper.UserDao;
 import com.mobvista.apptag.service.StorageService;
 import com.mobvista.apptag.utils.Query;
+import com.mobvista.apptag.utils.UploadUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
+import java.io.File;
+import java.util.*;
 
 /**
  * @package: com.mobvista.apptag.service.impl
@@ -73,4 +74,39 @@ public class StorageServiceImpl implements StorageService {
     public int count(String search) {
         return storageDao.count(search);
     }
+
+    @Override
+    public boolean writeToMysql(File file) {
+        try {
+            List<AppTagClass> list = new ArrayList<>(UploadUtil.readTxtFileIntoStringSet(file).values());
+            if (list.size() % 1000 == 0) {
+                for (int i = 0; i < list.size() / 1000; i++) {
+                    int fromIndex = i * 1000;
+                    int toIndex;
+                    if (i == list.size() / 1000) {
+                        toIndex = list.size();
+                    } else {
+                        toIndex = (i + 1) * 1000;
+                    }
+                    Set<AppTagClass> set = new HashSet<>(list.subList(fromIndex, toIndex));
+                    UploadUtil.insert(set);
+                }
+            } else {
+                for (int i = 0; i < list.size() / 1000 + 1; i++) {
+                    int fromIndex = i * 1000;
+                    int toIndex;
+                    if (i == list.size() / 1000) {
+                        toIndex = list.size();
+                    } else {
+                        toIndex = (i + 1) * 1000;
+                    }
+                    Set<AppTagClass> set = new HashSet<>(list.subList(fromIndex, toIndex));
+                    UploadUtil.insert(set);
+                }
+            }
+            return true;
+        } catch (Exception e) {
+            return false;
+        }
+    }
 }

src/main/resources/config.properties

 app_tag.path=/home/mobdev/workspace/AppTag/output/app_tag
 
-azkaban.server.http=http://ip-172-31-29-117:8688/
\ No newline at end of file
+# app_tag.path=/Users/wangjf/Workspace/AppTag/src/main/resources/datas
+
+azkaban.server.http=http://ip-172-31-29-117:8688/
+
+# azkaban.server.http=http://107.21.162.31:8688/
\ No newline at end of file

src/main/resources/templates/storage/list.html

@@ -12,6 +12,14 @@
         <div class="ibox">
             <div class="ibox-body">
                 <div class="fixed-table-toolbar">
+                    <form method="post" action="/storage/upload" enctype="multipart/form-data">
+                        <div class="columns pull-left col-md-3 nopadding">
+                            <input id="file" name="file" type="file" class="form-control">
+                        </div>
+                        <div class="columns pull-left">
+                            <button class="btn btn-success" type="submit">提交</button>
+                        </div>
+                    </form>
                     <div class="columns pull-right">
                         <button class="btn btn-success" onclick="reLoad(1)">查询</button>
                     </div>