Skip to content

A
AppTag
Commit 915f3b09 by jinfeng.wang
Options

fix

parent fa417647
Showing with 325 additions and 2 deletions
src/main/java/com/mobvista/apptag/main/AppTagClass.java 0 → 100644
package com.mobvista.apptag.main;
import com.alibaba.fastjson.JSONObject;
import java.util.Set;
/**
* @package: com.mobvista.apptag.utils
* @author: wangjf
* @date: 2020/3/16
* @time: 2:13 下午
* @email: jinfeng.wang@mobvista.com
* @phone: 152-1062-7698
*/
public class AppTagClass {
public String package_name;
public String app_name;
public String platform;
public Set<String> tag_list;
public JSONObject tag_result;
public String getPackage_name() {
return package_name;
}
public void setPackage_name(String package_name) {
this.package_name = package_name;
}
public String getApp_name() {
return app_name;
}
public void setApp_name(String app_name) {
this.app_name = app_name;
}
public String getPlatform() {
return platform;
}
public void setPlatform(String platform) {
this.platform = platform;
}
public Set<String> getTag_list() {
return tag_list;
}
public void setTag_list(Set<String> tag_list) {
this.tag_list = tag_list;
}
public JSONObject getTag_result() {
return tag_result;
}
public void setTag_result(JSONObject tag_result) {
this.tag_result = tag_result;
}
}
src/main/java/com/mobvista/apptag/main/AppTagMain.java 0 → 100644
package com.mobvista.apptag.main;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.util.*;
import java.util.regex.Pattern;
/**
* @package: com.mobvista.apptag.utils
* @author: wangjf
* @date: 2020/3/16
* @time: 2:12 下午
* @email: jinfeng.wang@mobvista.com
* @phone: 152-1062-7698
*/
public class AppTagMain {
private static final String CONNECTION_URL = "jdbc:mysql://dataplatform-app-tag.c5yzcdreb1xr.us-east-1.rds.amazonaws.com:3306/app_tag?useUnicode=true&characterEncoding=utf8&useSSL=false";
private static final String DRIVER = "com.mysql.jdbc.Driver";
private static final String USER = "apptag_rw";
private static final String PASSWORD = "7gyLEVtkER3u8c9";
public static void main(String[] args) {
try {
String filePath = "/Users/wangjf/Workspace/AppTag/src/main/resources/data/apptag_0909.txt";
List<AppTagClass> list = new ArrayList<>(readTxtFileIntoStringSet(filePath).values());
if (list.size() % 1000 == 0) {
for (int i = 0; i < list.size() / 1000; i++) {
int fromIndex = i * 1000;
int toIndex;
if (i == list.size() / 1000) {
toIndex = list.size();
} else {
toIndex = (i + 1) * 1000;
}
Set<AppTagClass> set = new HashSet<>(list.subList(fromIndex, toIndex));
insert(set);
}
} else {
for (int i = 0; i < list.size() / 1000 + 1; i++) {
int fromIndex = i * 1000;
int toIndex;
if (i == list.size() / 1000) {
toIndex = list.size();
} else {
toIndex = (i + 1) * 1000;
}
Set<AppTagClass> set = new HashSet<>(list.subList(fromIndex, toIndex));
insert(set);
}
}
} catch (Exception e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
}
static void insert(Set<AppTagClass> set) {
try {
Class.forName(DRIVER);
Connection con = DriverManager.getConnection(CONNECTION_URL, USER, PASSWORD);
// ON DUPLICATE KEY UPDATE tag = tag
String insertSql = "REPLACE INTO tag_list(package_name, app_name, platform, category, tag, flag, install) VALUES @VALUES";
StringBuilder stringBuilder = new StringBuilder();
Iterator<AppTagClass> iset = set.iterator();
while (iset.hasNext()) {
AppTagClass appTagClass = iset.next();
String platform;
String packageName = appTagClass.package_name;
String appName = appTagClass.app_name;
String category = String.join(",", appTagClass.tag_list);
Pattern iosPkgPtn = Pattern.compile("^id\\d+$");
Pattern adrPkgPtn = Pattern.compile("^[0-9a-zA-Z\\.\\_\\-]+$");
// Pattern otherAdrPkgPtn = Pattern.compile("^[\\_]+$");
if (iosPkgPtn.matcher(packageName).matches() || adrPkgPtn.matcher(packageName).matches()) {
if(iosPkgPtn.matcher(packageName).matches()){
platform = "IOS";
if (packageName != null && packageName.matches("^id\\d+$")) {
packageName = packageName.replace("id", "");
}
}else{
platform = "Android";
}
stringBuilder.append("('").append(packageName).append("','").append(appName).append("','")
.append(platform).append("','").append(category).append("','").append(1).append("','")
.append(1).append("',").append("30000000000),");
}
}
String instring = insertSql.replace("@VALUES", stringBuilder.substring(0, stringBuilder.length() - 1));
System.out.println(instring);
PreparedStatement stmt = con.prepareStatement(instring);
stmt.execute();
insertSql = "REPLACE INTO storage(package_name, app_name, platform, tag, update_time) VALUES @VALUES";
stringBuilder = new StringBuilder();
iset = set.iterator();
System.out.println("insert.size ==>> " + set.size());
while (iset.hasNext()) {
AppTagClass appTagClass = iset.next();
String platform;
String packageName = appTagClass.package_name;
String appName = appTagClass.app_name;
String tag = appTagClass.getTag_result().toJSONString();
Pattern iosPkgPtn = Pattern.compile("^id\\d+$");
Pattern adrPkgPtn = Pattern.compile("^[0-9a-zA-Z\\.\\_\\-]+$");
// Pattern adrPkgPtn = Pattern.compile("^\\w+\\..*$");
if (iosPkgPtn.matcher(packageName).matches() || adrPkgPtn.matcher(packageName).matches()) {
if(iosPkgPtn.matcher(packageName).matches()){
platform = "IOS";
if (packageName != null && packageName.matches("^id\\d+$")) {
packageName = packageName.replace("id", "");
}
}else{
platform = "Android";
}
stringBuilder.append("('").append(packageName).append("','").append(appName).append("','")
.append(platform).append("','").append(tag).append("','").append("2020-09-09 15:46:30'),");
}
}
instring = insertSql.replace("@VALUES", stringBuilder.substring(0, stringBuilder.length() - 1));
System.out.println(instring);
PreparedStatement stmt0 = con.prepareStatement(instring);
stmt0.execute();
con.close();
} catch (Exception e) {
System.out.println(e);
}
}
public static Map<String, AppTagClass> readTxtFileIntoStringSet(String filePath) {
Map<String, String> tmap = getMap();
Map<String, AppTagClass> map = new HashMap<>();
try {
File file = new File(filePath);
// 判断文件是否存在
if (file.isFile() && file.exists()) {
// 考虑到编码格式
InputStreamReader read = new InputStreamReader(new FileInputStream(file));
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt;
while ((lineTxt = bufferedReader.readLine()) != null) {
AppTagClass appTagClass = new AppTagClass();
String[] appTags = lineTxt.split(";");
String packageName = appTags[0];
String appName = appTags[1];
String platform = appTags[2];
String tag = appTags[5];
if (StringUtils.isNotBlank(appTags[6])) {
tag = appTags[6];
}
if (map.containsKey(packageName)) {
appTagClass = map.get(packageName);
Set<String> set = appTagClass.getTag_list();
set.add(tag);
appTagClass.setTag_list(set);
} else {
appTagClass.setPackage_name(packageName);
appTagClass.setApp_name(appName);
appTagClass.setPlatform(platform);
Set<String> set = new HashSet<>();
set.add(tag);
appTagClass.setTag_list(set);
}
String tagType = appTags[7];
String firstTag = appTags[8];
String secondTag = "";
if (appTags.length > 9 && StringUtils.isNotBlank(appTags[9])) {
secondTag = appTags[9];
}
String tagId = "";
if (StringUtils.isNotBlank(secondTag)) {
if (tmap.containsKey((tagType + "-" + firstTag + "-" + secondTag).toUpperCase())) {
tagId = tmap.get((tagType + "-" + firstTag + "-" + secondTag).toUpperCase());
}
} else {
if (tmap.containsKey((tagType + "-" + firstTag).toUpperCase())) {
tagId = tmap.get((tagType + "-" + firstTag).toUpperCase());
}
}
if (StringUtils.isBlank(tagId)) {
continue;
}
String[] tags = tagId.split("#");
if (map.containsKey(packageName)) {
appTagClass = map.get(packageName);
JSONObject jsonObject = appTagClass.getTag_result();
jsonObject.put(tags[0], tags[1]);
appTagClass.setTag_result(jsonObject);
} else {
appTagClass.setPackage_name(packageName);
appTagClass.setApp_name(appName);
appTagClass.setPlatform(platform);
JSONObject jsonObject = new JSONObject();
jsonObject.put(tags[0], tags[1]);
appTagClass.setTag_result(jsonObject);
}
map.put(packageName, appTagClass);
}
bufferedReader.close();
read.close();
} else {
System.out.println("找不到指定的文件");
}
} catch (Exception e) {
System.out.println("读取文件内容出错");
e.printStackTrace();
}
return map;
}
static Map<String, String> getMap() {
String filePath = "/Users/wangjf/Workspace/AppTag/src/main/resources/data/apptagId.txt";
Map<String, String> map = new HashMap<>();
try {
String encoding = "UTF8";
File file = new File(filePath);
// 判断文件是否存在
if (file.isFile() && file.exists()) {
// 考虑到编码格式
InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt;
while ((lineTxt = bufferedReader.readLine()) != null) {
String[] appTags = lineTxt.split(";");
String tagType = appTags[0];
String firstTag = appTags[1];
String secondTag = appTags[2];
String tagId;
String tagName = appTags[3];
if (StringUtils.isNotBlank(secondTag)) {
tagId = appTags[9];
map.put((tagType + "-" + firstTag + "-" + secondTag).toUpperCase(), tagId + "#" + tagName);
}
tagId = appTags[8];
map.put((tagType + "-" + firstTag).toUpperCase(), tagId + "#" + tagName);
}
bufferedReader.close();
read.close();
} else {
System.out.println("找不到指定的文件");
}
} catch (Exception e) {
System.out.println("读取文件内容出错");
e.printStackTrace();
}
return map;
}
}
src/main/resources/application.properties
......@@ -3,7 +3,7 @@ logging.level.root=INFO
logging.path=applog
mybatis.table.auto=update
mybatis.table.auto=none
mybatis.model.pack=com.mobvista.apptag.entity
......
src/main/resources/logback-spring.xml
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="60 seconds" debug="false">
<configuration scan="true" scanPeriod="60 seconds" debug="true">
<contextName>logback</contextName>
<!--输出到控制台-->
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment