定时任务 删除appinfo重复数据

src/main/java/common/task/SyncAppDataTask.java

@@ -20,11 +20,7 @@ import org.apache.commons.collections.map.HashedMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import util.AwsS3Util;
-import util.Constant;
-import util.DateUtil;
-import util.HttpClientUtil;
-import util.StringUtil;
+import util.*;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
@@ -170,6 +166,7 @@ public class SyncAppDataTask
     }
 
     public void syncAppInfo() {
+        System.out.println("syncAppInfo start");
         appInfoRepository.deleteAll();
         String skey = "export/crawler_data/";
 
@@ -183,6 +180,14 @@ public class SyncAppDataTask
             String s3key = String.valueOf(itemList.get(i));
             readS3File(s3Util, s3key, "appInfo");
         }
+        System.out.println("delete start");
+        //删除重复数据
+        try{
+            deleteRepeatData();
+        } catch (Exception e){
+            logger.error("delete faild " + e.getMessage(), e);
+        }
+        System.out.println("delete end");
     }
 
 
@@ -363,6 +368,14 @@ public class SyncAppDataTask
             }
             return appInfo;
     }
+    //删除app_info的重复数据
+    void deleteRepeatData(){
+        String sql = "DELETE FROM app_info WHERE id IN (SELECT id FROM ( SELECT id as id FROM app_info" +
+                " WHERE pkg_name IN ( SELECT pkg_name FROM app_info GROUP BY pkg_name " +
+                "HAVING COUNT(*) > 1) AND id NOT IN ( SELECT min(id) FROM app_info  GROUP BY " +
+                "id HAVING count(*) > 1 )) b)";
+        DBUtil.newInstance().excute(sql);
+    };
 
     /*public static void main(String[] args)
     {

src/main/java/util/DBUtil.java

@@ -88,6 +88,19 @@ public class DBUtil
 		return rs;
 	}
 
+	public void excute(String sql) {
+		Connection conn = getConn();
+		Statement statement = null;
+		try {
+			statement = conn.createStatement();
+			boolean b = statement.execute(sql);
+		} catch (SQLException e) {
+			e.printStackTrace();
+		}finally {
+			release(statement,conn);
+		}
+	}
+	
 	public PreparedStatement getStatement(Connection conn, String sql) {
 		PreparedStatement pstmt = null;
 		try {
@@ -128,15 +141,7 @@ public class DBUtil
 	}
 	
 	
-	public void release(ResultSet rs,Statement pstmt, Connection conn) {
-		try {
-			if (rs != null) {
-				rs.close();
-				rs = null;
-			}
-		} catch (SQLException e) {
-			logger.error(e.getMessage(), e);
-		} finally {
+	public void release(Statement pstmt, Connection conn) {
 		try {
 			if (pstmt != null) {
 				pstmt.close();
@@ -154,7 +159,7 @@ public class DBUtil
 				logger.error(e.getMessage(), e);
 			}
 		}
-		}
+
 	}
 	
 	public void ExcuteNonQueryBatch(List<String> sqlList) {