1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
package mobvista.dmp.datasource.iqiyi;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;
import com.alibaba.fastjson.JSONObject;
import com.google.common.util.concurrent.*;
import mobvista.dmp.util.DateUtil;
import mobvista.dmp.util.PropertyUtil;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.Date;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* @package: mobvista.dmp.datasource.iqiyi
* @author: wangjf
* @date: 2020/9/11
* @time: 10:02 下午
* @email: jinfeng.wang@mobvista.com
* @phone: 152-1062-7698
*/
public class IQiYiReadFile {
public static ch.qos.logback.classic.Logger logger = null;
private static final String INPUT = PropertyUtil.getProperty("config.properties", "iqiyi.lahuo.input.dir");
private static String dt = DateUtil.format(new Date(), "yyyyMMdd");
public static void main(String[] args) throws JoranException {
LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
JoranConfigurator configurator = new JoranConfigurator();
configurator.setContext(context);
context.reset();
configurator.doConfigure(IQiYiMain.class.getClassLoader().getResourceAsStream("logback-syslog.xml"));
logger = context.getLogger("IQiYiRequest");
ThreadPoolExecutor poolExecutor = new ThreadPoolExecutor(300, 500, 500, TimeUnit.MILLISECONDS,
new LinkedBlockingDeque<>(500), new CustomizableThreadFactory("IQiYiRequest-"), new ThreadPoolExecutor.CallerRunsPolicy());
int size = 1;
if (args.length >= 2) {
size = Integer.parseInt(args[0]);
dt = args[1];
}
for (int i = 0; i < size; i++) {
ListeningExecutorService listeningExecutor = MoreExecutors.listeningDecorator(poolExecutor);
MoreExecutors.addDelayedShutdownHook(listeningExecutor, 5, TimeUnit.SECONDS);
long start = System.currentTimeMillis();
String filePath = INPUT + dt + "_" + i + ".txt";
try {
List<ListenableFuture<String>> futures = new CopyOnWriteArrayList<>();
List<String> resultList = new CopyOnWriteArrayList<>();
File file = new File(filePath);
// 判断文件是否存在
if (file.isFile() && file.exists()) {
// 考虑到编码格式
InputStreamReader read = new InputStreamReader(new FileInputStream(file));
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt;
int j = 1;
long in_start = System.currentTimeMillis();
while ((lineTxt = bufferedReader.readLine()) != null) {
// logger.info(lineTxt);
final String deviceIds = lineTxt;
ListenableFuture listenableFuture = listeningExecutor.submit(() -> {
JSONObject jsonObject = IQiYiRequest.devicePeopleBatch(deviceIds);
return jsonObject.toJSONString();
});
Futures.addCallback(listenableFuture, new FutureCallback<String>() {
@Override
public void onSuccess(String result) {
resultList.add(result);
}
@Override
public void onFailure(Throwable t) {
resultList.add(new JSONObject().toJSONString());
}
});
futures.add(listenableFuture);
/*
if (j % 100000 == 0) {
ClickHouseConnection connection = null;
try {
connection = ClickHouseJdbc.connection();
} catch (SQLException e) {
try {
connection = ClickHouseJdbc.connection();
} catch (SQLException ex) {
logger.info("ClickHouse Connection Failure!");
}
}
try {
String datetime = DateUtil.format(new Date(), "yyyy-MM-dd HH");
IQiYiRequest.insertIQiYi(connection, futures, datetime.substring(0, 10), datetime.substring(11, 13));
logger.info("Times -->> " + resultList.size() / 100000 + ", ClickHouse Insert Success! Size -->> " + resultList.size());
} catch (SQLException e) {
logger.info("ClickHouse Insert Failure!");
}
long in_end = System.currentTimeMillis();
logger.info("Times -->> " + j / 100000 + ", Insert Size -->> " + futures.size() + ", Insert Run Time -->> " + (in_end - in_start));
futures = new CopyOnWriteArrayList<>();
in_start = System.currentTimeMillis();
}
*/
// j++;
}
bufferedReader.close();
read.close();
} else {
System.out.println("找不到指定的文件");
}
} catch (Exception e) {
System.out.println("读取文件内容出错");
e.printStackTrace();
}
poolExecutor.shutdown();
long end = System.currentTimeMillis();
logger.info("Read File." + filePath + " SUCCESS! Run Time -->> " + (end - start));
}
}
}