1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
package mobvista.dmp.datasource.baichuan;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;
import mobvista.dmp.common.ClickHouseJdbc;
import parquet.org.slf4j.LoggerFactory;
import ru.yandex.clickhouse.ClickHouseConnection;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
/**
* @package: mobvista.dmp.datasource.baichuan
* @author: wangjf
* @date: 2019-08-31
* @time: 07:49
* @email: jinfeng.wang@mobvista.com
* @phone: 152-1062-7698
*/
public class DeviceMain {
public static void main(String[] args) throws JoranException, SQLException, IOException {
LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
JoranConfigurator configurator = new JoranConfigurator();
configurator.setContext(context);
context.reset();
configurator.doConfigure(BaiChuanServer.class.getClassLoader().getResourceAsStream("logback-syslog.xml"));
Logger logger = context.getLogger("device");
long start = System.currentTimeMillis();
// ClickHouseDataSource dataSource = new ClickHouseDataSource(url.replace("host", SET_VALUES[1]), properties);
// Connection connection = dataSource.getConnection();
// connection.setCatalog("dwh");
ClickHouseConnection connection = null;
ResultSet resultSet;
try {
connection = ClickHouseJdbc.connection();
} catch (SQLException | InterruptedException e) {
try {
connection = ClickHouseJdbc.connection();
} catch (SQLException | InterruptedException ex) {
logger.info("ClickHouse Connection Failure!");
}
}
assert connection != null;
Statement statement = connection.createStatement();
final String sql = "SELECT device_id FROM baichuan_install_daily_all WHERE dt = '2019-09-04' AND package_name = '0' GROUP BY device_id LIMIT 1100000";
resultSet = statement.executeQuery(sql);
Map<Integer, Set> map = new HashMap<>();
Set<String> set = new HashSet<>();
int i = 0;
logger.info(String.valueOf(start));
String file = "/Users/wangjf/Workspace/data/device_id.txt";
FileWriter fstream = new FileWriter(file, true);
BufferedWriter out = new BufferedWriter(fstream);
while (resultSet.next()) {
out.write(resultSet.getString("device_id"));
out.newLine();
}
out.close();
long end = System.currentTimeMillis();
logger.info("all runtime ==>> " + (end - start));
}
}