package mobvista.dmp.datasource.baichuan; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.joran.JoranConfigurator; import ch.qos.logback.core.joran.spi.JoranException; import mobvista.dmp.common.ClickHouseJdbc; import parquet.org.slf4j.LoggerFactory; import ru.yandex.clickhouse.ClickHouseConnection; import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; /** * @package: mobvista.dmp.datasource.baichuan * @author: wangjf * @date: 2019-08-31 * @time: 07:49 * @email: jinfeng.wang@mobvista.com * @phone: 152-1062-7698 */ public class DeviceMain { public static void main(String[] args) throws JoranException, SQLException, IOException { LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory(); JoranConfigurator configurator = new JoranConfigurator(); configurator.setContext(context); context.reset(); configurator.doConfigure(BaiChuanServer.class.getClassLoader().getResourceAsStream("logback-syslog.xml")); Logger logger = context.getLogger("device"); long start = System.currentTimeMillis(); // ClickHouseDataSource dataSource = new ClickHouseDataSource(url.replace("host", SET_VALUES[1]), properties); // Connection connection = dataSource.getConnection(); // connection.setCatalog("dwh"); ClickHouseConnection connection = null; ResultSet resultSet; try { connection = ClickHouseJdbc.connection(); } catch (SQLException | InterruptedException e) { try { connection = ClickHouseJdbc.connection(); } catch (SQLException | InterruptedException ex) { logger.info("ClickHouse Connection Failure!"); } } assert connection != null; Statement statement = connection.createStatement(); final String sql = "SELECT device_id FROM baichuan_install_daily_all WHERE dt = '2019-09-04' AND package_name = '0' GROUP BY device_id LIMIT 1100000"; resultSet = statement.executeQuery(sql); Map<Integer, Set> map = new HashMap<>(); Set<String> set = new HashSet<>(); int i = 0; logger.info(String.valueOf(start)); String file = "/Users/wangjf/Workspace/data/device_id.txt"; FileWriter fstream = new FileWriter(file, true); BufferedWriter out = new BufferedWriter(fstream); while (resultSet.next()) { out.write(resultSet.getString("device_id")); out.newLine(); } out.close(); long end = System.currentTimeMillis(); logger.info("all runtime ==>> " + (end - start)); } }