package mobvista.dmp.datasource.baichuan; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.joran.JoranConfigurator; import ch.qos.logback.core.joran.spi.JoranException; import com.alibaba.fastjson.JSONObject; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.gson.JsonObject; import mobvista.dmp.common.ClickHouseJdbc; import mobvista.dmp.util.DateUtil; import mobvista.dmp.util.PropertyUtil; import org.slf4j.LoggerFactory; import ru.yandex.clickhouse.ClickHouseConnection; import ru.yandex.clickhouse.ClickHouseDataSource; import ru.yandex.clickhouse.except.ClickHouseException; import ru.yandex.clickhouse.settings.ClickHouseProperties; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; /** * @package: mobvista.dmp.datasource.baichuan * @author: wangjf * @date: 2019-08-31 * @time: 07:49 * @email: jinfeng.wang@mobvista.com * @phone: 152-1062-7698 */ public class BaiChuanMainV3 { private static Set<AsoDevice> deviceInfoSet = new CopyOnWriteArraySet<>(); private static String dt = DateUtil.format(new Date(), "yyyy-MM-dd"); public static void main(String[] args) throws JoranException { LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory(); JoranConfigurator configurator = new JoranConfigurator(); configurator.setContext(context); context.reset(); configurator.doConfigure(BaiChuanServer.class.getClassLoader().getResourceAsStream("logback-syslog.xml")); Logger logger = context.getLogger("baichuan"); String appId = "2"; String appOs = "2"; int fors = 25000; if (args.length >= 2) { fors = Integer.parseInt(args[0]); dt = args[1]; } long start = System.currentTimeMillis(); int flag = 1; for (int c = 0; c < fors; c++) { Map<Integer, KV> map = Collections.synchronizedMap(new HashMap<>(200)); List<String> list = Collections.synchronizedList(new ArrayList<>()); Random random = new Random(); int finalIpId = random.nextInt(3); // String[] ips = SET_VALUES[finalIpId].split(":"); // ips[new Random().nextInt(2)] // ClickHouseDataSource dataSource = new ClickHouseDataSource(url.replace("host", ips[new Random().nextInt(2)]), properties); // ClickHouseConnection connection = null; // ResultSet resultSet; ClickHouseConnection connection = null; ResultSet resultSet; try { connection = ClickHouseJdbc.connection(); } catch (SQLException | InterruptedException e) { try { connection = ClickHouseJdbc.connection(); } catch (SQLException | InterruptedException ex) { logger.info("ClickHouse Connection Failure!"); } } String serverUrl = PropertyUtil.getProperty("config.properties", "dsp.taobao.server.url"); String channel = PropertyUtil.getProperty("config.properties", "dsp.taobao.server.channel"); String adid = PropertyUtil.getProperty("config.properties", "dsp.taobao.server.adid"); try { String date = DateUtil.getDayByString(dt, "yyyy-MM-dd", -1); assert connection != null; resultSet = ClickHouseUtils.queryExcludeAli(connection, date, 200 * c, 200); int i = 0; logger.info(String.valueOf(start)); while (resultSet.next()) { KV kv = new KV(); kv.setDeviceId(resultSet.getString("device_id")); kv.setAppOs(2); map.put(i++, kv); } ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() .setNameFormat("ali-%d").build(); /** * 因百川并发限制,设置总线程数为 200 */ ExecutorService aliPool = new ThreadPoolExecutor(200, 300, 120L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(128), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy()); // CloseableHttpClient client = HttpClients.createDefault(); for (int reqId = 0; reqId < 200; reqId++) { int finalReqId = reqId; aliPool.execute(() -> { try { // runJob(finalIpId, list.get(finalReqId)); SSLUtil.turnOffSsl(); runRequest(logger, serverUrl, channel, adid, appId, map.get(finalReqId)); Thread.sleep(100); } catch (InterruptedException | NoSuchAlgorithmException | KeyManagementException e) { e.printStackTrace(); } }); } flag += 1; if (flag % 10 == 0) { ClickHouseUtils.insert(connection, new AtomicReference<>(deviceInfoSet)); logger.info("forId = " + c + ",insert ipId = " + finalIpId + ",deviceInfoSet.size = " + deviceInfoSet.size()); deviceInfoSet = new CopyOnWriteArraySet<>(); Thread.sleep(1000); } aliPool.shutdown(); Thread.sleep(200); } catch (SQLException | InterruptedException e) { e.printStackTrace(); } finally { if (connection != null) { try { connection.close(); } catch (SQLException e) { e.printStackTrace(); } } } try { Thread.sleep(1000); } catch (InterruptedException e) { e.printStackTrace(); } } long end = System.currentTimeMillis(); logger.info("all runtime ==>> " + (end - start)); } private static void runJob(int ipId, String deviceId) { Random random = new Random(); int ran = random.nextInt(2); AsoDevice asoDevice = new AsoDevice(); asoDevice.setDeviceType("idfa"); asoDevice.setPlatform("ios"); asoDevice.setPackageName("387682726"); asoDevice.setDeviceId(deviceId); asoDevice.setDt(DateUtil.format(new Date(), "yyyy-MM-dd")); if (ran == 0) { deviceInfoSet.add(asoDevice); } } private static void runRequest(Logger logger, String serverUrl, String channel, String adid, String appId, KV kv) throws KeyManagementException, NoSuchAlgorithmException { // JsonObject jsonObject = BaiChuanServerV2.request(logger, serverUrl, channel, adid, kv); JSONObject jsonObject = new JSONObject(); if (jsonObject.containsKey("result")) { AsoDevice asoDevice = new AsoDevice(); if ("1".equals(appId)) { if (kv.getAppOs() == 1) { asoDevice.setDeviceId(kv.getDeviceId()); asoDevice.setDeviceType("imei"); asoDevice.setPlatform("android"); asoDevice.setPackageName("com.tmall.wireless"); } else { asoDevice.setDeviceId(kv.getDeviceId()); asoDevice.setDeviceType("idfa"); asoDevice.setPlatform("ios"); asoDevice.setPackageName("518966501"); } } else { if (kv.getAppOs() == 1) { asoDevice.setDeviceId(kv.getDeviceId()); asoDevice.setDeviceType("imei"); asoDevice.setPlatform("android"); asoDevice.setPackageName("com.taobao.taobao"); } else { asoDevice.setDeviceId(kv.getDeviceId()); asoDevice.setDeviceType("idfa"); asoDevice.setPlatform("ios"); asoDevice.setPackageName("387682726"); } } asoDevice.setDt(dt); if (jsonObject.getBoolean("result")) { asoDevice.setPackageName("0"); } deviceInfoSet.add(asoDevice); } } }