package mobvista.dmp.datasource.baichuan; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.joran.JoranConfigurator; import ch.qos.logback.core.joran.spi.JoranException; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.taobao.api.ApiException; import com.taobao.api.response.AlibabaBaichuanAsoQueryResponse; import mobvista.dmp.common.ClickHouseJdbc; import mobvista.dmp.util.DateUtil; import org.slf4j.LoggerFactory; import ru.yandex.clickhouse.ClickHouseConnection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; /** * @package: mobvista.dmp.datasource.baichuan * @author: wangjf * @date: 2019-08-31 * @time: 07:49 * @email: jinfeng.wang@mobvista.com * @phone: 152-1062-7698 */ public class BaiChuanMain { private static Set<AsoDevice> deviceInfoSet = new CopyOnWriteArraySet<>(); private static String dt = DateUtil.format(new Date(), "yyyy-MM-dd"); public static void main(String[] args) throws JoranException { LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory(); JoranConfigurator configurator = new JoranConfigurator(); configurator.setContext(context); context.reset(); configurator.doConfigure(BaiChuanServer.class.getClassLoader().getResourceAsStream("logback-syslog.xml")); Logger logger = context.getLogger("baichuan"); String appId = "2"; String appOs = "2"; int fors = 1500; if (args.length >= 3) { appId = args[0]; appOs = args[1]; fors = Integer.parseInt(args[2]); dt = args[3]; } long start = System.currentTimeMillis(); for (int c = 0; c < fors; c++) { Map<Integer, Set<String>> map = Collections.synchronizedMap(new HashMap<>(10)); Random random = new Random(); int finalIpId = random.nextInt(3); String finalAppId = appId; String finalAppOs = appOs; // ips[new Random().nextInt(2)] String[] ips = ClickHouseJdbc.SET_VALUES[finalIpId].split(":"); // ClickHouseDataSource dataSource = new ClickHouseDataSource(ClickHouseJdbc.URL.replace("host", ips[new Random().nextInt(2)]), properties); // ClickHouseConnection connection = null; // ResultSet resultSet; ClickHouseConnection connection = null; ResultSet resultSet; try { connection = ClickHouseJdbc.connection(); } catch (SQLException | InterruptedException e) { try { connection = ClickHouseJdbc.connection(); } catch (SQLException | InterruptedException ex) { logger.info("ClickHouse Connection Failure!"); } } try { String date = DateUtil.getDayByString(dt, "yyyy-MM-dd", -1); assert connection != null; resultSet = ClickHouseUtils.query(connection, date, finalAppId, finalAppOs, 4000 * c, 4000); int i = 0; logger.info(String.valueOf(start)); Set<String> set = Collections.synchronizedSet(new HashSet<>()); while (resultSet.next()) { set.add(resultSet.getString("device_id")); /** * the array size of deviceInfoList must be less than 20.so,size of deviceId must less than 20 */ if (set.size() % 20 == 0) { map.put(i, set); set = Collections.synchronizedSet(new HashSet<>()); i = i + 1; } } ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() .setNameFormat("ali-%d").build(); /** * 因百川并发限制,设置总线程数为 200 */ ExecutorService aliPool = new ThreadPoolExecutor(200, 300, 120L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1024), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy()); for (int reqId = 0; reqId < 200; reqId++) { int finalReqId = reqId; aliPool.execute(() -> { // runJob(finalIpId, map.get(finalReqId)); try { if (map.keySet().contains(finalReqId)) { runRequest(finalAppId, finalAppOs, map.get(finalReqId)); } Thread.sleep(200); } catch (ApiException | InterruptedException e) { e.printStackTrace(); } }); } ClickHouseUtils.insert(connection, new AtomicReference<>(deviceInfoSet)); logger.info("forId = " + c + ",insert ipId = " + finalIpId + ",deviceInfoSet.size = " + deviceInfoSet.size()); deviceInfoSet = new CopyOnWriteArraySet<>(); aliPool.shutdown(); Thread.sleep(200); } catch (SQLException | InterruptedException e) { e.printStackTrace(); } finally { if (connection != null) { try { connection.close(); Thread.sleep(200); } catch (SQLException | InterruptedException e) { e.printStackTrace(); } } } try { Thread.sleep(200); } catch (InterruptedException e) { e.printStackTrace(); } } long end = System.currentTimeMillis(); logger.info("all runtime ==>> " + (end - start)); } private static void runJob(int ipId, Set<String> deviceSet) { Random random = new Random(); int ran = random.nextInt(2); for (String deviceId : deviceSet) { AsoDevice asoDevice = new AsoDevice(); asoDevice.setDeviceType("idfa"); asoDevice.setPlatform("ios"); asoDevice.setPackageName("387682726"); asoDevice.setDeviceId(deviceId); asoDevice.setDt(DateUtil.format(new Date(), "yyyy-MM-dd")); if (ran == 0) { deviceInfoSet.add(asoDevice); } } } private static void runRequest(String appId, String appOs, Set<String> deviceSet) throws ApiException { AlibabaBaichuanAsoQueryResponse rsp = BaiChuanServer.request(appId, appOs, deviceSet); if (rsp != null && rsp.getResult() != null && rsp.getResult().getSuccess()) { for (AlibabaBaichuanAsoQueryResponse.AsoDeviceCheckResult checkResult : rsp.getResult().getResults()) { AsoDevice asoDevice = new AsoDevice(); if ("1".equals(appId)) { if ("1".equals(appOs)) { asoDevice.setDeviceId(checkResult.getImei()); asoDevice.setDeviceType("imei"); asoDevice.setPlatform("android"); asoDevice.setPackageName("com.tmall.wireless"); } else { asoDevice.setDeviceId(checkResult.getIdfa()); asoDevice.setDeviceType("idfa"); asoDevice.setPlatform("ios"); asoDevice.setPackageName("518966501"); } } else { if ("1".equals(appOs)) { asoDevice.setDeviceId(checkResult.getImei()); asoDevice.setDeviceType("imei"); asoDevice.setPlatform("android"); asoDevice.setPackageName("com.taobao.taobao"); } else { asoDevice.setDeviceId(checkResult.getIdfa()); asoDevice.setDeviceType("idfa"); asoDevice.setPlatform("ios"); asoDevice.setPackageName("387682726"); } } // asoDevice.setDt(DateUtil.format(new Date(), "yyyy-MM-dd")); asoDevice.setDt(dt); if (checkResult.getIsNewDevice()) { asoDevice.setPackageName("0"); } deviceInfoSet.add(asoDevice); } } } }