DatatoryAdnMain.java

package mobvista.dmp.datasource.datatory;

import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import mobvista.dmp.common.ClickHouseJdbc;
import mobvista.dmp.datasource.baichuan.AsoDevice;
import mobvista.dmp.datasource.baichuan.BaiChuanServer;
import mobvista.dmp.util.DateUtil;
import mobvista.dmp.util.MySQLUtil;
import mobvista.dmp.util.PropertyUtil;
import org.slf4j.LoggerFactory;
import ru.yandex.clickhouse.ClickHouseConnection;
import ru.yandex.clickhouse.ClickHouseDataSource;
import ru.yandex.clickhouse.ClickHousePreparedStatement;
import ru.yandex.clickhouse.except.ClickHouseException;
import ru.yandex.clickhouse.settings.ClickHouseProperties;

import java.sql.SQLException;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.*;

/**
 * @package: mobvista.dmp.datasource.datatory
 * @author: wangjf
 * @date: 2019-08-31
 * @time: 07:49
 * @email: jinfeng.wang@mobvista.com
 * @phone: 152-1062-7698
 */
public class DatatoryAdnMain {
    private static final String[] SET_VALUES = PropertyUtil.getProperty("config.properties", "http.private.server.ip").split(",");
    private static String driver = PropertyUtil.getProperty("config.properties", "datasource.clickhouse.driverClassName");
    private static String url = PropertyUtil.getProperty("config.properties", "datasource.clickhouse.url");
    private static String username = PropertyUtil.getProperty("config.properties", "datasource.clickhouse.username");
    private static String password = PropertyUtil.getProperty("config.properties", "datasource.clickhouse.password");
    private static String database = PropertyUtil.getProperty("config.properties", "datasource.clickhouse.database");
    private static int timeout = Integer.parseInt(PropertyUtil.getProperty("config.properties", "datasource.clickhouse.timeout"));

    private static Set<AsoDevice> deviceInfoSet = new CopyOnWriteArraySet<>();

    public static void main(String[] args) throws JoranException {
        LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
        JoranConfigurator configurator = new JoranConfigurator();
        configurator.setContext(context);
        context.reset();
        configurator.doConfigure(BaiChuanServer.class.getClassLoader().getResourceAsStream("logback-syslog.xml"));
        Logger logger = context.getLogger("Datatory.Adn");

        String date = "";
        String part = "";
        if (args.length >= 1) {
            part = args[0];
            date = DateUtil.format(DateUtil.parse(part, "yyyyMMdd"), "yyyy-MM-dd");
        } else {
            logger.info("Please Input Partition Date");
            System.exit(1);
        }
        long start = System.currentTimeMillis();
        ClickHouseProperties properties = new ClickHouseProperties();
        properties.setUser(username);
        properties.setPassword(password);
        properties.setDatabase(database);
        properties.setSocketTimeout(timeout);
        properties.setConnectionTimeout(timeout);

        ThreadFactory namedThreadFactory = new ThreadFactoryBuilder()
                .setNameFormat(" datatory-service-%d").build();

        ExecutorService datatoryPool = new ThreadPoolExecutor(3, 5,
                360L, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<>(1024), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy());

        int randomIp = new Random().nextInt(3);
        ClickHousePreparedStatement preparedStatementDrop = null;
        logger.info(dropPartition(part));
        ClickHouseConnection dropConnection = null;
        try {
            dropConnection = ClickHouseJdbc.connectionByTime(randomIp);
        } catch (SQLException | InterruptedException e) {
            try {
                dropConnection = ClickHouseJdbc.connectionByTime(randomIp);
            } catch (SQLException | InterruptedException ex) {
                logger.info("ClickHouse Connection Failure!");
            }
        }
        try {
            preparedStatementDrop = (ClickHousePreparedStatement) dropConnection.prepareStatement(dropPartition(part));
            preparedStatementDrop.execute();
            Thread.sleep(120000);
        } catch (SQLException | InterruptedException e) {
            e.printStackTrace();
        } finally {
            try {
                preparedStatementDrop.close();
                dropConnection.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }

        for (int ipId = 0; ipId < SET_VALUES.length; ipId++) {
            int finalIpId = ipId;
            String finalDate = date;
            datatoryPool.execute(() -> {
                String[] ips = SET_VALUES[finalIpId].split(":"); // ips[new Random().nextInt(2)]
                ClickHouseDataSource dataSource = new ClickHouseDataSource(url.replace("host", ips[new Random().nextInt(2)]), properties);
                ClickHouseConnection connection = null;
                ClickHousePreparedStatement preparedStatement = null;
                try {
                    try {
                        connection = ClickHouseJdbc.connectionByTime(finalIpId);
                    } catch (ClickHouseException e) {
                        Thread.sleep(200);
                    }
                    assert connection != null;

                    logger.info(buildSql(finalDate));
                    preparedStatement = (ClickHousePreparedStatement) connection.prepareStatement(buildSql(finalDate));
                    preparedStatement.execute();
                } catch (SQLException | InterruptedException e) {
                    e.printStackTrace();
                    System.exit(255);
                } finally {
                    if (connection != null) {
                        try {
                            connection.close();
                        } catch (SQLException e) {
                            e.printStackTrace();
                        }
                        try {
                            Thread.sleep(200);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                    }
                    if (preparedStatement != null) {
                        try {
                            preparedStatement.close();
                        } catch (SQLException e) {
                            e.printStackTrace();
                        }
                        try {
                            Thread.sleep(200);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                    }
                }
            });
        }

        long end = System.currentTimeMillis();
        logger.info("all runtime ==>> " + (end - start));
        datatoryPool.shutdown();
    }

    private static String buildSql(String date) {
        String partition = MySQLUtil.getLastPartition("dwh", "ods_user_info");
        String dt = DateUtil.format(DateUtil.parse(partition, "yyyyMMdd"), "yyyy-MM-dd");

        String insertSql = "INSERT INTO dwh.adn_tracking_insight_daily(device_id, device_model, os_version, country, city, age, gender, install_apps, interest, campaign_id, event_name, event_type, app_id, log_type,dt) " +
                "SELECT device_id,COALESCE(b.device_model,a.device_model) device_model,COALESCE(b.os_version,a.os_version) os_version,COALESCE(b.country,a.country) country,b.city,a.age,a.gender,install_apps,interest, " +
                "campaign_id, event_name, event_type, app_id, log_type,b.dt FROM (SELECT * FROM dwh.ods_user_info WHERE dt = '@dt' AND update_date = '@dt') a ALL INNER JOIN (SELECT * FROM dwh.adn_tracking_daily_all) b USING device_id WHERE b.dt = '@date'";

        return insertSql.replace("@date", date)
                .replace("@dt", dt)
                .replace("@update_date", dt);
    }

    private static String dropPartition(String part) {
        String dropSql = "ALTER TABLE dwh.adn_tracking_insight_daily ON CLUSTER cluster_1st DROP PARTITION @part";

        return dropSql.replace("@part", part);
    }
}