1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
package mobvista.dmp.datasource.monitor;
import ch.qos.logback.core.joran.spi.JoranException;
import mobvista.dmp.common.ClickHouseJdbc;
import mobvista.dmp.util.DateUtil;
import ru.yandex.clickhouse.ClickHouseConnection;
import ru.yandex.clickhouse.ClickHousePreparedStatement;
import ru.yandex.clickhouse.except.ClickHouseException;
import java.sql.SQLException;
import java.util.Random;
/**
* @package: mobvista.dmp.datasource.monitor
* @author: wangjf
* @date: 2019-08-31
* @time: 07:49
* @email: jinfeng.wang@mobvista.com
* @phone: 152-1062-7698
*/
public class ResourceMonitor {
public static void main(String[] args) throws JoranException {
String date = "";
String part = "";
if (args.length >= 1) {
part = args[0];
date = DateUtil.format(DateUtil.parse(part, "yyyyMMdd"), "yyyy-MM-dd");
} else {
System.exit(1);
}
long start = System.currentTimeMillis();
Random random = new Random();
int selectIp = random.nextInt(3);
ClickHouseConnection connection = null;
ClickHousePreparedStatement preparedStatement = null;
ClickHousePreparedStatement preparedStatementDrop = null;
try {
try {
connection = ClickHouseJdbc.connectionByTime(selectIp);
} catch (ClickHouseException e) {
System.exit(255);
}
assert connection != null;
// drop partition
preparedStatementDrop = (ClickHousePreparedStatement) connection.prepareStatement(dropPartitionSql(part));
preparedStatementDrop.execute();
Thread.sleep(120000);
// insert data
preparedStatement = (ClickHousePreparedStatement) connection.prepareStatement(insertDataSql(date));
preparedStatement.execute();
} catch (SQLException | InterruptedException e) {
System.exit(255);
} finally {
try {
connection.close();
preparedStatement.close();
preparedStatementDrop.close();
System.exit(0);
} catch (SQLException throwables) {
System.exit(255);
}
}
}
private static String insertDataSql(String date) {
String insertSql = "INSERT INTO dataplatform_job_meta.yarn_application_metric_all(application_time, master_name, master_address, user_name,\n" +
" queue_name, application_id,\n" +
" application_name, application_type, application_status,\n" +
" application_core, application_memory,\n" +
" application_start_time, application_stop_time,\n" +
" application_elapsed_time, application_read_byte,\n" +
" application_write_byte, application_input_path,\n" +
" application_output_path, email, direct_leader, department)\n" +
"SELECT toUInt64(application_time) application_time,\n" +
" master_name,\n" +
" master_address,\n" +
" user_name,\n" +
" queue_name,\n" +
" toUInt64(replaceAll(replaceAll(application_id, 'application', ''), '_', '')) application_id,\n" +
" application_name,\n" +
" application_type,\n" +
" application_status,\n" +
" application_core,\n" +
" application_memory,\n" +
" application_start_time,\n" +
" application_stop_time,\n" +
" application_elapsed_time,\n" +
" application_read_byte,\n" +
" application_write_byte,\n" +
" application_input_path,\n" +
" application_output_path,\n" +
" email,\n" +
" direct_leader,\n" +
" department\n" +
"FROM dataplatform_job_meta.yarn_application_msg\n" +
"WHERE toDate(application_time / 1000) = '@date'";
return insertSql.replace("@date", date);
}
private static String dropPartitionSql(String part) {
String dropSql = "ALTER TABLE dataplatform_job_meta.yarn_application_metric ON CLUSTER cluster_1st DROP PARTITION @part";
return dropSql.replace("@part", part);
}
}