Skip to content

A
autodatacheck
Commit 0108269d by lipengbo
Options

修改错误报警插叙SQL

parent 49b51f14
Showing with 213 additions and 68 deletions
.idea/encodings.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<component name="Encoding" defaultCharsetForPropertiesFiles="UTF-8">
<file url="file://$PROJECT_DIR$" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/src/main/java/com/reyun/check/dao/MysqlDAO.java" charset="UTF-8" />
<file url="PROJECT" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
autodatacheck/checkconf/log4j.properties deleted 100644 → 0
### set log levels - for more verbose logging change 'info' to 'debug' ###
log4j.rootLogger=debug, stdout, file
### direct log messages to stdout ###
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p: %l - %m%n
log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.File=logs/etlengine.log
log4j.appender.file.MaxBackupIndex=10
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p: %l - %m%n
log4j.logger.org.apache=off
log4j.logger.kafka.utils.Logging=off
autodatacheck/checkconf/settings.properties deleted 100644 → 0
mailServerPort=25
MailServerHost=smtp.exmail.qq.com
MailUserName=reyunalarm@reyun.com
MailPassword=1391417Xk
MailFromAddress=reyunalarm@reyun.com
MailToUsers=lipengbo@reyun.com
###,wangpeiyang@reyun.com
###MailToUsers=etl@reyun.com
tkio_mid_install_day=tkio_truetime.tkio_mid_install_day,s3://reyundownload/tkio/tkio_mid_install_day
tkio_mid_reg_day=tkio_truetime.tkio_mid_reg_day,s3://reyundownload/tkio/tkio_mid_reg_day
tkio_mid_pay_day=tkio_truetime.tkio_mid_pay_day,s3://reyundownload/tkio/tkio_mid_pay_day
tkio_mid_clk_detail_day_down=tkio_offline.tkio_mid_clk_detail_day_down,s3://reyundownload/tkio/tkio_mid_clk_detail_day_down
accessKeyId=AKIAPMJ4QRXP6BECFUVQ
secretKey=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
mySQLConfConnectionDriver=com.mysql.jdbc.Driver
mySQLConfConnectionURL=jdbc:mysql://etl.cma5jkozme68.rds.cn-north-1.amazonaws.com.cn/tkio_v4
mySQLConfConnectionUser=reyun
mySQLConfConnectionPassword=Reyun123
mySQLDataConnectionDriver=com.mysql.jdbc.Driver
mySQLDataConnectionURL=jdbc:mysql://etl.cma5jkozme68.rds.cn-north-1.amazonaws.com.cn/tkio
mySQLDataConnectionUser=reyun
mySQLDataConnectionPassword=Reyun123
hiveTkioConnectionDriver=org.apache.hive.jdbc.HiveDriver
hiveTkioConnectionURL=jdbc:hive2://172.31.15.72:10000/default
hiveTkioConnectionUser=hive
hiveTkioConnectionPassword=
hiveTkioConnectionMaxIdle=20
maxWaitMillis=100000
maxTotal=2000
checkconf/settings.properties
......@@ -17,12 +17,15 @@ accessKeyId=AKIAPMJ4QRXP6BECFUVQ
secretKey=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
mySQLConfConnectionDriver=com.mysql.jdbc.Driver
mySQLConfConnectionURL=jdbc:mysql://etl.cma5jkozme68.rds.cn-north-1.amazonaws.com.cn/tkio_v4
mySQLConfConnectionUser=reyun
mySQLConfConnectionPassword=Reyun123
mySQLConfConnectionURL=jdbc:mysql://tkiohive.cma5jkozme68.rds.cn-north-1.amazonaws.com.cn/tkio_v4?characterEncoding=utf-8
mySQLConfConnectionUser=tkio_v4
mySQLConfConnectionPassword=Sr6SHYEqx
#mySQLConfConnectionURL=jdbc:mysql://etl.cma5jkozme68.rds.cn-north-1.amazonaws.com.cn/tkio?characterEncoding=utf-8
#mySQLConfConnectionUser=reyun
#mySQLConfConnectionPassword=Reyun123
mySQLDataConnectionDriver=com.mysql.jdbc.Driver
mySQLDataConnectionURL=jdbc:mysql://etl.cma5jkozme68.rds.cn-north-1.amazonaws.com.cn/tkio
mySQLDataConnectionURL=jdbc:mysql://etl.cma5jkozme68.rds.cn-north-1.amazonaws.com.cn/tkio?characterEncoding=utf-8
mySQLDataConnectionUser=reyun
mySQLDataConnectionPassword=Reyun123
......
src/main/java/com/reyun/check/App.java
package com.reyun.check;
import com.reyun.check.dao.MysqlDAO;
import com.reyun.check.dao.WarningDAO;
import com.reyun.check.mail.MailUtils;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import java.lang.reflect.InvocationTargetException;
import java.util.Map;
/**
* Hello world!
*
*/
public class App
{
public static void main( String[] args )
private static final Logger logger = Logger.getLogger(Main.class);
public static void main( String[] args )throws ClassNotFoundException
,NoSuchMethodException,InstantiationException,IllegalAccessException
,InvocationTargetException
{
System.out.println( "Hello World!" );
PropertyConfigurator.configure("checkconf/log4j.properties");
//ds,task_no,exec_sql,exec_state,error_info
String[] logarr=new String[5];
logarr[0]="9999-01-01";
logarr[1]="test";
logarr[2]="SQL语句";
logarr[3]="0";
logarr[4]="测试中文乱码";
MysqlDAO dao =new MysqlDAO();
dao.insertCheckLog(logarr);
/**
String ds="2018-03-01";
//发送告警邮件
String title ;
String content ;
MysqlDAO dao =new MysqlDAO();
Map<String,StringBuilder> sbmapw;
MailUtils mailUtils = new MailUtils(new WarningDAO().cfg.properties);
//检查数据校验任务执行-是否出错
title ="";
content="";
sbmapw=dao.getExecCheckIndicatorRst(ds);
if(sbmapw.get("title").toString().equals("")){
logger.info("检查数据校验任务执行-true");
}else{
title=sbmapw.get("title").toString();
content=sbmapw.get("content").toString();
mailUtils.sendMailContentEvery(title,content);
}
*/
}
}
src/main/java/com/reyun/check/Main.java
......@@ -53,6 +53,29 @@ public class Main {
MailUtils mailUtils = new MailUtils(new WarningDAO().cfg.properties);
mailUtils.sendMailContentEvery(title,content);
//检查数据校验任务执行-是否出错
title ="";
content="";
sbmapw=dao.getExecCheckTaskRst(ds);
if(sbmapw.get("title").toString().equals("")){
logger.info("检查数据校验任务执行-true");
}else{
title=sbmapw.get("title").toString();
content=sbmapw.get("content").toString();
mailUtils.sendMailContentEvery(title,content);
}
//检查按指标统计差执行-是否出错
title ="";
content="";
sbmapw=dao.getExecCheckIndicatorRst(ds);
if(sbmapw.get("title").toString().equals("")){
logger.info("检查按指标统计差异执行-true");
}else{
title=sbmapw.get("title").toString();
content=sbmapw.get("content").toString();
mailUtils.sendMailContentEvery(title,content);
}
//下载数据处理检查
new ExecThread().execDownloadCheck(4,ds);
//执行结束后删除10天以前的检查结果数据
......@@ -91,6 +114,30 @@ public class Main {
logger.info(content);
MailUtils mailUtils = new MailUtils(new WarningDAO().cfg.properties);
mailUtils.sendMailContentEvery(title,content);
//检查数据校验任务执行-是否出错
title ="";
content="";
sbmapw=dao.getExecCheckTaskRst(ds);
if(sbmapw.get("title").toString().equals("")){
logger.info("检查数据校验任务执行-true");
}else{
title=sbmapw.get("title").toString();
content=sbmapw.get("content").toString();
mailUtils.sendMailContentEvery(title,content);
}
//检查按指标统计差执行-是否出错
title ="";
content="";
sbmapw=dao.getExecCheckIndicatorRst(ds);
if(sbmapw.get("title").toString().equals("")){
logger.info("检查按指标统计差异执行-true");
}else{
title=sbmapw.get("title").toString();
content=sbmapw.get("content").toString();
mailUtils.sendMailContentEvery(title,content);
}
//执行结束后删除10天以前的检查结果数据
dao.deleteCheckRst(dsdel);
}
......@@ -141,7 +188,8 @@ public class Main {
}
taskno=cmd.getOptionValue("taskno");
//exectype="check";
//ds="2018-02-27";
//exectype="download";
//taskno="10001";
Main main =new Main();
......
src/main/java/com/reyun/check/dao/MysqlDAO.java
......@@ -507,14 +507,14 @@ public class MysqlDAO {
" ,group_concat(distinct t1.indicator_desc) indicator_list\n" +
"from\n" +
"(\n" +
" select distinct t1.task_no,t1.indicator_name,t2.indicator_desc,t3.task_desc,t4.maxrate\n" +
" select distinct t1.ds,t1.task_no,t1.indicator_name,t2.indicator_desc,t3.task_desc,t4.maxrate\n" +
" from tkio_data_check_result_indicator t1 \n" +
" inner join tkio_data_check_indicator t2 on t1.indicator_name=t2.indicator_name\n" +
" inner join tkio_data_check_config t3 on t1.task_no=t3.task_no\n" +
" left join tkio_data_check_indicator_value t4 on t1.task_no=t4.task_no and t1.indicator_name=t4.indicator_name\n" +
" where t1.ds='"+ds+"' and t1.rows_all_rate>t4.maxrate\n" +
" where t1.ds='"+ds+"' and t1.rows_all_rate>t4.maxrate \n" +
") t1\n" +
"inner join tkio_data_check_result_app t2 on t1.task_no=t2.task_no and t1.indicator_name=t2.indicator_name\n" +
"inner join tkio_data_check_result_app t2 on t1.ds=t2.ds and t1.task_no=t2.task_no and t1.indicator_name=t2.indicator_name\n" +
"where t2.indicator_rows_abn>0\n" +
"group by t2.appid \n" +
"order by count(distinct t1.indicator_desc) desc\n" +
......@@ -540,7 +540,7 @@ public class MysqlDAO {
content.append("<th>APPID</th>");
content.append("<th>异常指标数</th>");
content.append("<th>异常报表数</th>");
content.append("<th>异常指标报表</th>");
content.append("<th>异常指标列表</th>");
content.append("<th>异常报表列表</th>");
content.append("</tr>");
}
......@@ -555,6 +555,7 @@ public class MysqlDAO {
if (checkrstkey>0){
content.append("</table>");
title.append("***"+ds+"日数据异常，影响APP个数"+checkrstkey+"，请及时处理***");
String sqlindicator = " select t2.indicator_desc,t3.task_desc,t4.maxrate,t1.rows_all_rate\n" +
",t1.rows_all,t1.indicator_rows_abn,t1.apps_all,t1.indicator_apps_abn\n" +
"from tkio_data_check_result_indicator t1\n" +
......@@ -562,7 +563,8 @@ public class MysqlDAO {
"inner join tkio_data_check_config t3 on t1.task_no=t3.task_no\n" +
"left join tkio_data_check_indicator_value t4 on t1.task_no=t4.task_no and t1.indicator_name=t4.indicator_name\n" +
"where t1.ds='"+ds+"' and t1.rows_all_rate>t4.maxrate ";
pstmt = conn.prepareStatement(sql);
pstmt = conn.prepareStatement(sqlindicator);
rs = pstmt.executeQuery();
Integer i=1;
while (rs.next()) {
......@@ -593,8 +595,10 @@ public class MysqlDAO {
content.append("<td>"+rs.getString("INDICATOR_APPS_ABN")+"</td>");
content.append("</tr>");
}
if (i>0){
content.append("</table>");
}
}
}catch(Exception ex){
logger.info(ex.toString());
......@@ -668,4 +672,101 @@ public class MysqlDAO {
return sbmap;
}
public Map<String,StringBuilder> getExecCheckTaskRst(String ds) {
Connection conn = null;
PreparedStatement pstmt = null;
ResultSet rs = null;
String sql = " select t1.task_no,t1.task_desc,t3.row_id\n" +
",if(t3.row_id is null,'no exec',ifnull(t3.error_info,'success')) error_info" +
",DATE_ADD(t3.create_time,INTERVAL 8 hour) create_time \n" +
"from tkio_data_check_config t1\n" +
"left join tkio_data_check_log t2 on t1.task_no=t2.task_no \n" +
" and t2.exec_state=0 and t2.ds='"+ds+"'\n" +
"left join tkio_data_check_log t3 on t1.task_no=t3.task_no \n" +
" and t3.ds='"+ds+"'\n" +
"where t1.is_delete=0 and t2.row_id is null\n" +
"order by t1.task_no,t3.create_time";
Map<String,StringBuilder> sbmap=new HashMap<String, StringBuilder>();
StringBuilder title = new StringBuilder();
StringBuilder content = new StringBuilder();
try{
//sql=new String(sql.getBytes("ISO-8859-1"),"UTF-8");//将其转码为UTF-8
conn = this.cfg.getUtils().getMySQLConfConnection();
//conn = this.cfg.getUtils().getMySQLDataConnection();
logger.info(sql);
pstmt = conn.prepareStatement(sql);
rs = pstmt.executeQuery();
Integer checkrstkey=1;
while (rs.next()) {
if (checkrstkey==1){
title.append(ds+"日数据校验任务执行出错");
}
checkrstkey++;
content.append("<tr>");
content.append("<=========================================><br />");
content.append("<b>任务: ["+rs.getString("TASK_NO")+"]"+rs.getString("TASK_DESC")+"</b><br />");
content.append("<b>执行时间: ["+rs.getString("CREATE_TIME")+"]</b><br />");
content.append("<b>错误描述: ["+rs.getString("ERROR_INFO")+"]</b><br />");
}
}catch(Exception ex){
logger.info(ex.toString());
}finally {
this.cfg.getUtils().release(rs, pstmt, conn);
}
sbmap.put("title",title);
sbmap.put("content",content);
return sbmap;
}
public Map<String,StringBuilder> getExecCheckIndicatorRst(String ds) {
Connection conn = null;
PreparedStatement pstmt = null;
ResultSet rs = null;
String sql = " select t1.indicator_name task_no,t1.indicator_desc task_desc,t3.row_id\n" +
" ,if(t3.row_id is null,'no exec',ifnull(t3.error_info,'success')) error_info" +
" ,DATE_ADD(t3.create_time,INTERVAL 8 hour) create_time \n" +
" from tkio_data_check_indicator t1\n" +
" left join tkio_data_check_log t2 on t1.indicator_name=t2.task_no \n" +
" and t2.exec_state=0 and t2.ds='"+ds+"' \n" +
" left join tkio_data_check_log t3 on t1.indicator_name=t3.task_no \n" +
" and t3.ds='"+ds+"'\n" +
" where t1.is_delete=0 and t2.row_id is null\n" +
" order by t1.indicator_name,t3.create_time";
Map<String,StringBuilder> sbmap=new HashMap<String, StringBuilder>();
StringBuilder title = new StringBuilder();
StringBuilder content = new StringBuilder();
try{
//sql=new String(sql.getBytes("ISO-8859-1"),"UTF-8");//将其转码为UTF-8
conn = this.cfg.getUtils().getMySQLConfConnection();
//conn = this.cfg.getUtils().getMySQLDataConnection();
logger.info(sql);
pstmt = conn.prepareStatement(sql);
rs = pstmt.executeQuery();
Integer checkrstkey=1;
while (rs.next()) {
if (checkrstkey==1){
title.append(ds+"日检查按指标统计差执行出错");
}
checkrstkey++;
content.append("<tr>");
content.append("<=========================================><br />");
content.append("<b>任务: ["+rs.getString("TASK_NO")+"]"+rs.getString("TASK_DESC")+"</b><br />");
content.append("<b>执行时间: ["+rs.getString("CREATE_TIME")+"]</b><br />");
content.append("<b>错误描述: ["+rs.getString("ERROR_INFO")+"]</b><br />");
}
}catch(Exception ex){
logger.info(ex.toString());
}finally {
this.cfg.getUtils().release(rs, pstmt, conn);
}
sbmap.put("title",title);
sbmap.put("content",content);
return sbmap;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment