OPENTECH / yoho-node-ci · Commits

Go to a project

Authored by 姜枫
Commit 83957c20a0cf4062bda48f0901798adf90a4a710 1 parent c10c9e9f

add log scanner

Inline Side-by-side
Showing 10 changed files with 442 additions and 6 deletions
@@ -8,7 +8,7 @@ const influx = require('influx'); @@ -8,7 +8,7 @@ const influx = require('influx');
8 8
9 let client = influx({ 9 let client = influx({
10 hosts: [{ 10 hosts: [{
11 - host: 'influxdblog.yohoops.org', 11 + host: '54.222.219.223',
12 port: 8086, 12 port: 8086,
13 protocol: 'http' 13 protocol: 'http'
14 }], 14 }],
@@ -28,6 +28,18 @@ const db = { @@ -28,6 +28,18 @@ const db = {
28 } 28 }
29 }); 29 });
30 }); 30 });
  31 + },
  32 +
  33 + createContinuousQuery: (key, query) => {
  34 + return new Promise((resolve, reject) => {
  35 + client.createContinuousQuery(key, query, (err, result) => {
  36 + if (err) {
  37 + reject(err);
  38 + } else {
  39 + resolve(result);
  40 + }
  41 + });
  42 + });
31 } 43 }
32 }; 44 };
33 45
  1 +/**
  2 + *
  3 + * @author: jiangfeng<jeff.jiang@yoho.cn>
  4 + * @date: 16/8/31
  5 + */
  6 +
  7 +"use strict";
  8 +
  9 +const schedule = require('node-schedule');
  10 +
  11 +const jobMap = new Map();
  12 +
  13 +const jobs = {
  14 + run(key, cron, fn) {
  15 + let job = schedule.scheduleJob(cron, fn);
  16 +
  17 + jobMap.set(key, job);
  18 + },
  19 +
  20 + cancel(key) {
  21 + let job = jobMap.get(key);
  22 +
  23 + schedule.cancelJob(job);
  24 + jobMap.delete(key);
  25 + },
  26 +
  27 + getJobs() {
  28 + return jobMap;
  29 + }
  30 +};
  31 +
  32 +module.exports = jobs;
@@ -8,6 +8,7 @@ import DeployModel from './deploy'; @@ -8,6 +8,7 @@ import DeployModel from './deploy';
8 import UserModel from './user'; 8 import UserModel from './user';
9 import HotfixModel from './hotfix'; 9 import HotfixModel from './hotfix';
10 import OperationLoggerModel from './operation_logger'; 10 import OperationLoggerModel from './operation_logger';
  11 +import LogScannerModel from './log_scanner';
11 12
12 shelljs.mkdir('-p', config.dbDir); 13 shelljs.mkdir('-p', config.dbDir);
13 14
@@ -18,6 +19,7 @@ const DeployInfo = new DeployModel(); @@ -18,6 +19,7 @@ const DeployInfo = new DeployModel();
18 const User = new UserModel(); 19 const User = new UserModel();
19 const Hotfix = new HotfixModel(); 20 const Hotfix = new HotfixModel();
20 const OperationLogger = new OperationLoggerModel(); 21 const OperationLogger = new OperationLoggerModel();
  22 +const LogScanner = new LogScannerModel();
21 23
22 User.init(); 24 User.init();
23 25
@@ -28,5 +30,6 @@ export { @@ -28,5 +30,6 @@ export {
28 DeployInfo, 30 DeployInfo,
29 User, 31 User,
30 Hotfix, 32 Hotfix,
31 - OperationLogger 33 + OperationLogger,
  34 + LogScanner
32 }; 35 };
  1 +/**
  2 + *
  3 + * @author: jiangfeng<jeff.jiang@yoho.cn>
  4 + * @date: 16/8/31
  5 + */
  6 +
  7 +'use strict';
  8 +
  9 +import Model from './model';
  10 +
  11 +class LogScanner extends Model {
  12 +
  13 + constructor() {
  14 + super('log_scanner');
  15 + }
  16 +
  17 +}
  18 +
  19 +export default LogScanner;
@@ -9,10 +9,16 @@ @@ -9,10 +9,16 @@
9 import Router from 'koa-router'; 9 import Router from 'koa-router';
10 10
11 import { 11 import {
12 - Project 12 + Project,
  13 + LogScanner
13 } from '../../models'; 14 } from '../../models';
14 15
15 import InfluxDB from '../../logger/influxdb'; 16 import InfluxDB from '../../logger/influxdb';
  17 +import Operation from '../../logger/operation';
  18 +import Schedulers from '../../logger/schedulers';
  19 +import ScannerRunner from './scanner_runner';
  20 +
  21 +import _ from 'lodash';
16 22
17 const r = new Router(); 23 const r = new Router();
18 24
@@ -27,7 +33,7 @@ const monitor = { @@ -27,7 +33,7 @@ const monitor = {
27 name: '线上环境', 33 name: '线上环境',
28 value: 'production', 34 value: 'production',
29 checked: env === 'production' 35 checked: env === 'production'
30 - },{ 36 + }, {
31 name: '灰度环境', 37 name: '灰度环境',
32 value: 'preview', 38 value: 'preview',
33 checked: env === 'preview' 39 checked: env === 'preview'
@@ -79,15 +85,111 @@ const monitor = { @@ -79,15 +85,111 @@ const monitor = {
79 where += ' order by time desc'; 85 where += ' order by time desc';
80 } 86 }
81 87
82 - let sql = `select * from ${influxName} ${where} limit ${limit} OFFSET ${(page -1) * limit}`;  
83 - 88 + let sql = `select * from ${influxName} ${where} limit ${limit} OFFSET ${(page - 1) * limit}`;
  89 +
84 let logs = await InfluxDB.query(sql); 90 let logs = await InfluxDB.query(sql);
85 91
86 ctx.body = logs[0]; 92 ctx.body = logs[0];
  93 + },
  94 +
  95 + async scanners(ctx) {
  96 + let data = await LogScanner.findAll();
  97 + let jobs = Schedulers.getJobs();
  98 +
  99 + _.forEach(data, d => {
  100 + d.sql = `select * from ${d.tableName} where ${d.tableWhere} and time > {lastTime} `;
  101 + d.state = jobs.has(d._id);
  102 + });
  103 + await ctx.render('action/log_scanners', {scanners: data});
  104 + },
  105 +
  106 + async scanners_new(ctx) {
  107 + await ctx.render('action/log_scanners_form');
  108 + },
  109 + async scanners_edit(ctx) {
  110 + let id = ctx.query.id;
  111 + let scanner = await LogScanner.findById(id);
  112 +
  113 + await ctx.render('action/log_scanners_form', scanner);
  114 + },
  115 + scanners_save: async(ctx, next) => {
  116 + let scanner = ctx.request.body;
  117 + let _id = ctx.request.body._id;
  118 +
  119 + if (_id) {
  120 + await LogScanner.update({
  121 + _id: _id
  122 + }, {
  123 + $set: scanner
  124 + });
  125 + await Operation.action(ctx.session.user, 'EDIT_SCANNER', '修改日志扫描', scanner);
  126 + } else {
  127 + delete scanner._id;
  128 + await LogScanner.insert(scanner);
  129 + await Operation.action(ctx.session.user, 'NEW_SCANNER', '新增日志扫描', scanner);
  130 + }
  131 +
  132 + ctx.redirect('/monitor/scanners');
  133 + ctx.status = 301;
  134 + },
  135 + scanners_del: async(ctx, next) => {
  136 + let id = ctx.request.body.id;
  137 + let scanner = await LogScanner.findById(id);
  138 +
  139 + await LogScanner.removeById(id);
  140 + await Operation.action(ctx.session.user, 'DELETE_SCANNER', '删除日志扫描', scanner);
  141 + ctx.body = {
  142 + code: 200
  143 + };
  144 + },
  145 +
  146 + async scanner_start(ctx) {
  147 + let id = ctx.request.body.id;
  148 + let scanner = await LogScanner.findById(id);
  149 + let jobs = Schedulers.getJobs();
  150 +
  151 + if (jobs.has(id)) {
  152 + return ctx.body = {
  153 + code: 400,
  154 + message: '扫描正在运行中'
  155 + };
  156 + }
  157 +
  158 + if (scanner) {
  159 + let runner = new ScannerRunner(scanner);
  160 + Schedulers.run(id, scanner.cron, runner.run.bind(runner));
  161 + Operation.action(ctx.session.user, 'START_SCANNER', '启动日志扫描', scanner);
  162 + return ctx.body = {
  163 + code: 200
  164 + };
  165 + } else {
  166 + return ctx.body = {
  167 + code: 400,
  168 + message: '配置不存在'
  169 + };
  170 + }
  171 + },
  172 +
  173 + async scanner_stop(ctx) {
  174 + let id = ctx.request.body.id;
  175 + Schedulers.cancel(id);
  176 + return ctx.body = {
  177 + code: 200
  178 + };
87 } 179 }
  180 +
88 }; 181 };
89 182
90 r.get('/log', monitor.log); 183 r.get('/log', monitor.log);
91 r.get('/log/query', monitor.query); 184 r.get('/log/query', monitor.query);
92 185
  186 +r.get('/scanners', monitor.scanners);
  187 +r.get('/scanners/new', monitor.scanners_new);
  188 +r.get('/scanners/edit', monitor.scanners_edit);
  189 +
  190 +r.post('/scanners/save', monitor.scanners_save);
  191 +r.post('/scanners/del', monitor.scanners_del);
  192 +r.post('/scanners/start', monitor.scanner_start);
  193 +r.post('/scanners/stop', monitor.scanner_stop);
  194 +
93 export default r; 195 export default r;
  1 +/**
  2 + *
  3 + * @author: jiangfeng<jeff.jiang@yoho.cn>
  4 + * @date: 16/9/1
  5 + */
  6 +
  7 +"use strict";
  8 +
  9 +import InfluxDB from '../../logger/influxdb';
  10 +import _ from 'lodash';
  11 +
  12 +class ScannerRunner {
  13 +
  14 + constructor(scanner) {
  15 + this.scanner = scanner;
  16 + this.lastTime = null;
  17 + }
  18 +
  19 + run () {
  20 + let sql = `select * from ${this.scanner.tableName} `;
  21 +
  22 + if (this.scanner.tableWhere) {
  23 + sql += ` where ${this.scanner.tableWhere} `;
  24 + }
  25 +
  26 + if (this.lastTime) {
  27 + sql += ` and time > '${this.lastTime}'`;
  28 + }
  29 +
  30 + sql += ' order by time desc ';
  31 +
  32 + if (!this.lastTime) {
  33 + sql += ' limit 100';
  34 + }
  35 +
  36 + console.log(`scanner [${this.scanner._id}] run : ${sql}`);
  37 +
  38 + InfluxDB.query(sql).then(data => {
  39 + data = data[0] || [];
  40 +
  41 + let times = _.map(data, d => {
  42 + return d.time;
  43 + });
  44 +
  45 + this.lastTime = _.head(times);
  46 +
  47 +
  48 + });
  49 + }
  50 +}
  51 +
  52 +
  53 +export default ScannerRunner;
  1 +<div class="pageheader">
  2 + <div class="media">
  3 + <div class="pageicon pull-left">
  4 + <i class="fa fa-th-list"></i>
  5 + </div>
  6 + <div class="media-body">
  7 + <ul class="breadcrumb">
  8 + <li><a href=""><i class="glyphicon glyphicon-home"></i></a></li>
  9 + <li><a href="">监控中心</a></li>
  10 + <li>日志扫描</li>
  11 + </ul>
  12 + <h4>日志扫描</h4>
  13 + </div>
  14 + </div>
  15 + <!-- media -->
  16 +</div>
  17 +<!-- pageheader -->
  18 +
  19 +<div class="contentpanel">
  20 + <div class="panel panel-primary-head">
  21 + <div class="panel-heading">
  22 + <div class="pull-right">
  23 + <a id="new-page" href="/monitor/scanners/new" class="btn btn-success btn-rounded"><i
  24 + class="glyphicon glyphicon-plus"></i> 新增扫描</a>
  25 + </div>
  26 + <h4 class="panel-title">日志扫描</h4>
  27 + <p>&nbsp;</p>
  28 + </div>
  29 + <!-- panel-heading -->
  30 +
  31 + <table id="table-scanners" class="table table-striped table-bordered responsive">
  32 + <thead class="">
  33 + <tr>
  34 + <th>标题</th>
  35 + <th>项目</th>
  36 + <th>执行规则</th>
  37 + <th>下次执行时间</th>
  38 + <th>执行语句</th>
  39 + <th></th>
  40 + </tr>
  41 + </thead>
  42 +
  43 + <tbody>
  44 + {{#each scanners}}
  45 + <tr>
  46 + <td>{{name}}</td>
  47 + <td>{{project}}</td>
  48 + <td>{{cron}}</td>
  49 + <td>{{nextTime}}</td>
  50 + <td>{{sql}}</td>
  51 + <td data-id='{{_id}}'>
  52 + {{#if state}}
  53 + <button class="btn btn-warning btn-xs scanner-stop">停止</button>
  54 + &nbsp;
  55 + {{^}}
  56 + <button class="btn btn-info btn-xs scanner-start">启动</button>
  57 + &nbsp;
  58 + {{/if}}
  59 + <button class="btn btn-success btn-xs scanner-edit">修改</button>
  60 + &nbsp;
  61 + <button class="btn btn-danger btn-xs scanner-del">删除</button>
  62 + </td>
  63 + </tr>
  64 + {{/each}}
  65 + </tbody>
  66 + </table>
  67 + </div>
  68 + <!-- panel -->
  69 +</div>
  70 +
  71 +
  72 +<script>
  73 + $(document).on('ready pjax:success', function() {
  74 + $("#table-scanners").DataTable({
  75 + pageLength: 25,
  76 + retrieve: true,
  77 + responsive: true,
  78 + searching: false
  79 + });
  80 + $(document).pjax('#new-page', '#pjax-container')
  81 +
  82 + $('.scanner-del').click(function() {
  83 + var id = $(this).parent().data('id');
  84 + $.post('/monitor/scanners/del', {id: id}, function(ret) {
  85 + if (ret.code == 200) {
  86 + var i = layer.alert('操作成功', function() {
  87 + layer.close(i);
  88 + $.pjax.reload('#pjax-container');
  89 + });
  90 + }
  91 + });
  92 + });
  93 +
  94 + $('.scanner-edit').click(function() {
  95 + var id = $(this).parent().data('id');
  96 + $.pjax({
  97 + url: '/monitor/scanners/edit?id=' + id,
  98 + container: '#pjax-container'
  99 + });
  100 + });
  101 +
  102 + $('.scanner-start').click(function() {
  103 + var id = $(this).parent().data('id');
  104 +
  105 + $.post('/monitor/scanners/start', {id: id}, function() {
  106 + $.pjax.reload('#pjax-container')
  107 + });
  108 + });
  109 +
  110 + $('.scanner-stop').click(function() {
  111 + var id = $(this).parent().data('id');
  112 +
  113 + $.post('/monitor/scanners/stop', {id: id}, function() {
  114 + $.pjax.reload('#pjax-container')
  115 + });
  116 + });
  117 + });
  118 +
  119 +</script>
  1 +
  2 +<div class="pageheader">
  3 + <div class="media">
  4 + <div class="pageicon pull-left">
  5 + <i class="fa fa-th-list"></i>
  6 + </div>
  7 + <div class="media-body">
  8 + <ul class="breadcrumb">
  9 + <li><a href=""><i class="glyphicon glyphicon-home"></i></a></li>
  10 + <li><a href="">监控中心</a></li>
  11 + <li>日志扫描</li>
  12 + </ul>
  13 + <h4>日志扫描配置</h4>
  14 + </div>
  15 + </div>
  16 + <!-- media -->
  17 +</div>
  18 +<!-- pageheader -->
  19 +<div class="panel panel-default">
  20 + <form action="/monitor/scanners/save" method="POST" data-pjax>
  21 + <input type="hidden" name="_id" value="{{_id}}">
  22 + <div class="panel-body">
  23 + <div class="row">
  24 + <div class="col-sm-6">
  25 + <div class="form-group">
  26 + <label class="control-label">标题</label>
  27 + <input type="text" name="name" value="{{name}}" class="form-control" placeholder="规则标题">
  28 + </div>
  29 + <!-- form-group -->
  30 + </div>
  31 + <!-- col-sm-6 -->
  32 +
  33 + <div class="col-sm-6">
  34 + <div class="form-group">
  35 + <label class="control-label">项目</label>
  36 + <input type="text" name="project" value="{{project}}" class="form-control" placeholder="项目名称">
  37 + </div>
  38 + <!-- form-group -->
  39 + </div>
  40 + <!-- col-sm-6 -->
  41 + </div>
  42 + <!-- row -->
  43 +
  44 + <div class="row">
  45 + <div class="col-sm-6">
  46 + <div class="form-group">
  47 + <label class="control-label">cron表达式</label>
  48 + <input type="text" name="cron" value="{{cron}}" class="form-control" placeholder="执行规则">
  49 + </div>
  50 + <!-- form-group -->
  51 + </div>
  52 + <!-- col-sm-6 -->
  53 + </div>
  54 +
  55 + <div class="row">
  56 + <div class="col-sm-6">
  57 + <div class="form-group">
  58 + <label class="control-label">数据库名称</label>
  59 + <input type="text" name="tableName" value="{{tableName}}" class="form-control" placeholder="ex: log">
  60 + </div>
  61 + <!-- form-group -->
  62 + </div>
  63 + <!-- col-sm-6 -->
  64 + <div class="col-sm-6">
  65 + <div class="form-group">
  66 + <label class="control-label">筛选条件</label>
  67 + <input type="text" name="tableWhere" value="{{tableWhere}}" class="form-control" placeholder="ex: level='error' and message =~ /TIMEOUT/">
  68 + </div>
  69 + <!-- form-group -->
  70 + </div>
  71 + <!-- col-sm-6 -->
  72 + </div>
  73 +
  74 + <!-- row -->
  75 + </div>
  76 + <!-- panel-body -->
  77 + <div class="panel-footer">
  78 + <button type="submit" class="btn btn-primary">保存</button>
  79 + </div>
  80 + </form>
  81 + <!-- panel-footer -->
  82 +</div>
  83 +
  84 +<script>
  85 + $(document).on('ready pjax:success', function() {
  86 + $(document.body).off().on('submit', 'form[data-pjax]', function(event) {
  87 + event.preventDefault(); // stop default submit behavior
  88 + $.pjax.submit(event, '#pjax-container', {
  89 + type: 'POST'
  90 + });
  91 + });
  92 + });
  93 +</script>
  94 +
@@ -23,6 +23,7 @@ @@ -23,6 +23,7 @@
23 <li class="parent"><a href=""><i class="fa fa-eye"></i> <span>监控中心</span></a> 23 <li class="parent"><a href=""><i class="fa fa-eye"></i> <span>监控中心</span></a>
24 <ul class="children"> 24 <ul class="children">
25 <li><a href="/monitor/log">实时日志</a></li> 25 <li><a href="/monitor/log">实时日志</a></li>
  26 + <li><a href="/monitor/scanners">日志扫描</a></li>
26 </ul> 27 </ul>
27 </li> 28 </li>
28 {{#if is_master}} 29 {{#if is_master}}
@@ -54,6 +54,7 @@ @@ -54,6 +54,7 @@
54 "moment": "^2.13.0", 54 "moment": "^2.13.0",
55 "nedb": "^1.8.0", 55 "nedb": "^1.8.0",
56 "nedb-promise": "^2.0.0", 56 "nedb-promise": "^2.0.0",
  57 + "node-schedule": "^1.1.1",
57 "qn": "^1.3.0", 58 "qn": "^1.3.0",
58 "qs": "^6.2.0", 59 "qs": "^6.2.0",
59 "shelljs": "^0.7.0", 60 "shelljs": "^0.7.0",