add log scanner

@@ -8,7 +8,7 @@ const influx = require('influx');
 
 let client = influx({
     hosts: [{
-        host: 'influxdblog.yohoops.org',
+        host: '54.222.219.223',
         port: 8086,
         protocol: 'http'
     }],
@@ -28,6 +28,18 @@ const db = {
                 }
             });
         });
+    },
+
+    createContinuousQuery: (key, query) => {
+        return new Promise((resolve, reject) => {
+            client.createContinuousQuery(key, query, (err, result) => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve(result);
+                }
+            });
+        });
     }
 };
 

+/**
+ *
+ * @author: jiangfeng<jeff.jiang@yoho.cn>
+ * @date: 16/8/31
+ */
+
+"use strict";
+
+const schedule = require('node-schedule');
+
+const jobMap = new Map();
+
+const jobs = {
+    run(key, cron, fn) {
+        let job = schedule.scheduleJob(cron, fn);
+
+        jobMap.set(key, job);
+    },
+
+    cancel(key) {
+        let job = jobMap.get(key);
+
+        schedule.cancelJob(job);
+        jobMap.delete(key);
+    },
+
+    getJobs() {
+        return jobMap;
+    }
+};
+
+module.exports = jobs;

@@ -8,6 +8,7 @@ import DeployModel from './deploy';
 import UserModel from './user';
 import HotfixModel from './hotfix';
 import OperationLoggerModel from './operation_logger';
+import LogScannerModel from './log_scanner';
 
 shelljs.mkdir('-p', config.dbDir);
 
@@ -18,6 +19,7 @@ const DeployInfo = new DeployModel();
 const User = new UserModel();
 const Hotfix = new HotfixModel();
 const OperationLogger = new OperationLoggerModel();
+const LogScanner = new LogScannerModel();
 
 User.init();
 
@@ -28,5 +30,6 @@ export {
     DeployInfo,
     User,
     Hotfix,
-    OperationLogger
+    OperationLogger,
+    LogScanner
 };
\ No newline at end of file

+/**
+ *
+ * @author: jiangfeng<jeff.jiang@yoho.cn>
+ * @date: 16/8/31
+ */
+
+'use strict';
+
+import Model from './model';
+
+class LogScanner extends Model {
+
+    constructor() {
+        super('log_scanner');
+    }
+
+}
+
+export default LogScanner;

@@ -9,10 +9,16 @@
 import Router from 'koa-router';
 
 import {
-    Project
+    Project,
+    LogScanner
 } from '../../models';
 
 import InfluxDB from '../../logger/influxdb';
+import Operation from '../../logger/operation';
+import Schedulers from '../../logger/schedulers';
+import ScannerRunner  from './scanner_runner';
+
+import _ from 'lodash';
 
 const r = new Router();
 
@@ -27,7 +33,7 @@ const monitor = {
                 name: '线上环境',
                 value: 'production',
                 checked: env === 'production'
-            },{
+            }, {
                 name: '灰度环境',
                 value: 'preview',
                 checked: env === 'preview'
@@ -79,15 +85,111 @@ const monitor = {
             where += ' order by time desc';
         }
 
-        let sql = `select * from ${influxName} ${where}  limit ${limit} OFFSET ${(page -1) * limit}`;
-        
+        let sql = `select * from ${influxName} ${where}  limit ${limit} OFFSET ${(page - 1) * limit}`;
+
         let logs = await InfluxDB.query(sql);
 
         ctx.body = logs[0];
+    },
+
+    async scanners(ctx) {
+        let data = await LogScanner.findAll();
+        let jobs = Schedulers.getJobs();
+
+        _.forEach(data, d => {
+            d.sql = `select * from ${d.tableName} where ${d.tableWhere} and time > {lastTime} `;
+            d.state = jobs.has(d._id);
+        });
+        await ctx.render('action/log_scanners', {scanners: data});
+    },
+
+    async scanners_new(ctx) {
+        await ctx.render('action/log_scanners_form');
+    },
+    async scanners_edit(ctx) {
+        let id = ctx.query.id;
+        let scanner = await LogScanner.findById(id);
+
+        await ctx.render('action/log_scanners_form', scanner);
+    },
+    scanners_save: async(ctx, next) => {
+        let scanner = ctx.request.body;
+        let _id = ctx.request.body._id;
+
+        if (_id) {
+            await LogScanner.update({
+                _id: _id
+            }, {
+                $set: scanner
+            });
+            await Operation.action(ctx.session.user, 'EDIT_SCANNER', '修改日志扫描', scanner);
+        } else {
+            delete scanner._id;
+            await LogScanner.insert(scanner);
+            await Operation.action(ctx.session.user, 'NEW_SCANNER', '新增日志扫描', scanner);
+        }
+
+        ctx.redirect('/monitor/scanners');
+        ctx.status = 301;
+    },
+    scanners_del: async(ctx, next) => {
+        let id = ctx.request.body.id;
+        let scanner = await LogScanner.findById(id);
+
+        await LogScanner.removeById(id);
+        await Operation.action(ctx.session.user, 'DELETE_SCANNER', '删除日志扫描', scanner);
+        ctx.body = {
+            code: 200
+        };
+    },
+
+    async scanner_start(ctx) {
+        let id = ctx.request.body.id;
+        let scanner = await LogScanner.findById(id);
+        let jobs = Schedulers.getJobs();
+
+        if (jobs.has(id)) {
+            return ctx.body = {
+                code: 400,
+                message: '扫描正在运行中'
+            };
+        }
+
+        if (scanner) {
+            let runner = new ScannerRunner(scanner);
+            Schedulers.run(id, scanner.cron, runner.run.bind(runner));
+            Operation.action(ctx.session.user, 'START_SCANNER', '启动日志扫描', scanner);
+            return ctx.body = {
+                code: 200
+            };
+        } else {
+            return ctx.body = {
+                code: 400,
+                message: '配置不存在'
+            };
+        }
+    },
+
+    async scanner_stop(ctx) {
+        let id = ctx.request.body.id;
+        Schedulers.cancel(id);
+        return ctx.body = {
+            code: 200
+        };
     }
+
 };
 
 r.get('/log', monitor.log);
 r.get('/log/query', monitor.query);
 
+r.get('/scanners', monitor.scanners);
+r.get('/scanners/new', monitor.scanners_new);
+r.get('/scanners/edit', monitor.scanners_edit);
+
+r.post('/scanners/save', monitor.scanners_save);
+r.post('/scanners/del', monitor.scanners_del);
+r.post('/scanners/start', monitor.scanner_start);
+r.post('/scanners/stop', monitor.scanner_stop);
+
 export default r;
\ No newline at end of file

+/**
+ *
+ * @author: jiangfeng<jeff.jiang@yoho.cn>
+ * @date: 16/9/1
+ */
+
+"use strict";
+
+import InfluxDB from '../../logger/influxdb';
+import _ from 'lodash';
+
+class ScannerRunner {
+
+    constructor(scanner) {
+        this.scanner = scanner;
+        this.lastTime = null;
+    }
+
+    run () {
+        let sql = `select * from ${this.scanner.tableName} `;
+
+        if (this.scanner.tableWhere) {
+            sql += ` where ${this.scanner.tableWhere} `;
+        }
+
+        if (this.lastTime) {
+            sql += ` and time > '${this.lastTime}'`;
+        }
+
+        sql += ' order by time desc ';
+
+        if (!this.lastTime) {
+            sql += ' limit 100';
+        }
+
+        console.log(`scanner [${this.scanner._id}] run : ${sql}`);
+
+        InfluxDB.query(sql).then(data => {
+            data = data[0] || [];
+
+            let times = _.map(data, d => {
+                return d.time;
+            });
+
+            this.lastTime = _.head(times);
+
+
+        });
+    }
+}
+
+
+export default ScannerRunner;
\ No newline at end of file

+<div class="pageheader">
+    <div class="media">
+        <div class="pageicon pull-left">
+            <i class="fa fa-th-list"></i>
+        </div>
+        <div class="media-body">
+            <ul class="breadcrumb">
+                <li><a href=""><i class="glyphicon glyphicon-home"></i></a></li>
+                <li><a href="">监控中心</a></li>
+                <li>日志扫描</li>
+            </ul>
+            <h4>日志扫描</h4>
+        </div>
+    </div>
+    <!-- media -->
+</div>
+<!-- pageheader -->
+
+<div class="contentpanel">
+    <div class="panel panel-primary-head">
+        <div class="panel-heading">
+            <div class="pull-right">
+                <a id="new-page" href="/monitor/scanners/new" class="btn btn-success btn-rounded"><i
+                        class="glyphicon glyphicon-plus"></i> 新增扫描</a>
+            </div>
+            <h4 class="panel-title">日志扫描</h4>
+            <p>&nbsp;</p>
+        </div>
+        <!-- panel-heading -->
+
+        <table id="table-scanners" class="table table-striped table-bordered responsive">
+            <thead class="">
+            <tr>
+                <th>标题</th>
+                <th>项目</th>
+                <th>执行规则</th>
+                <th>下次执行时间</th>
+                <th>执行语句</th>
+                <th></th>
+            </tr>
+            </thead>
+
+            <tbody>
+            {{#each scanners}}
+                <tr>
+                    <td>{{name}}</td>
+                    <td>{{project}}</td>
+                    <td>{{cron}}</td>
+                    <td>{{nextTime}}</td>
+                    <td>{{sql}}</td>
+                    <td data-id='{{_id}}'>
+                        {{#if state}}
+                            <button class="btn btn-warning btn-xs scanner-stop">停止</button>
+                            &nbsp;
+                        {{^}}
+                            <button class="btn btn-info btn-xs scanner-start">启动</button>
+                            &nbsp;
+                        {{/if}}
+                        <button class="btn btn-success btn-xs scanner-edit">修改</button>
+                        &nbsp;
+                        <button class="btn btn-danger btn-xs scanner-del">删除</button>
+                    </td>
+                </tr>
+            {{/each}}
+            </tbody>
+        </table>
+    </div>
+    <!-- panel -->
+</div>
+
+
+<script>
+    $(document).on('ready pjax:success', function() {
+        $("#table-scanners").DataTable({
+            pageLength: 25,
+            retrieve: true,
+            responsive: true,
+            searching: false
+        });
+        $(document).pjax('#new-page', '#pjax-container')
+
+        $('.scanner-del').click(function() {
+            var id = $(this).parent().data('id');
+            $.post('/monitor/scanners/del', {id: id}, function(ret) {
+                if (ret.code == 200) {
+                    var i = layer.alert('操作成功', function() {
+                        layer.close(i);
+                        $.pjax.reload('#pjax-container');
+                    });
+                }
+            });
+        });
+
+        $('.scanner-edit').click(function() {
+            var id = $(this).parent().data('id');
+            $.pjax({
+                url: '/monitor/scanners/edit?id=' + id,
+                container: '#pjax-container'
+            });
+        });
+
+        $('.scanner-start').click(function() {
+            var id = $(this).parent().data('id');
+
+            $.post('/monitor/scanners/start', {id: id}, function() {
+                $.pjax.reload('#pjax-container')
+            });
+        });
+
+        $('.scanner-stop').click(function() {
+            var id = $(this).parent().data('id');
+
+            $.post('/monitor/scanners/stop', {id: id}, function() {
+                $.pjax.reload('#pjax-container')
+            });
+        });
+    });
+
+</script>
\ No newline at end of file

+
+<div class="pageheader">
+    <div class="media">
+        <div class="pageicon pull-left">
+            <i class="fa fa-th-list"></i>
+        </div>
+        <div class="media-body">
+            <ul class="breadcrumb">
+                <li><a href=""><i class="glyphicon glyphicon-home"></i></a></li>
+                <li><a href="">监控中心</a></li>
+                <li>日志扫描</li>
+            </ul>
+            <h4>日志扫描配置</h4>
+        </div>
+    </div>
+    <!-- media -->
+</div>
+<!-- pageheader -->
+<div class="panel panel-default">
+    <form action="/monitor/scanners/save" method="POST" data-pjax>
+        <input type="hidden" name="_id" value="{{_id}}">
+        <div class="panel-body">
+            <div class="row">
+                <div class="col-sm-6">
+                    <div class="form-group">
+                        <label class="control-label">标题</label>
+                        <input type="text" name="name" value="{{name}}" class="form-control" placeholder="规则标题">
+                    </div>
+                    <!-- form-group -->
+                </div>
+                <!-- col-sm-6 -->
+
+                <div class="col-sm-6">
+                    <div class="form-group">
+                        <label class="control-label">项目</label>
+                        <input type="text" name="project" value="{{project}}" class="form-control" placeholder="项目名称">
+                    </div>
+                    <!-- form-group -->
+                </div>
+                <!-- col-sm-6 -->
+            </div>
+            <!-- row -->
+
+            <div class="row">
+                <div class="col-sm-6">
+                    <div class="form-group">
+                        <label class="control-label">cron表达式</label>
+                        <input type="text" name="cron" value="{{cron}}" class="form-control" placeholder="执行规则">
+                    </div>
+                    <!-- form-group -->
+                </div>
+                <!-- col-sm-6 -->
+            </div>
+
+            <div class="row">
+                <div class="col-sm-6">
+                    <div class="form-group">
+                        <label class="control-label">数据库名称</label>
+                        <input type="text" name="tableName" value="{{tableName}}" class="form-control" placeholder="ex: log">
+                    </div>
+                    <!-- form-group -->
+                </div>
+                <!-- col-sm-6 -->
+                <div class="col-sm-6">
+                    <div class="form-group">
+                        <label class="control-label">筛选条件</label>
+                        <input type="text" name="tableWhere" value="{{tableWhere}}" class="form-control" placeholder="ex: level='error' and message =~ /TIMEOUT/">
+                    </div>
+                    <!-- form-group -->
+                </div>
+                <!-- col-sm-6 -->
+            </div>
+
+            <!-- row -->
+        </div>
+        <!-- panel-body -->
+        <div class="panel-footer">
+            <button type="submit" class="btn btn-primary">保存</button>
+        </div>
+    </form>
+    <!-- panel-footer -->
+</div>
+
+<script>
+    $(document).on('ready pjax:success', function() {
+        $(document.body).off().on('submit', 'form[data-pjax]', function(event) {
+            event.preventDefault(); // stop default submit behavior
+            $.pjax.submit(event, '#pjax-container', {
+                type: 'POST'
+            });
+        });
+    });
+</script>
+

@@ -23,6 +23,7 @@
         <li class="parent"><a href=""><i class="fa fa-eye"></i> <span>监控中心</span></a>
             <ul class="children">
                 <li><a href="/monitor/log">实时日志</a></li>
+                <li><a href="/monitor/scanners">日志扫描</a></li>
             </ul>
         </li>
         {{#if is_master}}

@@ -54,6 +54,7 @@
     "moment": "^2.13.0",
     "nedb": "^1.8.0",
     "nedb-promise": "^2.0.0",
+    "node-schedule": "^1.1.1",
     "qn": "^1.3.0",
     "qs": "^6.2.0",
     "shelljs": "^0.7.0",