Merge branch 'feature/crawlerTaskLog' into 'master'

Feature/crawler task log



See merge request !22

@@ -3,6 +3,7 @@
 const Router = require('koa-router');
 const Router = require('koa-router');
 const _ = require('lodash');
 const _ = require('lodash');
 const md5 = require('md5');
 const md5 = require('md5');
+const moment = require('moment');
 const pager = require('../utils/pager');
 const pager = require('../utils/pager');
 const seoModel = require('../models/seoModel');
 const seoModel = require('../models/seoModel');
 
 
@@ -14,6 +15,18 @@ const TYPE_LIST = [
     {name: '店铺', type: 'shop', lt: 'ShopId'},
     {name: '店铺', type: 'shop', lt: 'ShopId'},
     {name: '链接', type: 'url', lt: 'URL'}
     {name: '链接', type: 'url', lt: 'URL'}
 ];
 ];
+const JOB_TASK = [
+    {
+        k: 'global:yoho:task:minute',
+        v: '1 */10 * * * *',
+        info: '每10分钟执行一次'
+    },
+    {
+        k: 'global:yoho:task:day',
+        v: '30 30 1 * * *',
+        info: '每天的1点30分30s执行一次'
+    }
+];
 
 
 const tdk = {
 const tdk = {
     // tdk 列表
     // tdk 列表
@@ -459,7 +472,6 @@ const category = {
     }
     }
 }
 }
 
 
-
 const friendLink = {
 const friendLink = {
     index: async(ctx, next) => {
     index: async(ctx, next) => {
         let type = ctx.query.type || 'text';
         let type = ctx.query.type || 'text';
@@ -624,6 +636,85 @@ const friendLink = {
     },
     },
 };
 };
 
 
+const task = {
+    index: async(ctx, next) => {
+        let tasks = await ctx.redis.multi(_.map(JOB_TASK, job => ['hvals', job.k])).execAsync().then(rdata => {
+            let result = [];
+
+            _.each(rdata, (items, rk) => {
+                _.each(items, item => {
+                    result.push(Object.assign(JSON.parse(item), {
+                        job_name: JOB_TASK[rk].info,
+                        type: rk,
+                    }));
+                })
+            });
+
+            return result;
+        });
+
+        await ctx.render('action/seo_task', {
+            title: '定时任务管理',
+            job: JOB_TASK,
+            tasks: tasks
+        });
+    },
+    add: async(ctx, next) => {
+        let params = ctx.request.body || {};
+        params.type = parseInt(`0${params.type}`, 10);
+
+        let task = _.get(JOB_TASK, `[${params.type}]`, {});
+
+        if (!task.k) {
+            return ctx.body = {code: 400, message: 'invalid type...'};
+        }
+
+        let code = md5(params.url || '');
+
+        return ctx.redis.hsetAsync(task.k, code, JSON.stringify({
+            url: params.url,
+            time: params.time || 1000,
+            name: params.name,
+            code: code
+        })).then(status => {
+            return ctx.body = {code: 200, message: 'success...'};
+        });
+    },
+    del: async(ctx, next) => {
+        let params = ctx.request.body || {};
+        let data = JSON.parse(_.get(ctx.request.body, 'data', '[]'));
+
+        ctx.redis.multi(
+            _.map(data, job => ['hdel', JOB_TASK[parseInt(`0${job.tpe}`)].k || '', job.code || ''])
+        ).execAsync().then(status => {
+            return ctx.body = {
+                code: 200,
+                message: 'success',
+                data: ''
+            };
+        });
+    },
+    log: async(ctx, next) => {
+        let params = ctx.request.body || {};
+        let key = params.code || '';
+        let start = params.start || 0;
+        let stop = params.stop || 100;
+
+        return ctx.redis.lrangeAsync(`global:yoho:task:log:${key}`, start, stop).then(rdata => {
+            let data = _.map(rdata, item => {
+                item = JSON.parse(item || '{}');
+                return Object.assign(item, {time: moment(item.time).format('YYYY-MM-DD HH:mm:ss')});
+            });
+
+            return ctx.body = {
+                code: 200,
+                message: 'success',
+                data: data
+            };
+        });
+    },
+};
+
 r.get('/', tdk.index);
 r.get('/', tdk.index);
 r.get('/tdk', tdk.index);
 r.get('/tdk', tdk.index);
 r.post('/tdk/add', tdk.add);
 r.post('/tdk/add', tdk.add);
@@ -650,4 +741,10 @@ r.post('/friendlink/add', friendLink.add);
 r.post('/friendlink/edit', friendLink.edit);
 r.post('/friendlink/edit', friendLink.edit);
 r.post('/friendlink/delete', friendLink.delete);
 r.post('/friendlink/delete', friendLink.delete);
 
 
+// 定时任务管理
+r.get('/task', task.index);
+r.post('/task/add', task.add);
+r.post('/task/del', task.del);
+r.post('/task/log', task.log);
+
 module.exports = r;
 module.exports = r;

@@ -148,4 +148,4 @@
             console.log('connect fail');
             console.log('connect fail');
         });
         });
     });
     });
-</script>
+</script>

@@ -21,7 +21,7 @@
             <a class="btn btn-default deleteAll">删除</a>
             <a class="btn btn-default deleteAll">删除</a>
             <a class="btn btn-default" href="/keywords/add">增加</a>
             <a class="btn btn-default" href="/keywords/add">增加</a>
             <a class="btn btn-default" href="javascript:sendUrl();">推送百度</a>
             <a class="btn btn-default" href="javascript:sendUrl();">推送百度</a>
-            <a class="btn btn-default rand-words" href="javascript:void(0);">关键词随机关联</a>
+            <!--<a class="btn btn-default rand-words" href="javascript:void(0);">关键词随机关联</a>-->
             <div class="input-append pull-right">
             <div class="input-append pull-right">
                 <form id="query-form" action="/keywords/expand" class="query-form" method="get">
                 <form id="query-form" action="/keywords/expand" class="query-form" method="get">
                     <!-- <div class="btn-group">
                     <!-- <div class="btn-group">
@@ -272,4 +272,4 @@
             alert(data.message);
             alert(data.message);
         })
         })
     }
     }
-</script>
+</script>

@@ -264,4 +264,4 @@
             }
             }
         });
         });
     });
     });
-</script>
+</script>

@@ -141,4 +141,4 @@
     });
     });
 
 
 </script>
 </script>
-{{/if}}
+{{/if}}

@@ -196,7 +196,7 @@
                 {{/each}}
                 {{/each}}
                 {{#unless linkList}}
                 {{#unless linkList}}
                     <tr>
                     <tr>
-                        <td class="text-center" colspan="6">暂无数据</td>
+                        <td class="text-center" colspan="5">暂无数据</td>
                     </tr>
                     </tr>
                 {{/unless}}
                 {{/unless}}
                 </tbody>
                 </tbody>

+<style type="text/css">
+    .task-add {
+        padding-top: 20px;
+    }
+    .task-add ul {
+        list-style: none;
+    }
+    .task-add ul li {
+        height: 40px;
+    }
+    .task-add select,
+    .task-add input {
+        width: 70%;
+        height: 30px;
+        padding: 5px;
+    }
+    .task-add .job-select {
+        margin-left: -3px;
+    }
+
+    .task-log {
+        padding: 20px;
+    }
+    .task-log table, .task-log table tr th,.task-log table tr td {
+        border: none;
+    }
+    .task-log tbody tr {
+        height: 40px;
+    }
+    .task-log table {
+        border-collapse: separate;
+        border-spacing: 0 4px;
+    }
+    .task-log tbody tr:nth-child(odd) td:first-child {
+        border-left: 2px solid #00C656;
+    }
+    .task-log tbody tr:nth-child(even) td:first-child {
+        border-left: 2px solid #4F95E8;
+    }
+    .task-log tbody tr.empty td {
+        border: none !important;
+    }
+</style>
+<div class="pageheader">
+    <div class="media">
+        <div class="pageicon pull-left">
+            <i class="fa fa-th-list"></i>
+        </div>
+        <div class="media-body">
+            <ul class="breadcrumb">
+                <li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
+                <li>{{title}}</li>
+            </ul>
+            <h4>{{title}}</h4>
+        </div>
+    </div>
+    <!-- media -->
+</div>
+<!-- pageheader -->
+
+<div class="contentpanel task-panel" style="padding-bottom:0;">
+    <div class="panel panel-default">
+        <div class="panel-body">
+            <button class="btn btn-default btn-add" >增加</button>
+            <button class="btn btn-default delete-all" type="submit">删除</button>
+        </div>
+    </div><!--/panel-default-->
+
+    <div class="panel panel-default">
+        <div class="panel-body">
+            <table class="table table-striped table-bordered tab-task">
+                <thead>
+                    <tr>
+                        <th width="10%">&nbsp;&nbsp;&nbsp;
+                            <label><input type="checkbox" style="vertical-align: text-bottom;" onclick="javascript:$('.tab-task input[type=checkbox]').attr('checked', this.checked);" />&nbsp;全选</label>
+                        </th>
+                        <th class="text-center" width="20%">任务名称</th>
+                        <th class="text-center" width="30%">链接</th>
+                        <th class="text-center" width="10%">延迟时间</th>
+                        <th class="text-center" width="15%">执行频率</th>
+                        <th class="text-center" width="15%">操作</th>
+                    </tr>
+                </thead>
+
+                <tbody>
+                {{#each tasks}}
+                    <tr data-code="{{code}}" data-type="{{type}}">
+                        <td>&nbsp;&nbsp;&nbsp;&nbsp;<input type="checkbox" /></td>
+                        <td class="text-center td-name" >{{name}}</td>
+                        <td class="text-center" >{{url}}</td>
+                        <td class="text-center" >{{time}}</td>
+                        <td class="text-center" >{{job_name}}</td>
+                        <td class="text-center">
+                            <a href="javascript:;" class="log-btn">执行日志</a>
+                            <a href="javascript:;" class="del-btn">删除</a>
+                        </td>
+                    </tr>
+                {{/each}}
+                {{#unless tasks}}
+                    <tr>
+                        <td class="text-center" colspan="6">暂无数据</td>
+                    </tr>
+                {{/unless}}
+                </tbody>
+            </table>
+            {{# pager}}
+                <div class="text-right">
+                {{#if pages}}
+                    <ul class="pagination">
+                        {{# prePage}}
+                            <li><a href="{{url}}">上一页</a></li>
+                        {{/ prePage}}
+                        {{# pages}}
+                            <li class="{{#unless url}}disabled {{/unless}}{{#if cur}}active{{/if}}"><a {{#if url}}href="{{url}}"{{^}}href="javascript:;"{{/if}}>{{num}}</a></li>
+                        {{/ pages}}
+                        {{# nextPage}}
+                            <li><a href="{{url}}">下一页</a></li>
+                        {{/ nextPage}}
+                    </ul>
+                {{/if}}
+                </div>
+            {{/ pager}}
+        </div>
+    </div><!--/panel-default-->
+</div><!--/contentpanel-->
+
+<script id="task-add-tpl" type="text/tmpl">
+<div class="task-add">
+    <ul>
+        <li>
+            执行频率：
+            <select class="job-select" data-placeholder="请选择..." >
+                <option value="-1">请选择...</option>
+                {{#each job}}
+                    <option value="{{@index}}">{{info}}</option>
+                {{/each}}
+            </select>
+        </li>
+        <li>任务名称：<input type="text" class="job-name" placeholder='请输入任务名称...'></li>
+        <li>执行链接：<input type="text" class="job-link" placeholder='请输入执行链接...'></li>
+        <li>延迟时间：<input type="text" class="job-time" placeholder='请输入延迟时间...'></li>
+    </ul>
+    <p><span class="err-tip"></span></p>
+</div>
+</script>
+
+<script id="task-log-tpl" type="text/tmpl">
+<div class="task-log">
+    <table class="table table-striped table-bordered">
+        <thead>
+            <tr>
+                <th class="text-center" width="20%">执行时间</th>
+                <th class="text-center" width="20%">状态码</th>
+                <th class="text-center" width="50%">执行结果</th>
+                <th class="text-right" width="10%">
+                    <a href="javascript:;" class="btn btn-default btn-refresh-log">刷新</a>
+                </th>
+            </tr>
+        </thead>
+        <tbody></tbody>
+    </table>
+</div>
+</script>
+
+<script>
+    var tasks = {
+        init: function() {
+            var that = this;
+            var taskData = {};
+
+            this.$base = $('.contentpanel');
+
+            this.$base.on('click', '.btn-add', function() {
+                that.add();
+            });
+
+            // 单个删除
+            this.$base.on('click', '.del-btn', function() {
+                let $tr = $(this).closest('tr');
+                that.del([$tr.data()], () => $tr.remove());
+            });
+
+            // 日志
+            this.$base.on('click', '.log-btn', function() {
+                let $tr = $(this).closest('tr');
+
+                taskData = $tr.data();
+
+                return that.getTaskLog(taskData).then(rdata => {
+                    layer.open({
+                        type: 1,
+                        area: ['800px', '500px'],
+                        title: '《' + $tr.find('.td-name').text() + '》执行明细',
+                        content: that.taskLogTpl
+                    });
+
+                    that.rendLog(rdata.data);
+                });
+            });
+
+            // 日志刷新
+            $('body').on('click', '.btn-refresh-log', function() {
+                var layerIndex = layer.load(1);
+
+                return that.getTaskLog(taskData).then(rdata => {
+                    setTimeout(function() {
+                        layer.close(layerIndex);
+                        that.rendLog(rdata.data);
+                    }, 2000);
+                });
+            });
+
+            // 删除全部
+            this.$base.on('click', '.delete-all', function() {
+                let arr = [];
+
+                $('.tab-task input[type=checkbox]:checked').each(function(){
+                    arr.push($(this).closest('tr').data());
+                });
+
+                that.del(arr, () => history.go(0));
+            });
+
+            this.taskAddTpl = $('#task-add-tpl').html();
+            this.taskLogTpl = $('#task-log-tpl').html();
+        },
+        add: function() {
+            var that = this;
+            var layerIndex = layer.open({
+                type: 1,
+                area: ['450px'],
+                title: '任务添加',
+                content: that.taskAddTpl,
+                btn: '添加',
+                yes: function() {
+                    var data = {
+                        type: parseInt($('.job-select').val(), 10),
+                        name: $('.job-name').val(),
+                        url: $('.job-link').val(),
+                        time: parseInt($('.job-time').val() || 0),
+                    };
+
+                    if (data.type < 0) {
+                        return that.tips('请选择执行频率...', '.job-select');
+                    } else if (data.name === '') {
+                        return that.tips('任务名称必填...', '.job-name');
+                    } else if (data.url === '' || data.url.indexOf('http') === -1) {
+                        return that.tips('请输入正确的执行链接...', '.job-link');
+                    }
+
+                    return that.ajax({
+                        url: '/seo/task/add',
+                        type: 'POST',
+                        data: data,
+                    }).then(rdata => {
+                        if (rdata.code !== 200) {
+                            return alert(rdata.message);
+                        }
+
+                        layer.close(layerIndex);
+
+                        history.go(0)
+                    });
+                }
+            });
+        },
+        del: function(data, callback) {
+            if (data.length <= 0) {
+                return false;
+            }
+
+            var that = this;
+            var layerIndex = layer.confirm('您确定要删除吗！！！', {
+                btn: ['确定', '取消']
+            }, function() {
+                return that.ajax({
+                    url: '/seo/task/del',
+                    type: 'POST',
+                    data: {data: JSON.stringify(data)},
+                }).then(res => {
+                    callback();
+                    layer.close(layerIndex);
+                    return res;
+                });
+            });
+        },
+        rendLog: function(rdata) {
+            var $tbody = $('.task-log tbody');
+            $tbody.empty();
+
+            if (rdata.length <= 0) {
+                $tbody.append(`<tr class="empty">
+                    <td class="text-center" colspan="4">暂无数据</td>
+                </tr>`);
+            } else {
+                $.each(rdata, (key, item)=> {
+                    $tbody.append(`<tr>
+                        <td class="text-center" >${item.time}</td>
+                        <td class="text-center" >${item.code}</td>
+                        <td class="text-center" colspan="2">${item.message}</td>
+                    </tr>`);
+                });
+            }
+        },
+        getTaskLog: function(data) {
+            return this.ajax({
+                url: '/seo/task/log',
+                type: 'POST',
+                data: data,
+            });
+        },
+        ajax: function(options) {
+            return Promise.resolve($.ajax(options));
+        },
+        tips: function(tip, dom) {
+            return layer.tips(tip, dom, {
+                tips: [1, '#000'],
+                time: 2000
+            });
+        }
+    };
+
+$(function() {
+    tasks.init();
+});
+</script>

@@ -77,6 +77,7 @@
                 <li><a href="/seo/tdk"><span>TDK管理</span></a></li>
                 <li><a href="/seo/tdk"><span>TDK管理</span></a></li>
                 <li><a href="/seo/category"><span>品类描述管理</span></a></li>
                 <li><a href="/seo/category"><span>品类描述管理</span></a></li>
                 <li><a href="/seo/friendlink"><span>友链管理</span></a></li>
                 <li><a href="/seo/friendlink"><span>友链管理</span></a></li>
+                <li><a href="/seo/task"><span>定时任务管理</span></a></li>
             </ul>
             </ul>
         </li>
         </li>