Authored by 毕凯

Merge branch 'feature/crawlerTaskLog' into 'master'

Feature/crawler task log



See merge request !22
... ... @@ -3,6 +3,7 @@
const Router = require('koa-router');
const _ = require('lodash');
const md5 = require('md5');
const moment = require('moment');
const pager = require('../utils/pager');
const seoModel = require('../models/seoModel');
... ... @@ -14,6 +15,18 @@ const TYPE_LIST = [
{name: '店铺', type: 'shop', lt: 'ShopId'},
{name: '链接', type: 'url', lt: 'URL'}
];
const JOB_TASK = [
{
k: 'global:yoho:task:minute',
v: '1 */10 * * * *',
info: '每10分钟执行一次'
},
{
k: 'global:yoho:task:day',
v: '30 30 1 * * *',
info: '每天的1点30分30s执行一次'
}
];
const tdk = {
// tdk 列表
... ... @@ -459,7 +472,6 @@ const category = {
}
}
const friendLink = {
index: async(ctx, next) => {
let type = ctx.query.type || 'text';
... ... @@ -624,6 +636,85 @@ const friendLink = {
},
};
const task = {
index: async(ctx, next) => {
let tasks = await ctx.redis.multi(_.map(JOB_TASK, job => ['hvals', job.k])).execAsync().then(rdata => {
let result = [];
_.each(rdata, (items, rk) => {
_.each(items, item => {
result.push(Object.assign(JSON.parse(item), {
job_name: JOB_TASK[rk].info,
type: rk,
}));
})
});
return result;
});
await ctx.render('action/seo_task', {
title: '定时任务管理',
job: JOB_TASK,
tasks: tasks
});
},
add: async(ctx, next) => {
let params = ctx.request.body || {};
params.type = parseInt(`0${params.type}`, 10);
let task = _.get(JOB_TASK, `[${params.type}]`, {});
if (!task.k) {
return ctx.body = {code: 400, message: 'invalid type...'};
}
let code = md5(params.url || '');
return ctx.redis.hsetAsync(task.k, code, JSON.stringify({
url: params.url,
time: params.time || 1000,
name: params.name,
code: code
})).then(status => {
return ctx.body = {code: 200, message: 'success...'};
});
},
del: async(ctx, next) => {
let params = ctx.request.body || {};
let data = JSON.parse(_.get(ctx.request.body, 'data', '[]'));
ctx.redis.multi(
_.map(data, job => ['hdel', JOB_TASK[parseInt(`0${job.tpe}`)].k || '', job.code || ''])
).execAsync().then(status => {
return ctx.body = {
code: 200,
message: 'success',
data: ''
};
});
},
log: async(ctx, next) => {
let params = ctx.request.body || {};
let key = params.code || '';
let start = params.start || 0;
let stop = params.stop || 100;
return ctx.redis.lrangeAsync(`global:yoho:task:log:${key}`, start, stop).then(rdata => {
let data = _.map(rdata, item => {
item = JSON.parse(item || '{}');
return Object.assign(item, {time: moment(item.time).format('YYYY-MM-DD HH:mm:ss')});
});
return ctx.body = {
code: 200,
message: 'success',
data: data
};
});
},
};
r.get('/', tdk.index);
r.get('/tdk', tdk.index);
r.post('/tdk/add', tdk.add);
... ... @@ -650,4 +741,10 @@ r.post('/friendlink/add', friendLink.add);
r.post('/friendlink/edit', friendLink.edit);
r.post('/friendlink/delete', friendLink.delete);
// 定时任务管理
r.get('/task', task.index);
r.post('/task/add', task.add);
r.post('/task/del', task.del);
r.post('/task/log', task.log);
module.exports = r;
... ...
... ... @@ -148,4 +148,4 @@
console.log('connect fail');
});
});
</script>
\ No newline at end of file
</script>
... ...
... ... @@ -21,7 +21,7 @@
<a class="btn btn-default deleteAll">删除</a>
<a class="btn btn-default" href="/keywords/add">增加</a>
<a class="btn btn-default" href="javascript:sendUrl();">推送百度</a>
<a class="btn btn-default rand-words" href="javascript:void(0);">关键词随机关联</a>
<!--<a class="btn btn-default rand-words" href="javascript:void(0);">关键词随机关联</a>-->
<div class="input-append pull-right">
<form id="query-form" action="/keywords/expand" class="query-form" method="get">
<!-- <div class="btn-group">
... ... @@ -272,4 +272,4 @@
alert(data.message);
})
}
</script>
\ No newline at end of file
</script>
... ...
... ... @@ -264,4 +264,4 @@
}
});
});
</script>
\ No newline at end of file
</script>
... ...
... ... @@ -141,4 +141,4 @@
});
</script>
{{/if}}
\ No newline at end of file
{{/if}}
... ...
... ... @@ -196,7 +196,7 @@
{{/each}}
{{#unless linkList}}
<tr>
<td class="text-center" colspan="6">暂无数据</td>
<td class="text-center" colspan="5">暂无数据</td>
</tr>
{{/unless}}
</tbody>
... ...
<style type="text/css">
.task-add {
padding-top: 20px;
}
.task-add ul {
list-style: none;
}
.task-add ul li {
height: 40px;
}
.task-add select,
.task-add input {
width: 70%;
height: 30px;
padding: 5px;
}
.task-add .job-select {
margin-left: -3px;
}
.task-log {
padding: 20px;
}
.task-log table, .task-log table tr th,.task-log table tr td {
border: none;
}
.task-log tbody tr {
height: 40px;
}
.task-log table {
border-collapse: separate;
border-spacing: 0 4px;
}
.task-log tbody tr:nth-child(odd) td:first-child {
border-left: 2px solid #00C656;
}
.task-log tbody tr:nth-child(even) td:first-child {
border-left: 2px solid #4F95E8;
}
.task-log tbody tr.empty td {
border: none !important;
}
</style>
<div class="pageheader">
<div class="media">
<div class="pageicon pull-left">
<i class="fa fa-th-list"></i>
</div>
<div class="media-body">
<ul class="breadcrumb">
<li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
<li>{{title}}</li>
</ul>
<h4>{{title}}</h4>
</div>
</div>
<!-- media -->
</div>
<!-- pageheader -->
<div class="contentpanel task-panel" style="padding-bottom:0;">
<div class="panel panel-default">
<div class="panel-body">
<button class="btn btn-default btn-add" >增加</button>
<button class="btn btn-default delete-all" type="submit">删除</button>
</div>
</div><!--/panel-default-->
<div class="panel panel-default">
<div class="panel-body">
<table class="table table-striped table-bordered tab-task">
<thead>
<tr>
<th width="10%">&nbsp;&nbsp;&nbsp;
<label><input type="checkbox" style="vertical-align: text-bottom;" onclick="javascript:$('.tab-task input[type=checkbox]').attr('checked', this.checked);" />&nbsp;全选</label>
</th>
<th class="text-center" width="20%">任务名称</th>
<th class="text-center" width="30%">链接</th>
<th class="text-center" width="10%">延迟时间</th>
<th class="text-center" width="15%">执行频率</th>
<th class="text-center" width="15%">操作</th>
</tr>
</thead>
<tbody>
{{#each tasks}}
<tr data-code="{{code}}" data-type="{{type}}">
<td>&nbsp;&nbsp;&nbsp;&nbsp;<input type="checkbox" /></td>
<td class="text-center td-name" >{{name}}</td>
<td class="text-center" >{{url}}</td>
<td class="text-center" >{{time}}</td>
<td class="text-center" >{{job_name}}</td>
<td class="text-center">
<a href="javascript:;" class="log-btn">执行日志</a>
<a href="javascript:;" class="del-btn">删除</a>
</td>
</tr>
{{/each}}
{{#unless tasks}}
<tr>
<td class="text-center" colspan="6">暂无数据</td>
</tr>
{{/unless}}
</tbody>
</table>
{{# pager}}
<div class="text-right">
{{#if pages}}
<ul class="pagination">
{{# prePage}}
<li><a href="{{url}}">上一页</a></li>
{{/ prePage}}
{{# pages}}
<li class="{{#unless url}}disabled {{/unless}}{{#if cur}}active{{/if}}"><a {{#if url}}href="{{url}}"{{^}}href="javascript:;"{{/if}}>{{num}}</a></li>
{{/ pages}}
{{# nextPage}}
<li><a href="{{url}}">下一页</a></li>
{{/ nextPage}}
</ul>
{{/if}}
</div>
{{/ pager}}
</div>
</div><!--/panel-default-->
</div><!--/contentpanel-->
<script id="task-add-tpl" type="text/tmpl">
<div class="task-add">
<ul>
<li>
执行频率:
<select class="job-select" data-placeholder="请选择..." >
<option value="-1">请选择...</option>
{{#each job}}
<option value="{{@index}}">{{info}}</option>
{{/each}}
</select>
</li>
<li>任务名称:<input type="text" class="job-name" placeholder='请输入任务名称...'></li>
<li>执行链接:<input type="text" class="job-link" placeholder='请输入执行链接...'></li>
<li>延迟时间:<input type="text" class="job-time" placeholder='请输入延迟时间...'></li>
</ul>
<p><span class="err-tip"></span></p>
</div>
</script>
<script id="task-log-tpl" type="text/tmpl">
<div class="task-log">
<table class="table table-striped table-bordered">
<thead>
<tr>
<th class="text-center" width="20%">执行时间</th>
<th class="text-center" width="20%">状态码</th>
<th class="text-center" width="50%">执行结果</th>
<th class="text-right" width="10%">
<a href="javascript:;" class="btn btn-default btn-refresh-log">刷新</a>
</th>
</tr>
</thead>
<tbody></tbody>
</table>
</div>
</script>
<script>
var tasks = {
init: function() {
var that = this;
var taskData = {};
this.$base = $('.contentpanel');
this.$base.on('click', '.btn-add', function() {
that.add();
});
// 单个删除
this.$base.on('click', '.del-btn', function() {
let $tr = $(this).closest('tr');
that.del([$tr.data()], () => $tr.remove());
});
// 日志
this.$base.on('click', '.log-btn', function() {
let $tr = $(this).closest('tr');
taskData = $tr.data();
return that.getTaskLog(taskData).then(rdata => {
layer.open({
type: 1,
area: ['800px', '500px'],
title: '《' + $tr.find('.td-name').text() + '》执行明细',
content: that.taskLogTpl
});
that.rendLog(rdata.data);
});
});
// 日志刷新
$('body').on('click', '.btn-refresh-log', function() {
var layerIndex = layer.load(1);
return that.getTaskLog(taskData).then(rdata => {
setTimeout(function() {
layer.close(layerIndex);
that.rendLog(rdata.data);
}, 2000);
});
});
// 删除全部
this.$base.on('click', '.delete-all', function() {
let arr = [];
$('.tab-task input[type=checkbox]:checked').each(function(){
arr.push($(this).closest('tr').data());
});
that.del(arr, () => history.go(0));
});
this.taskAddTpl = $('#task-add-tpl').html();
this.taskLogTpl = $('#task-log-tpl').html();
},
add: function() {
var that = this;
var layerIndex = layer.open({
type: 1,
area: ['450px'],
title: '任务添加',
content: that.taskAddTpl,
btn: '添加',
yes: function() {
var data = {
type: parseInt($('.job-select').val(), 10),
name: $('.job-name').val(),
url: $('.job-link').val(),
time: parseInt($('.job-time').val() || 0),
};
if (data.type < 0) {
return that.tips('请选择执行频率...', '.job-select');
} else if (data.name === '') {
return that.tips('任务名称必填...', '.job-name');
} else if (data.url === '' || data.url.indexOf('http') === -1) {
return that.tips('请输入正确的执行链接...', '.job-link');
}
return that.ajax({
url: '/seo/task/add',
type: 'POST',
data: data,
}).then(rdata => {
if (rdata.code !== 200) {
return alert(rdata.message);
}
layer.close(layerIndex);
history.go(0)
});
}
});
},
del: function(data, callback) {
if (data.length <= 0) {
return false;
}
var that = this;
var layerIndex = layer.confirm('您确定要删除吗!!!', {
btn: ['确定', '取消']
}, function() {
return that.ajax({
url: '/seo/task/del',
type: 'POST',
data: {data: JSON.stringify(data)},
}).then(res => {
callback();
layer.close(layerIndex);
return res;
});
});
},
rendLog: function(rdata) {
var $tbody = $('.task-log tbody');
$tbody.empty();
if (rdata.length <= 0) {
$tbody.append(`<tr class="empty">
<td class="text-center" colspan="4">暂无数据</td>
</tr>`);
} else {
$.each(rdata, (key, item)=> {
$tbody.append(`<tr>
<td class="text-center" >${item.time}</td>
<td class="text-center" >${item.code}</td>
<td class="text-center" colspan="2">${item.message}</td>
</tr>`);
});
}
},
getTaskLog: function(data) {
return this.ajax({
url: '/seo/task/log',
type: 'POST',
data: data,
});
},
ajax: function(options) {
return Promise.resolve($.ajax(options));
},
tips: function(tip, dom) {
return layer.tips(tip, dom, {
tips: [1, '#000'],
time: 2000
});
}
};
$(function() {
tasks.init();
});
</script>
... ...
... ... @@ -77,6 +77,7 @@
<li><a href="/seo/tdk"><span>TDK管理</span></a></li>
<li><a href="/seo/category"><span>品类描述管理</span></a></li>
<li><a href="/seo/friendlink"><span>友链管理</span></a></li>
<li><a href="/seo/task"><span>定时任务管理</span></a></li>
</ul>
</li>
... ...