fe / yoho-seo · Commits

Go to a project

Authored by 郝肖肖
Commit 1fca9a5ae4e217cdcd83c8cc2a0a91d1b239694f 1 parent ff90508b

'task-log'

Inline Side-by-side
Showing 3 changed files with 24 additions and 41 deletions
@@ -4,7 +4,6 @@ const ROOT_PATH = global.ROOT_PATH; @@ -4,7 +4,6 @@ const ROOT_PATH = global.ROOT_PATH;
4 const _ = require('lodash'); 4 const _ = require('lodash');
5 const redis = require(`${ROOT_PATH}/libs/redis`); 5 const redis = require(`${ROOT_PATH}/libs/redis`);
6 const YohoApiModel = require('./../interface/yoho-api'); 6 const YohoApiModel = require('./../interface/yoho-api');
7 -const TaskLogModel = require('./../interface/task-log');  
8 const co = Promise.coroutine; 7 const co = Promise.coroutine;
9 const rp = require('request-promise'); 8 const rp = require('request-promise');
10 const senUrl = 'http://data.zz.baidu.com/urls'; 9 const senUrl = 'http://data.zz.baidu.com/urls';
@@ -18,7 +17,6 @@ class BaiduLinkSubmit extends global.yoho.BaseModel { @@ -18,7 +17,6 @@ class BaiduLinkSubmit extends global.yoho.BaseModel {
18 super(ctx); 17 super(ctx);
19 18
20 this.yohoApiModel = new YohoApiModel(ctx); 19 this.yohoApiModel = new YohoApiModel(ctx);
21 - this.taskLogModel = new TaskLogModel(ctx);  
22 } 20 }
23 21
24 // 向百度发送数据 22 // 向百度发送数据
@@ -115,18 +113,7 @@ class BaiduLinkSubmit extends global.yoho.BaseModel { @@ -115,18 +113,7 @@ class BaiduLinkSubmit extends global.yoho.BaseModel {
115 113
116 return rdata; 114 return rdata;
117 115
118 - })().then(rdata => {  
119 - let key = this.taskLogModel.getKey('http://127.0.0.1:6005/goodsUrl');  
120 -  
121 - return this.taskLogModel.ltrim(key).then(() => {  
122 - return Promise.all([  
123 - this.taskLogModel.add(key, rdata[0]),  
124 - this.taskLogModel.add(key, rdata[1])  
125 - ]);  
126 - }).then(() => {  
127 - return rdata;  
128 - });  
129 - }); 116 + })();
130 } 117 }
131 118
132 // 提交逛详情URL 119 // 提交逛详情URL
@@ -194,18 +181,7 @@ class BaiduLinkSubmit extends global.yoho.BaseModel { @@ -194,18 +181,7 @@ class BaiduLinkSubmit extends global.yoho.BaseModel {
194 Object.assign(rdata[1], {message: `WAP: ${message}`}); 181 Object.assign(rdata[1], {message: `WAP: ${message}`});
195 182
196 return rdata; 183 return rdata;
197 - })().then(rdata => {  
198 - let key = this.taskLogModel.getKey('http://127.0.0.1:6005/guangUrl');  
199 -  
200 - return this.taskLogModel.ltrim(key).then(() => {  
201 - return Promise.all([  
202 - this.taskLogModel.add(key, rdata[0]),  
203 - this.taskLogModel.add(key, rdata[1])  
204 - ]);  
205 - }).then(() => {  
206 - return rdata;  
207 - });  
208 - }); 184 + })();
209 } 185 }
210 186
211 } 187 }
@@ -6,7 +6,6 @@ const co = Promise.coroutine; @@ -6,7 +6,6 @@ const co = Promise.coroutine;
6 const rp = require('request-promise'); 6 const rp = require('request-promise');
7 const redis = require(`${ROOT_PATH}/libs/redis`); 7 const redis = require(`${ROOT_PATH}/libs/redis`);
8 const YohoApiModel = require('./../interface/yoho-api'); 8 const YohoApiModel = require('./../interface/yoho-api');
9 -const TaskLogModel = require('./../interface/task-log');  
10 const senUrl = 'http://data.zz.baidu.com/urls?appid=1583402501013173&token=K0L5PUhk1XOko81r&type='; 9 const senUrl = 'http://data.zz.baidu.com/urls?appid=1583402501013173&token=K0L5PUhk1XOko81r&type=';
11 10
12 // http://ziyuan.baidu.com/xzh/commit/push?appid=1583402501013173&qq-pf-to=pcqq.c2c 11 // http://ziyuan.baidu.com/xzh/commit/push?appid=1583402501013173&qq-pf-to=pcqq.c2c
@@ -16,7 +15,6 @@ class XzhIndexModel extends global.yoho.BaseModel { @@ -16,7 +15,6 @@ class XzhIndexModel extends global.yoho.BaseModel {
16 super(ctx); 15 super(ctx);
17 16
18 this.yohoApiModel = new YohoApiModel(ctx); 17 this.yohoApiModel = new YohoApiModel(ctx);
19 - this.taskLogModel = new TaskLogModel(ctx);  
20 } 18 }
21 19
22 index() { 20 index() {
@@ -65,15 +63,7 @@ class XzhIndexModel extends global.yoho.BaseModel { @@ -65,15 +63,7 @@ class XzhIndexModel extends global.yoho.BaseModel {
65 tids = []; 63 tids = [];
66 64
67 return Object.assign(rdata, {message: message}); 65 return Object.assign(rdata, {message: message});
68 - })().then(rdata => {  
69 - let key = this.taskLogModel.getKey('http://127.0.0.1:6005/sendXzh');  
70 -  
71 - return this.taskLogModel.ltrim(key).then(() => {  
72 - return this.taskLogModel.add(key, rdata);  
73 - }).then(() => {  
74 - return rdata;  
75 - });  
76 - }); 66 + })();
77 } 67 }
78 68
79 // 向百度发送数据 69 // 向百度发送数据
@@ -2,12 +2,15 @@ @@ -2,12 +2,15 @@
2 2
3 const ROOT_PATH = global.ROOT_PATH; 3 const ROOT_PATH = global.ROOT_PATH;
4 const os = require('os'); 4 const os = require('os');
  5 +const _ = require('lodash');
5 const util = require(`${ROOT_PATH}/libs/util`); 6 const util = require(`${ROOT_PATH}/libs/util`);
6 const schedule = require('node-schedule'); 7 const schedule = require('node-schedule');
7 const rp = require('request-promise'); 8 const rp = require('request-promise');
8 const redis = require(`${ROOT_PATH}/libs/redis`); 9 const redis = require(`${ROOT_PATH}/libs/redis`);
9 const config = require(`${ROOT_PATH}/config/config`); 10 const config = require(`${ROOT_PATH}/config/config`);
10 const jobTask = config.jobTask; 11 const jobTask = config.jobTask;
  12 +const md5 = require('yoho-md5');
  13 +const TASK_KEY_LOG = 'global:yoho:task:log';
11 14
12 const SeoTask = { 15 const SeoTask = {
13 intHandle() { 16 intHandle() {
@@ -39,13 +42,27 @@ const SeoTask = { @@ -39,13 +42,27 @@ const SeoTask = {
39 json: true, 42 json: true,
40 gzip: true 43 gzip: true
41 }).then(rdata => { 44 }).then(rdata => {
42 - console.log(`url: ${url}, result: ${JSON.stringify(rdata)}`);  
43 - return rdata; 45 + return this.addLog(url, rdata);
44 }).catch(e => { 46 }).catch(e => {
45 - console.error(`execTask err: ${e.message}`);  
46 - return {code: 400, message: e.message}; 47 + return this.addLog(url, {code: e.statusCode || 500, message: e.message});
47 }); 48 });
48 }); 49 });
  50 + },
  51 + addLog(url, rdata) {
  52 + let key = `${TASK_KEY_LOG}:${md5(url)}`;
  53 +
  54 + console.log(`url: ${url}, result: ${JSON.stringify(rdata)}`);
  55 +
  56 + rdata = _.isArray(rdata) ? rdata : [rdata];
  57 +
  58 + return redis.ltrimAsync(key, 0, 1000).then(() => {
  59 + return Promise.all(_.map(
  60 + rdata,
  61 + item => redis.lpushAsync(key, JSON.stringify(Object.assign({time: Date.now()}, item)))
  62 + ));
  63 + }).then(() => {
  64 + return rdata;
  65 + });
49 } 66 }
50 }; 67 };
51 68