asynchronous.js
1.91 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
'use strict';
const cache = global.yoho.cache.master;
const _ = require('lodash');
const logger = global.yoho.logger;
const ASYNCHRONOUSPAGES = {
'/product/index/isFavoriteShop': 1,
'/common/suggestfeedback': 1,
'/product/detail/hotarea': 1,
'/common/getbanner': 1,
'/passport/cert/headerTip': 1
};
function isNormalSpider(userAgent) {
let normalReg = /(spider)|(bot.html)/i;
if (normalReg.test(userAgent)) {
return true;
} else {
return false;
}
}
module.exports = (limiter, policy) => {
const ua = limiter.req.header('User-Agent');
const synchronizeKey = `pc:limiter:synchronize:${limiter.remoteIp}`; // 同步
const asynchronousKey = `pc:limiter:asynchronous:${limiter.remoteIp}`; // 异步
const spiderKey = `pc:limiter:spider:${limiter.remoteIp}`; // 异步
// 正常蜘蛛直接过
if (isNormalSpider(ua)) {
return Promise.resolve(true);
}
const req = limiter.req,
res = limiter.res;
res.on('render', function() {
cache.incrAsync(synchronizeKey, 1).catch(e=>console.log(e)); // eslint-disable-line
});
return cache.getMultiAsync([synchronizeKey, asynchronousKey, spiderKey]).then((results) => {
logger.debug(results);
if (results[spiderKey]) {
return Promise.resolve(policy);
}
// 默认数据设置
if (!results[synchronizeKey] && !_.isNumber(results[synchronizeKey])) {
cache.setAsync(synchronizeKey, 1, 600);
}
// 默认数据设置
if (ASYNCHRONOUSPAGES[req.path] > 0) {
cache.setAsync(asynchronousKey, 1, 600);
cache.setAsync(synchronizeKey, 1, 600);
}
if (results[synchronizeKey] > 10 && !results[asynchronousKey]) {
cache.setAsync(spiderKey, 1, 60 * 60 * 24);
return Promise.resolve(policy);
}
return Promise.resolve(true);
});
};