Merge branch 'hotfix/spider' into gray

@@ -5,6 +5,7 @@ const logger = global.yoho.logger;
 const ip = require('./rules/ip-list');
 const userAgent = require('./rules/useragent');
 const qpsLimiter = require('./rules/qps-limit');
+const asynchronous = require('./rules/asynchronous');
 
 // const fakerLimiter = require('./rules/faker-limit');
 const captchaPolicy = require('./policies/captcha');
@@ -53,7 +54,8 @@ module.exports = (req, res, next) => {
         Promise.all([
             limiter(userAgent, captchaPolicy, context),
             limiter(ip, captchaPolicy, context),
-            limiter(qpsLimiter, captchaPolicy, context)
+            limiter(qpsLimiter, captchaPolicy, context),
+            limiter(asynchronous, captchaPolicy, context)
 
             // limiter(fakerLimiter, reporterPolicy, context)
         ]).then((results) => {

+'use strict';
+
+const cache = global.yoho.cache.master;
+const _ = require('lodash');
+const logger = global.yoho.logger;
+
+
+const ASYNCHRONOUSPAGES = {
+    '/product/index/isFavoriteShop': 1,
+    '/common/suggestfeedback': 1,
+    '/product/detail/hotarea': 1
+};
+
+function isNormalSpider(userAgent) {
+    let normalReg = /(spider)|(bot.html)/i;
+
+    if (normalReg.test(userAgent)) {
+        return true;
+    } else {
+        return false;
+    }
+
+}
+
+module.exports = (limiter, policy) => {
+    const ua = limiter.req.header('User-Agent');
+    const synchronizeKey = `pc:limiter:synchronize:${limiter.remoteIp}`; // 同步
+    const asynchronousKey = `pc:limiter:asynchronous:${limiter.remoteIp}`; // 异步
+    const spiderKey = `pc:limiter:spider:${limiter.remoteIp}`; // 异步
+
+    // 正常蜘蛛直接过
+    if (isNormalSpider(ua)) {
+        return Promise.resolve(true);
+    }
+
+
+    const req = limiter.req,
+        res = limiter.res;
+
+    res.on('render', function() {
+        cache.incrAsync(synchronizeKey, 1).catch(e=>console.log(e)); // eslint-disable-line
+    });
+
+    return cache.getMultiAsync([synchronizeKey, asynchronousKey, spiderKey]).then((results) => {
+        logger.debug(results);
+
+        if (results[spiderKey]) {
+            return Promise.resolve(policy);
+        }
+
+        // 默认数据设置
+        if (!results[synchronizeKey] && !_.isNumber(results[synchronizeKey])) {
+            cache.setAsync(synchronizeKey, 1, 600);  // 设置key，1m失效
+        }
+
+        // 默认数据设置
+        if (ASYNCHRONOUSPAGES[req.path] > 0) {
+            cache.setAsync(asynchronousKey, 1, 600);  // 设置key，1m失效
+        }
+
+        if (results[synchronizeKey] > 10 && !results[asynchronousKey]) {
+            cache.setAsync(spiderKey, 1, 60 * 60 * 24);
+            return Promise.reject(policy);
+        }
+
+    });
+};