Merge branch 'feature/limiter' into 'gray'

Feature/limiter



See merge request !1325

 {
     "extends": "yoho",
+    "parser": "babel-eslint",
     "parserOptions": {
         "sourceType": "module"
     },
@@ -10,4 +11,4 @@
         "camelcase": "off"
     }
 
-}
\ No newline at end of file
+}

@@ -59,7 +59,7 @@ exports.submit = (req, res) => {
         if (geetestRes) {
             logger.info('geetest success');
 
-            let remoteIp = req.get('X-Forwarded-For') || req.ip;
+            let remoteIp = req.yoho.clientIp;
 
             if (remoteIp.indexOf(',') > 0) {
                 let arr = remoteIp.split(',');
@@ -67,7 +67,25 @@ exports.submit = (req, res) => {
                 remoteIp = arr[0];
             }
 
-            let operations = [cache.delAsync(`${config.app}:limiter:${remoteIp}`)];
+            // pc:limiter:IP 和PC端共用
+            let operations = [cache.delAsync(`pc:limiter:${remoteIp}`)];
+
+            // 验证码之后一小时之内不再限制qps
+            if (req.session.apiLimitValidate) {
+                operations.push(cache.setAsync(
+                    `${config.app}:limiter:api:ishuman:${remoteIp}`,
+                    1,
+                    config.LIMITER_IP_TIME
+                ));
+            } else {
+                operations.push(cache.setAsync(
+                    `${config.app}:limiter:ishuman:${remoteIp}`,
+                    1,
+                    config.LIMITER_IP_TIME
+                ));
+            }
+
+            delete req.session.apiLimitValidate;
 
             if (req.body.pid) {
                 let riskPid = decodeURIComponent(req.body.pid) + ':' + _.get(req.yoho, 'clientIp', '');

@@ -160,6 +160,7 @@ module.exports = {
         // 100s 最多访问200次
         600: 200
     },
+    LIMITER_IP_TIME: 3600, // 超出访问限制ip限制访问1小时
     superCapture: '93c70db61fe276f93ce781ad17dc47cd',
     from: from
 };

@@ -4,8 +4,10 @@
  */
 const headerModel = require('../models/header');
 const logger = global.yoho.logger;
+const cache = global.yoho.cache.master;
 const helpers = global.yoho.helpers;
 const sender = global.yoho.apmSender;
+const config = global.yoho.config;
 const hostname = require('os').hostname();
 const routeEncode = require('./route-encode');
 const _ = require('lodash');
@@ -20,6 +22,51 @@ const forceNoCache = (res) => {
     }
 };
 
+const _err510 = (req, res, code, err) => {
+    res.status(code);
+
+    if (req.xhr) {
+        return res.json({
+            code: err.code,
+            message: '服务器繁忙请稍后重试！'
+        });
+    }
+
+    return res.render('error/510', {
+        err: err,
+        module: 'common',
+        page: 'error',
+        title: '服务器繁忙 | Yoho!Buy有货 | 潮流购物逛不停',
+        pageHeader: headerModel.setNav({
+            navTitle: '服务器繁忙请稍后重试！'
+        }),
+        pageFooter: true,
+        isErr: true
+    });
+};
+
+const _err500 = (req, res, code, err) => {
+    res.status(code);
+
+    if (req.xhr) {
+        return res.json({
+            code: err.code,
+            message: '服务器错误！'
+        });
+    }
+
+    return res.render('error/500', {
+        err: err,
+        module: 'common',
+        page: 'error',
+        title: '服务器错误 | Yoho!Buy有货 | 潮流购物逛不停',
+        pageHeader: headerModel.setNav({
+            navTitle: '服务器错误！'
+        }),
+        pageFooter: true,
+        isErr: true
+    });
+};
 
 exports.notFound = () => {
     return (req, res) => {
@@ -51,11 +98,12 @@ exports.notFound = () => {
  * @return {[type]}
  */
 exports.serverError = () => {
-    return (err, req, res, next) => {
-        forceNoCache(res);
+    return async(err, req, res, next) => {
         const uid = req.user ? req.user.uid : 0;
         const udid = _.get(req, 'cookies.udid', 'yoho');
 
+        forceNoCache(res);
+
         err.code = err.code || err.statusCode || 500;
 
         if (req.isApmReport && err.code !== 401) {
@@ -63,7 +111,7 @@ exports.serverError = () => {
             sender.addMessage({
                 measurement: 'error-report',
                 tags: {
-                    app: global.yoho.config.appName, // 应用名称
+                    app: config.appName, // 应用名称
                     hostname,
                     type: 'server',
                     route: `[${req.method}]${_.get(req, 'route.path', '')}`, // 请求路由
@@ -106,33 +154,45 @@ exports.serverError = () => {
                     refer: req.originalUrl
                 }));
             }
-        }
+        } else if (err.code === 9999991 || err.code === 9999992) {
+            let remoteIp = req.yoho.clientIp;
 
-        logger.error(`error at path: ${req.url}`);
-        logger.error(err);
+            const isHuman = await cache.getAsync(`${config.app}:limiter:api:ishuman:${remoteIp}`);
 
-        if (!res.headersSent) {
-            res.status(err.code);
+            if (!isHuman) {
+                if (remoteIp.indexOf(',') > 0) {
+                    let arr = remoteIp.split(',');
 
-            if (req.xhr) {
-                return res.json({
-                    code: err.code,
-                    message: '服务器错误！'
+                    remoteIp = arr[0];
+                }
+                cache.setAsync(`${config.app}:limiter:${remoteIp}`, 1, config.LIMITER_IP_TIME);
+
+                let limitAPI = helpers.urlFormat('/3party/check', {refer: req.get('Referer') || ''});
+                let limitPage = helpers.urlFormat('/3party/check', {
+                    refer: req.protocol + '://' + req.get('host') + req.originalUrl
                 });
+
+                req.session.apiLimitValidate = true;
+                if (req.xhr) {
+                    return res.status(510).json({
+                        code: err.code,
+                        data: {refer: limitAPI}
+                    });
+                }
+
+                return res.redirect(limitPage);
             }
 
-            return res.render('error/500', {
-                err: err,
-                module: 'common',
-                page: 'error',
-                title: '服务器错误 | Yoho!Buy有货 | 潮流购物逛不停',
-                pageHeader: headerModel.setNav({
-                    navTitle: '服务器错误！'
-                }),
-                pageFooter: true,
-                isErr: true
-            });
+            return _err510(req, res, 510, err);
+        }
+
+        logger.error(`error at path: ${req.url}`);
+        logger.error(err);
+
+        if (!res.headersSent) {
+            return _err500(req, res, err.code, err);
         }
-        next(err);
+
+        return next(err);
     };
 };

@@ -70,7 +70,7 @@ module.exports = (req, res, next) => {
     // 排除条件：ip白名单/路径白名单/异步请求/登录用户
     const excluded = _.includes(IP_WHITE_LIST, remoteIp) ||
         _.includes(PATH_WHITE_LIST, req.path) || req.xhr || !_.isEmpty(_.get(req, 'user.uid'));
-    const enabled = !_.get(req.app.locals, 'wap.sys.noLimite');
+    const enabled = !_.get(req.app.locals, 'wap.sys.noLimiter');
 
     logger.debug(`request remote ip: ${remoteIp}; excluded: ${excluded}; enabled: ${enabled}`);
 

@@ -2,6 +2,7 @@
 
 const helpers = global.yoho.helpers;
 const _ = require('lodash');
+
 const WHITE_LIST = [
     '/3party/check',
     '/3party/check/submit',
@@ -27,8 +28,7 @@ module.exports = (req, res, next) => {
     }
 
     if (req.xhr) {
-        return res.json({
-            code: 400,
+        return res.status(510).json({
             data: {refer: limitAPI}
         });
     }

@@ -26,9 +26,9 @@ function isNormalSpider(userAgent) {
 
 module.exports = (limiter, policy) => {
     const ua = limiter.req.header('User-Agent');
-    const synchronizeKey = `wap:limiter:synchronize:${limiter.remoteIp}`; // 同步
-    const asynchronousKey = `wap:limiter:asynchronous:${limiter.remoteIp}`; // 异步
-    const spiderKey = `wap:limiter:spider:${limiter.remoteIp}`; // 异步
+    const synchronizeKey = `pc:limiter:synchronize:${limiter.remoteIp}`; // 同步
+    const asynchronousKey = `pc:limiter:asynchronous:${limiter.remoteIp}`; // 异步
+    const spiderKey = `pc:limiter:spider:${limiter.remoteIp}`; // 异步
 
     // 正常蜘蛛直接过
     if (isNormalSpider(ua)) {

@@ -9,7 +9,7 @@ module.exports = (limiter, policy) => {
         res = limiter.res,
         next = limiter.next; // eslint-disable-line
 
-    const key = `wap:limiter:faker:${limiter.remoteIp}`;
+    const key = `pc:limiter:faker:${limiter.remoteIp}`;
 
     if (req.header('X-Requested-With') === 'XMLHttpRequest') {
         cache.decrAsync(key, 1);

@@ -10,9 +10,6 @@ const cache = global.yoho.cache.master;
 const config = global.yoho.config;
 const _ = require('lodash');
 
-// 超出访问限制ip限制访问1小时
-const limiterIpTime = 3600;
-
 // 页面访问限制
 const MAX_TIMES = config.REQUEST_LIMIT;
 
@@ -27,12 +24,18 @@ module.exports = (limiter, policy) => {
         getOp[key] = cache.getAsync(ruleKeys[key]);
     });
 
+    getOp.human = cache.getAsync(`${config.app}:limiter:ishuman:${limiter.remoteIp}`);
+
     return Promise.props(getOp).then((results) => {
 
         logger.debug(MAX_TIMES);
         logger.debug(_.values(ruleKeys));
         logger.debug(results);
 
+        if (results.human) { // 经过验证码之后1小时有效期内不再验证qps
+            return Promise.resolve(true);
+        }
+
         // 遍历限制规则，若满足返回相应处理策略， 否则页面访问次数加1
         let operation = [];
 
@@ -42,9 +45,10 @@ module.exports = (limiter, policy) => {
             if (!results[key]) {
                 operation.push(cache.setAsync(cacheKey, 1, +key));
             } else if (+results[key] > +val) {
+                logger.warn(`pc:limiter:${limiter.remoteIp}`);
 
                 // ip限制1小时
-                operation.push(cache.setAsync(`pc:limiter:${limiter.remoteIp}`, 1, limiterIpTime));
+                operation.push(cache.setAsync(`pc:limiter:${limiter.remoteIp}`, 1, config.LIMITER_IP_TIME));
                 return Promise.resolve(policy);
             } else {
                 operation.push(cache.incrAsync(cacheKey, 1));

+<div class="err-page yoho-page">
+    <div class="err-bg"></div>
+</div>

@@ -74,13 +74,14 @@
     "xml2js": "^0.4.19",
     "yoho-express-session": "^2.0.0",
     "yoho-md5": "^2.0.0",
-    "yoho-node-lib": "=0.6.2",
+    "yoho-node-lib": "=0.6.5",
     "yoho-zookeeper": "^1.0.9"
   },
   "devDependencies": {
     "@mapbox/stylelint-processor-arbitrary-tags": "^0.2.0",
     "autoprefixer": "^7.2.4",
     "babel-core": "^6.26.0",
+    "babel-eslint": "^8.2.2",
     "babel-loader": "^7.1.2",
     "babel-plugin-transform-runtime": "^6.23.0",
     "babel-preset-env": "^1.6.0",

@@ -49,6 +49,8 @@ $(document).ajaxError((event, xhr) => {
                 window.location.href = `/signin.html?refer=${encodeURIComponent(window.location.href)}`;
             }
         }
+    } else if (xhr.status === 510 && xhr.responseJSON.data && xhr.responseJSON.data.refer) {
+        window.location.href = xhr.responseJSON.data.refer;
     } else if (xhr.status === 403 && xhr.responseJSON.code === 4403) {
         tip.show(xhr.responseJSON.message);
         setTimeout(function() {