Merge branch 'feature/anti-spider' into 'master'

爬虫添加验证页面



See merge request !338

@@ -51,6 +51,11 @@ app.set('etag', false);
 
 
 app.enable('trust proxy');
 app.enable('trust proxy');
 
 
+// 请求限制中间件
+if (!app.locals.devEnv) {
+    app.use(require('./doraemon/middleware/limiter'));
+}
+
 // 指定libray目录
 // 指定libray目录
 global.utils = path.resolve('./utils');
 global.utils = path.resolve('./utils');
 
 

+'use strict';
+const _ = require('lodash');
+const cache = global.yoho.cache.master;
+
+exports.index = (req, res) => {
+    res.render('check', {
+        width750: true,
+        localCss: true
+    });
+};
+
+exports.submit = (req, res) => {
+    let captchaCode = _.get(req.session, 'captcha');
+    let remoteIp = req.get('X-Forwarded-For') || req.ip;
+
+    if (remoteIp.indexOf(',') > 0) {
+        let arr = remoteIp.split(',');
+
+        remoteIp = arr[0];
+    }
+
+    if (req.body.captcha === captchaCode) {
+        let key = `pc:limiter:${remoteIp}`;
+        cache.delAsync(key).then(() => {
+            return res.json({
+                code: 200
+            });
+        }).catch(() => {
+            return res.json({
+                code: 400
+            });
+        });
+    } else {
+        return res.json({
+            code: 400
+        });
+    }
+
+};

@@ -9,9 +9,12 @@
 const router = require('express').Router(); // eslint-disable-line
 const router = require('express').Router(); // eslint-disable-line
 const cRoot = './controllers';
 const cRoot = './controllers';
 const ads = require(`${cRoot}/ads`);
 const ads = require(`${cRoot}/ads`);
+const check = require(`${cRoot}/check`);
 
 
 // routers
 // routers
 
 
 router.get('/ads', ads.index);
 router.get('/ads', ads.index);
+router.get('/check', check.index);
+router.post('/check/submit', check.submit);
 
 
 module.exports = router;
 module.exports = router;

+<div class="check-page">
+    <div class="title">请输入正确的验证码，继续访问</div>
+    <div id="js-img-check"></div>
+    <div class="submit">
+        确认
+    </div>
+</div>

@@ -72,7 +72,6 @@ module.exports = {
         udp: { // send by udp
         udp: { // send by udp
             measurement: 'yohobuy_wap_node_log',
             measurement: 'yohobuy_wap_node_log',
             level: 'error', // logger level
             level: 'error', // logger level
-            host: 'influxdblog.web.yohoops.org', // influxdb host
             port: '4444' // influxdb port
             port: '4444' // influxdb port
         },
         },
         console: {
         console: {
@@ -107,7 +106,8 @@ module.exports = {
         key: '7e6f3307b64cc87c79c472814b88f7fb',
         key: '7e6f3307b64cc87c79c472814b88f7fb',
         appSecret: 'ce21ae4a3f93852279175a167e54509b',
         appSecret: 'ce21ae4a3f93852279175a167e54509b',
         notifyUrl: domains.service + 'payment/weixin_notify',
         notifyUrl: domains.service + 'payment/weixin_notify',
-    }
+    },
+    maxQps: 1200
 };
 };
 
 
 if (isProduction) {
 if (isProduction) {

+'use strict';
+const limiter = require('../middleware/limiter/index');
+
+module.exports = (req, res, next) => {
+    return limiter(req, res, next);
+};

+'use strict';
+
+const _ = require('lodash');
+const logger = global.yoho.logger;
+const ip = require('./rules/ip-list');
+const userAgent = require('./rules/useragent');
+const qpsLimiter = require('./rules/qps-limit');
+const fakerLimiter = require('./rules/faker-limit');
+const captchaPolicy = require('./policies/captcha');
+const reporterPolicy = require('./policies/reporter');
+
+const IP_WHITE_LIST = [
+    // '106.38.38.146',
+    // '218.94.75.58'
+];
+
+const limiter = (rule, policy, context) => {
+    return rule(context, policy);
+};
+
+module.exports = (req, res, next) => {
+    let remoteIp = req.get('X-Forwarded-For') || req.connection.remoteAddress;
+    logger.debug('request remote ip: ', remoteIp);
+
+    if (remoteIp.indexOf(',') > 0) {
+        let arr = remoteIp.split(',');
+
+        remoteIp = arr[0];
+    }
+
+    const excluded = _.includes(IP_WHITE_LIST, remoteIp);
+    const enabled = !_.get(req.app.locals, 'wap.sys.noLimiter');
+
+    // 判断获取remoteIp成功，并且开关未关闭
+    if (enabled && remoteIp && !excluded) {
+        const context = {
+            req: req,
+            res: res,
+            next: next,
+            remoteIp: remoteIp
+        };
+
+        Promise.all([
+            limiter(userAgent, captchaPolicy, context),
+            limiter(ip, captchaPolicy, context),
+            limiter(qpsLimiter, captchaPolicy, context),
+            //limiter(fakerLimiter, reporterPolicy, context)
+        ]).then((results) => {
+            let allPass = true, exclusion = false, policy = null;
+
+            logger.debug('limiter result: ' + JSON.stringify(results));
+
+            _.forEach(results, (result) => {
+                if (typeof result === 'object' && !exclusion) {
+                    exclusion = result.exclusion;
+                }
+
+                if (!excluded && typeof result === 'function') {
+                    allPass = false;
+                }
+
+                if (typeof result === 'function') {
+                    policy = result;
+                }
+            });
+
+            if (exclusion) {
+                return next();
+            } else if (!allPass && policy) {
+                policy(req, res, next);
+            } else {
+                return next();
+            }
+
+        }).catch((err) => {
+            logger.error(err);
+            return next();
+        });
+    } else {
+        return next();
+    }
+};

+'use strict';
+
+const helpers = global.yoho.helpers;
+const _ = require('lodash');
+
+const WHITE_LIST = [
+    '/3party/check',
+    '/passport/imagesNode',
+    '/passport/cert/headerTip',
+    '/passport/captcha/get',
+    '/3party/check/submit'
+];
+
+module.exports = (req, res, next) => {
+    let refer = req.method === 'GET' ? req.get('Referer') : '';
+    let limitAPI = helpers.urlFormat('/3party/check', {refer: refer});
+    let limitPage = helpers.urlFormat('/3party/check', {refer: req.protocol + '://' + req.get('host') + req.originalUrl});
+
+    if (_.indexOf(WHITE_LIST, req.path) >= 0) {
+        return next();
+    }
+
+    if (req.xhr) {
+        return res.json({
+            code: 400,
+            data: {refer: limitAPI}
+        });
+    }
+
+    return res.redirect(limitPage);
+};

+'use strict';
+
+module.exports = (req, res, next) => {
+    return next();
+};

+'use strict';
+
+const logger = global.yoho.logger;
+const cache = global.yoho.cache.master;
+const ONE_DAY = 60 * 60 * 24;
+
+module.exports = (limiter, policy) => {
+    const req = limiter.req,
+        res = limiter.res,
+        next = limiter.next;
+
+    const key = `pc:limiter:faker:${limiter.remoteIp}`;
+
+    if (req.header('X-Requested-With') === 'XMLHttpRequest') {
+        cache.decrAsync(key, 1);
+    }
+
+    res.on('render', function() {
+        cache.incrAsync(key, 1);
+    });
+
+    return cache.getAsync(key).then((result) => {
+        if (result) {
+            if (result > 100) {
+                return Promise.resolve(policy);//policy(req, res, next);
+            } else {
+                return Promise.resolve(true);
+            }
+        } else {
+            cache.setAsync(key, 1, ONE_DAY);  // 设置key，1m失效
+            return Promise.resolve(true);
+        }
+    });
+};

+'use strict';
+
+const cache = global.yoho.cache.master;
+const _ = require('lodash');
+
+module.exports = (limiter) => {
+    const key = `pc:limiter:${limiter.remoteIp}`;
+
+    return cache.getAsync(key).then((result) => {
+        if (result && _.isNumber(result)) {
+            return Promise.resolve({
+                exclusion: result === -1
+            });
+        } else {
+            return Promise.resolve(true);
+        }
+    });
+};

+'use strict';
+
+const logger = global.yoho.logger;
+const cache = global.yoho.cache.master;
+const config = global.yoho.config;
+const ONE_DAY = 60 * 60 * 24;
+const MAX_QPS = config.maxQps;
+const _ = require('lodash');
+
+const PAGES = {
+    '/product/\\/pro_([\\d]+)_([\\d]+)\\/(.*)/': 5,
+    '/product/list/index': 5
+};
+
+function urlJoin(a, b) {
+    if (_.endsWith(a, '/') && _.startsWith(b, '/')) {
+        return a + b.substring(1, b.length);
+    } else if (!_.endsWith(a, '/') && !_.startsWith(b, '/')) {
+        return a + '/' + b;
+    } else {
+        return a + b;
+    }
+}
+
+module.exports = (limiter, policy) => {
+    const req = limiter.req,
+        res = limiter.res,
+        next = limiter.next;
+
+    const key = `pc:limiter:${limiter.remoteIp}`;
+
+    res.on('render', function() {
+        let route = req.route ? req.route.path : '';
+        let appPath = req.app.mountpath;
+
+        if (_.isArray(route) && route.length > 0) {
+            route = route[0];
+        }
+
+        let pageKey = urlJoin(appPath, route.toString()); // route may be a regexp
+        let pageIncr = PAGES[pageKey] || 0;
+
+        if (/^\/p([\d]+)/.test(req.path)) {
+            pageIncr = 5;
+        }
+
+        if (pageIncr > 0) {
+            cache.incrAsync(key, pageIncr, (err) => {});
+        }
+    });
+
+    return cache.getAsync(key).then((result) => {
+        logger.debug('qps limiter: ' + key + '@' + result + ' max: ' + MAX_QPS);
+
+        if (result && _.isNumber(result)) {
+
+            if (result === -1) {
+                return Promise.resolve(true);
+            }
+
+            if (result > MAX_QPS) {       // 判断 qps
+                cache.touch(key, ONE_DAY);
+                logger.debug('req limit', key);
+
+                return Promise.resolve(policy);
+            } else {
+                cache.incrAsync(key, 1); // qps + 1
+                return Promise.resolve(true);
+
+            }
+        } else {
+            cache.setAsync(key, 1, 60);  // 设置key，1m失效
+            return Promise.resolve(true);
+        }
+    });
+};

+'use strict';
+
+const cache = global.yoho.cache.master;
+const _ = require('lodash');
+const logger = global.yoho.logger;
+
+
+module.exports = (limiter, policy) => {
+    const req = limiter.req,
+        res = limiter.res,
+        next = limiter.next;
+    const blackKey = 'pc:limiter:ua:black',
+        whiteKey = 'pc:limiter:ua:white';
+
+    const ua = limiter.req.header('User-Agent');
+
+    return Promise.all([
+        cache.getAsync(blackKey),
+        cache.getAsync(whiteKey)
+    ]).then((args) => {
+        const blacklist = args[0] || [], whitelist = args[1] || [];
+
+        if (blacklist.length === 0 && whitelist.length === 0) {
+            return Promise.resolve(true);
+        }
+
+        const test = (list) => {
+            let result = false;
+
+            _.each(list, (item) => {
+                let regexp;
+
+                try {
+                    regexp = new RegExp(item);
+                } catch (e) {
+                    logger.error(e);
+                }
+
+                if (regexp.test(ua)) {
+                    result = true;
+                }
+            });
+
+            return result;
+        };
+
+        if (test(blacklist)) {
+            return Promise.resolve(policy);
+        } else if (test(whitelist)) {
+            return Promise.resolve({
+                exclusion: true
+            });
+        } else {
+            return Promise.resolve(true);
+        }
+    });
+
+};

 {
 {
   "name": "m-yohobuy-node",
   "name": "m-yohobuy-node",
-  "version": "5.4.26",
+  "version": "5.4.27",
   "private": true,
   "private": true,
   "description": "A New Yohobuy Project With Express",
   "description": "A New Yohobuy Project With Express",
   "repository": {
   "repository": {

+require('3party/check.page.css');
+require('../common');
+// 图片验证码
+let ImgCheck = require('plugin/img-check');
+
+let imgCheck = new ImgCheck('#js-img-check', {
+    useREM: {
+        rootFontSize: 40,
+        picWidth: 150
+    }
+});
+
+imgCheck.init();
+
+$(function() {
+    $('.submit').on('click', function() {
+        $.ajax({
+            method: 'POST',
+            url: '/3party/check/submit',
+            data: {
+                captcha: $.trim(imgCheck.getResults())
+            },
+            success: function(ret) {
+                if (ret.code === 200) {
+                    window.location.href = decodeURIComponent(window.queryString.refer) || '//m.yohobuy.com';
+                } else {
+                    imgCheck.refresh();
+                }
+            }
+        });
+    });
+});

+@import "layout/img-check";
+
+.check-page {
+    margin: 20px auto;
+    width: 700px;
+
+    .submit {
+        width: 100%;
+        height: 100px;
+        line-height: 100px;
+        text-align: center;
+        font-size: 32px;
+        color: #fff;
+        background: #5cb85c;
+        border-radius: 10px;
+    }
+}