Merge branch 'feature/risk' into 'release/9.19'

Feature/risk



See merge request !1544

@@ -137,6 +137,10 @@ app.use((req, res, next) => {
     next();
     next();
 });
 });
 
 
+function isOpenApmRisk(req) {
+    return _.get(req.app.locals, 'wap.open.apmrisk', false);
+}
+
 // dispatcher
 // dispatcher
 try {
 try {
     const tdkUrl = require('./doraemon/middleware/redis-url');
     const tdkUrl = require('./doraemon/middleware/redis-url');
@@ -154,6 +158,8 @@ try {
     const pageCache = require('./doraemon/middleware/page-cache');
     const pageCache = require('./doraemon/middleware/page-cache');
     const downloadBar = require('./doraemon/middleware/download-bar');
     const downloadBar = require('./doraemon/middleware/download-bar');
     const routeEncode = require('./doraemon/middleware/route-encode');
     const routeEncode = require('./doraemon/middleware/route-encode');
+    const ifElseMd = require('./doraemon/middleware/ifElseMd');
+    const riskManagementApm = require('./doraemon/middleware/risk-management2');
     const riskManagement = require('./doraemon/middleware/risk-management');
     const riskManagement = require('./doraemon/middleware/risk-management');
     const statics = require('./doraemon/middleware/statics');
     const statics = require('./doraemon/middleware/statics');
 
 
@@ -168,7 +174,10 @@ try {
 
 
     // 请求限制中间件
     // 请求限制中间件
     if (!app.locals.devEnv) {
     if (!app.locals.devEnv) {
-        app.use(require('./doraemon/middleware/limiter'));
+        const limiter = require('./doraemon/middleware/limiter');
+        const limiterApm = require('./doraemon/middleware/limiter/index2');
+
+        app.use(ifElseMd(isOpenApmRisk, limiterApm, limiter));
     }
     }
 
 
     if (app.locals.devEnv) {
     if (app.locals.devEnv) {
@@ -181,7 +190,7 @@ try {
     app.use(pageCache());
     app.use(pageCache());
     app.use(routeEncode.md);
     app.use(routeEncode.md);
     app.use(downloadBar());
     app.use(downloadBar());
-    app.use(riskManagement());
+    app.use(ifElseMd(isOpenApmRisk, riskManagementApm(), riskManagement()));
     app.use(statics(app));
     app.use(statics(app));
 
 
     require('./dispatch')(app);
     require('./dispatch')(app);

@@ -44,6 +44,7 @@ exports.index = (req, res) => {
     });
     });
 };
 };
 
 
+const limitKey = 'limit2';
 
 
 const submitValidate = {
 const submitValidate = {
     errRes: {
     errRes: {
@@ -54,6 +55,7 @@ const submitValidate = {
     },
     },
     clearLimitIp(req) {
     clearLimitIp(req) {
         let remoteIp = req.yoho.clientIp;
         let remoteIp = req.yoho.clientIp;
+        let operations = [];
 
 
         if (remoteIp.indexOf(',') > 0) {
         if (remoteIp.indexOf(',') > 0) {
             let arr = remoteIp.split(',');
             let arr = remoteIp.split(',');
@@ -61,21 +63,51 @@ const submitValidate = {
             remoteIp = arr[0];
             remoteIp = arr[0];
         }
         }
 
 
-        let operations = [cache.delAsync(`${config.app}:limiter:${remoteIp}`)];
+        const isOpenApmrisk = _.get(req.app.locals, 'wap.open.apmrisk', false);
 
 
-        // 验证码之后一小时之内不再限制qps
-        if (req.session.apiLimitValidate || req.session.apiRiskValidate) {
-            operations.push(cache.setAsync(
-                `${config.app}:limiter:api:ishuman:${remoteIp}`,
-                1,
-                config.LIMITER_IP_TIME
-            ));
+        // 新的计数
+        if (isOpenApmrisk) {
+            operations.push(cache.delAsync(`${config.app}:${limitKey}:${remoteIp}`));
+
+            if (req.session.apiLimitValidate || req.session.apiRiskValidate) {
+                operations.push(cache.setAsync(
+                    `${config.app}:limiter:api:ishuman:${remoteIp}`,
+                    1,
+                    config.LIMITER_IP_TIME
+                ));
+            } else {
+                operations.push(cache.setAsync(
+                    `${config.app}:${limitKey}:ishuman:${remoteIp}`,
+                    1,
+                    config.LIMITER_IP_TIME
+                ));
+
+            }
+
+            _.forEach(config.REQUEST_LIMIT, (val, key) => {
+                operations.push(cache.delAsync(`${config.app}:${limitKey}:${key}:max:${remoteIp}`));
+            });
         } else {
         } else {
-            operations.push(cache.setAsync(
-                `${config.app}:limiter:ishuman:${remoteIp}`,
-                1,
-                config.LIMITER_IP_TIME
-            ));
+            operations.push(cache.delAsync(`${config.app}:limiter:${remoteIp}`));
+
+            // 验证码之后一小时之内不再限制qps
+            if (req.session.apiLimitValidate || req.session.apiRiskValidate) {
+                operations.push(cache.setAsync(
+                    `${config.app}:limiter:api:ishuman:${remoteIp}`,
+                    1,
+                    config.LIMITER_IP_TIME
+                ));
+            } else {
+                operations.push(cache.setAsync(
+                    `${config.app}:limiter:ishuman:${remoteIp}`,
+                    1,
+                    config.LIMITER_IP_TIME
+                ));
+            }
+
+            _.forEach(config.REQUEST_LIMIT, (val, key) => {
+                operations.push(cache.delAsync(`${config.app}:limiter:${key}:max:${remoteIp}`));
+            });
         }
         }
 
 
         delete req.session.apiLimitValidate;
         delete req.session.apiLimitValidate;
@@ -87,10 +119,6 @@ const submitValidate = {
             operations.push(cache.delAsync(riskPid));
             operations.push(cache.delAsync(riskPid));
         }
         }
 
 
-        _.forEach(config.REQUEST_LIMIT, (val, key) => {
-            operations.push(cache.delAsync(`${config.app}:limiter:${key}:max:${remoteIp}`));
-        });
-
         return Promise.all(operations);
         return Promise.all(operations);
     },
     },
     geetest(req, res) {
     geetest(req, res) {
@@ -135,7 +163,7 @@ const submitValidate = {
     imgCheckRisk(req, res) {
     imgCheckRisk(req, res) {
         const self = this;
         const self = this;
 
 
-        co(function * () {
+        co(function*() {
             let result = yield req.ctx(checkModel).verifyImgCheckRisk(req.cookies.udid, req.body.captcha);
             let result = yield req.ctx(checkModel).verifyImgCheckRisk(req.cookies.udid, req.body.captcha);
 
 
             if (result.code === 200) {
             if (result.code === 200) {

@@ -200,7 +200,11 @@ exports.serverError = () => {
             }
             }
 
 
             if (!isHuman) {
             if (!isHuman) {
-                cache.setAsync(`${config.app}:limiter:${remoteIp}`, 1, config.LIMITER_IP_TIME);
+                if (_.get(req.app.locals, 'wap.open.apmrisk', false)) {
+                    cache.setAsync(`${config.app}:limit2:${remoteIp}`, 1, config.LIMITER_IP_TIME);
+                } else {
+                    cache.setAsync(`${config.app}:limiter:${remoteIp}`, 1, config.LIMITER_IP_TIME);
+                }
 
 
                 req.session[sessionLimitKey] = true;
                 req.session[sessionLimitKey] = true;
 
 

+
+
+module.exports = (predict, ifTrueFn, elseFn) => {
+    if (!ifTrueFn) {
+        ifTrueFn = (req, res, next) => next();
+    }
+
+    if (!elseFn) {
+        elseFn = (req, res, next) => next();
+    }
+
+    return (req, res, next) => {
+        if (predict(req, res)) {
+            ifTrueFn(req, res, next);
+        } else {
+            elseFn(req, res, next);
+        }
+    };
+};

+'use strict';
+
+const _ = require('lodash');
+const logger = global.yoho.logger;
+const ip = require('./rules/ip-list2');
+const userAgent = require('./rules/useragent2');
+
+// const asynchronous = require('./rules/asynchronous');
+// const fakerLimiter = require('./rules/faker-limit');
+const captchaPolicy = require('./policies/captcha');
+
+// const reporterPolicy = require('./policies/reporter');
+
+const limiter = (rule, policy, context) => {
+    return rule(context, policy);
+};
+
+module.exports = (req, res, next) => {
+    const remoteIp = req.yoho.clientIp || '';
+    const enabled = !_.get(req.app.locals, 'wap.sys.noLimiter');
+
+    // 开关为关或者未获取到remoteIp，放行
+    if (!enabled || !remoteIp) {
+        logger.debug(`request remote ip: ${remoteIp}; enabled: ${enabled}`);
+        return next();
+    }
+
+    (async function() {
+        const context = {
+            req: req,
+            res: res,
+            next: next,
+            remoteIp: remoteIp
+        };
+
+        let results = await Promise.all([
+            limiter(userAgent, captchaPolicy, context),
+            limiter(ip, captchaPolicy, context),
+        ]);
+
+        let allPass = true, exclusion = false, policy = null;
+
+        logger.debug('limiter result: ' + JSON.stringify(results));
+
+        _.forEach(results, (result) => {
+            if (typeof result === 'object' && !exclusion) {
+                exclusion = result.exclusion;
+            }
+
+            if (typeof result === 'function') {
+                allPass = false;
+                policy = result;
+            }
+        });
+
+        if (exclusion) {
+            return next();
+        } else if (!allPass && policy) {
+            policy(req, res, next);
+        } else {
+            return next();
+        }
+    }()).catch(err => {
+        logger.error(err);
+        return next();
+    });
+};

+'use strict';
+
+const cache = global.yoho.cache.master;
+const logger = global.yoho.logger;
+const config = global.yoho.config;
+
+const limitKey = 'limit2';
+
+module.exports = (limiter, policy) => {
+    const ipBlackKey = `pc:limiter:${limiter.remoteIp}`; // ci ip黑名单
+    const ipLimitKey = `${config.app}:${limitKey}:${limiter.remoteIp}`; // 业务黑名单
+
+    return Promise.all([
+        cache.getAsync(ipBlackKey),
+        cache.getAsync(ipLimitKey)
+    ]).then(result => {
+        let ipBlackRes = result[0];
+        let ipLimitRes = result[1];
+
+        logger.debug(ipBlackKey, ipBlackRes);
+        logger.debug(ipLimitKey, ipLimitRes);
+
+        if ((ipBlackRes && +ipBlackRes > 0) || (ipLimitRes && +ipLimitRes > 0)) {
+            return Promise.resolve(policy);
+        } else {
+            return Promise.resolve(true);
+        }
+    });
+};

+'use strict';
+
+const cache = global.yoho.cache.master;
+const _ = require('lodash');
+const logger = global.yoho.logger;
+
+
+module.exports = (limiter, policy) => {
+    const blackKey = 'pc:limiter:ua:black';
+
+    const ua = limiter.req.header('User-Agent');
+
+    cache.getAsync(blackKey).then((args) => {
+        let blacklist = [];
+
+        try {
+            blacklist = JSON.parse(args) || [];
+        } catch (error) {
+            logger.error(error);
+        }
+
+        if (blacklist.length === 0) {
+            return Promise.resolve(true);
+        }
+
+        const test = (list) => {
+            let result = false;
+
+            _.each(list, (item) => {
+                let regexp;
+
+                try {
+                    regexp = new RegExp(item);
+                } catch (e) {
+                    logger.error(e);
+                }
+
+                if (regexp.test(ua)) {
+                    result = true;
+                }
+            });
+
+            return result;
+        };
+
+        if (test(blacklist)) {
+            return Promise.resolve(policy);
+        } else {
+            return Promise.resolve(true);
+        }
+    });
+
+};

+/**
+ * 控制路由请求次数
+ * @date: 2018/03/05
+ */
+'use strict';
+
+const _ = require('lodash');
+const cache = global.yoho.cache.master;
+const helpers = global.yoho.helpers;
+const pathToRegexp = require('path-to-regexp');
+const logger = global.yoho.logger;
+const md5 = require('yoho-md5');
+
+const statusCode = {
+    code: 4403,
+    data: {},
+    message: '亲，您的访问次数过多，请稍后再试哦...'
+};
+
+const _jumpUrl = (req, res, next, result) => {
+    if (result.code === 4403) {
+        if (req.xhr) {
+            res.set({
+                'Cache-Control': 'no-cache',
+                Pragma: 'no-cache',
+                Expires: (new Date(1900, 0, 1, 0, 0, 0, 0)).toUTCString()
+            });
+            return res.status(403).json(result);
+        }
+        return res.redirect(`${result.data.url}&refer=${req.originalUrl}`);
+    }
+
+    return next();
+};
+
+const limitKey = 'limit2';
+
+module.exports = () => {
+    return (req, res, next) => {
+        // default open
+        if (_.get(req.app.locals.wap, 'close.risk', false)) {
+            return next();
+        }
+
+        let ip = _.get(req.yoho, 'clientIp', '');
+        let path = req.path || '';
+        let risks = _.get(req.app.locals.wap, 'json.risk', []);
+        let router = {};
+
+        logger.debug(`risk => risks: ${JSON.stringify(risks)}, path: ${path}, ip: ${ip}`); // eslint-disable-line
+        if (_.isEmpty(path) || _.isEmpty(risks)) {
+            return next();
+        }
+
+        _.isArray(risks) && risks.some(item => {
+            if (item.state === 'off') {
+                return false;
+            }
+
+            if (!item.regRoute) {
+                item.regRoute = pathToRegexp(item.route);
+            }
+
+            if (item.regRoute.test(path)) {
+                router = item;
+                return true;
+            }
+
+            return false;
+        });
+
+        logger.debug(`risk => router: ${JSON.stringify(router)}, path: ${path}`); // eslint-disable-line
+        if (_.isEmpty(router)) {
+            return next();
+        }
+
+        let keyPath = md5(`${router.regRoute}`);
+        let limitEnable = `wap:risk:${limitKey}:${keyPath}:${ip}`;
+        let checkUrl = helpers.urlFormat('/3party/check', {
+            pid: `wap:risk:${limitKey}:${keyPath}`
+        });
+
+        cache.getAsync(limitEnable)
+            .then(result => {
+                logger.debug(`risk => getCache: ${JSON.stringify(result)}, path: ${path}`); // eslint-disable-line
+                if (result) {
+                    return Object.assign({}, statusCode, {
+                        data: {
+                            url: checkUrl
+                        }
+                    });
+                } else {
+                    return {
+                        code: 200
+                    };
+                }
+            }).then(result => {
+                logger.debug(`risk => result: ${JSON.stringify(result)}, path: ${path}`); // eslint-disable-line
+                return _jumpUrl(req, res, next, result);
+            }).catch(e => {
+                console.log(`risk => path: ${path}, err: ${e.message}`);
+                return next();
+            });
+    };
+};