Merge branch 'release/9.19' into 'master'

Release/9.19



See merge request !380

 proxy: d:\soft\traefik\traefik.exe -c d:\soft\traefik\conf.toml
-mem: d:\soft\memcached\memcached.exe -vv
+redis: d:\soft\redis\redis-server.exe d:\soft\redis\redis.windows.conf

@@ -141,10 +141,15 @@ app.use((req, res, next) => {
 
 const logger = global.yoho.logger;
 
+function isOpenApmRisk(req) {
+    return _.get(req.app.locals, 'pc.open.apmrisk', false);
+}
+
 // dispatcher
 try {
     const setYohoData = require('./doraemon/middleware/set-yoho-data');
     const riskManagement = require('./doraemon/middleware/risk-management');
+    const riskManagementApm = require('./doraemon/middleware/risk-management2');
     const htaccess = require('./doraemon/middleware/htaccess');
     const subDomain = require('./doraemon/middleware/sub-domain');
     const mobileRefer = require('./doraemon/middleware/mobile-refer');
@@ -158,10 +163,11 @@ try {
     const devtool = require('./doraemon/middleware/devtools');
     const routeEncode = require('./doraemon/middleware/route-encode');
     const footerData = require('./doraemon/middleware/footerData');
+    const ifElseMd = require('./doraemon/middleware/ifElseMd');
 
     // YOHO 前置中间件
     app.use(setYohoData());
-    app.use(riskManagement());
+    app.use(ifElseMd(isOpenApmRisk, riskManagementApm(), riskManagement()));
     app.use(seo());
     app.use(htaccess());
     app.use(subDomain());
@@ -171,7 +177,10 @@ try {
 
     // 请求限制中间件
     if (!app.locals.devEnv) {
-        app.use(require('./doraemon/middleware/limiter'));
+        const limiter = require('./doraemon/middleware/limiter');
+        const limiterApm = require('./doraemon/middleware/limiter/index2');
+
+        app.use(ifElseMd(isOpenApmRisk, limiterApm, limiter));
     }
 
     app.use(setPageInfo());
@@ -187,7 +196,7 @@ try {
     require('./dispatch')(app);
 
     app.use('/crm/common/ok.jsp', (req, res) => {
-        res.status(204).end();
+        res.status(200).end();
     });
 
     app.all('*', errorHandler.notFound()); // 404

@@ -10,6 +10,12 @@ const humanExpire = 3600;
 
 const PAGE = 'PC';
 
+const limitKey = 'limit2';
+
+
+/**
+ * note: 这里要注意，新的计数是指APM处理的情况
+ */
 module.exports = class extends global.yoho.BaseModel {
     constructor(ctx) {
         super(ctx);
@@ -24,18 +30,36 @@ module.exports = class extends global.yoho.BaseModel {
             pid && operations.push(cache.delAsync(`${pid}:${remoteIp}`));
         }
 
-        operations.push(cache.delAsync(`${config.app}:limiter:${remoteIp}`));
+        const isOpenApmrisk = _.get(this.ctx.req.app.locals, 'pc.open.apmrisk', false);
+
+        // 新的计数
+        if (isOpenApmrisk) {
+            operations.push(cache.delAsync(`${config.app}:${limitKey}:${remoteIp}`));
+
+            // 验证码之后一小时之内不再限制qps
+            if (apiLimitValidate) {
+                operations.push(cache.setAsync(`${config.app}:limiter:api:ishuman:${remoteIp}`, 1, humanExpire));
+            } else {
+                operations.push(cache.setAsync(`${config.app}:${limitKey}:ishuman:${remoteIp}`, 1, humanExpire));
+            }
 
-        // 验证码之后一小时之内不再限制qps
-        if (apiLimitValidate) {
-            operations.push(cache.setAsync(`${config.app}:limiter:api:ishuman:${remoteIp}`, 1, humanExpire));
+            _.forEach(config.REQUEST_LIMIT, (val, key) => {
+                operations.push(cache.delAsync(`${config.app}:${limitKey}:${key}:max:${remoteIp}`));
+            });
         } else {
-            operations.push(cache.setAsync(`${config.app}:limiter:ishuman:${remoteIp}`, 1, humanExpire));
-        }
+            operations.push(cache.delAsync(`${config.app}:limiter:${remoteIp}`));
 
-        _.forEach(config.REQUEST_LIMIT, (val, key) => {
-            operations.push(cache.delAsync(`${config.app}:limiter:${key}:max:${remoteIp}`));
-        });
+            // 验证码之后一小时之内不再限制qps
+            if (apiLimitValidate) {
+                operations.push(cache.setAsync(`${config.app}:limiter:api:ishuman:${remoteIp}`, 1, humanExpire));
+            } else {
+                operations.push(cache.setAsync(`${config.app}:limiter:ishuman:${remoteIp}`, 1, humanExpire));
+            }
+
+            _.forEach(config.REQUEST_LIMIT, (val, key) => {
+                operations.push(cache.delAsync(`${config.app}:limiter:${key}:max:${remoteIp}`));
+            });
+        }
 
         return Promise.all(operations);
     }

@@ -79,7 +79,8 @@ module.exports = {
     report: {
         host: 'localhost',
         port: 6009,
-        db: 'web-apm'
+        db: 'web-apm',
+        immediate: true
     },
     useOneapm: false,
     useCache: true,
@@ -248,9 +249,10 @@ if (isProduction) {
             }
         },
         report: {
-            host: 'badjs.yoho.cn',
+            host: 'badjs.yohoops.org',
             port: 80,
-            db: 'web-apm'
+            db: 'web-apm',
+            immediate: true
         },
         monitorReport: {
             host: '10.66.4.25',

@@ -133,7 +133,11 @@ exports.serverError = () => {
                 }
 
                 if (!isHuman) {
-                    cache.setAsync(`${config.app}:limiter:${remoteIp}`, 1, limiterIpTime);
+                    if (_.get(req.app.locals, 'pc.open.apmrisk', false)) {
+                        cache.setAsync(`${config.app}:limit2:${remoteIp}`, 1, limiterIpTime);
+                    } else {
+                        cache.setAsync(`${config.app}:limiter:${remoteIp}`, 1, limiterIpTime);
+                    }
 
                     req.session[sessionLimitKey] = true;
 

+
+
+module.exports = (predict, ifTrueFn, elseFn) => {
+    if (!ifTrueFn) {
+        ifTrueFn = (req, res, next) => next();
+    }
+
+    if (!elseFn) {
+        elseFn = (req, res, next) => next();
+    }
+
+    return (req, res, next) => {
+        if (predict(req, res)) {
+            ifTrueFn(req, res, next);
+        } else {
+            elseFn(req, res, next);
+        }
+    };
+};

+'use strict';
+
+const _ = require('lodash');
+const logger = global.yoho.logger;
+const ip = require('./rules/ip-list2');
+const userAgent = require('./rules/useragent2');
+const pathWhiteList = require('./rules/path-white-list');
+
+// const asynchronous = require('./rules/asynchronous');
+// const fakerLimiter = require('./rules/faker-limit');
+const captchaPolicy = require('./policies/captcha');
+
+// const reporterPolicy = require('./policies/reporter');
+
+const _excluded = (req) => {
+    return Boolean(
+        _.includes(pathWhiteList(), req.path)
+    );
+};
+
+const limiter = (rule, policy, context) => {
+    return rule(context, policy);
+};
+
+module.exports = (req, res, next) => {
+    const remoteIp = req.yoho.clientIp || '';
+    const enabled = !_.get(req.app.locals, 'pc.sys.noLimiter');
+    let excluded = _excluded(req);
+
+    // 开关为关或者未获取到remoteIp，放行
+    if (!enabled || !remoteIp || excluded) {
+        logger.debug(`request remote ip: ${remoteIp}; enabled: ${enabled}`);
+        return next();
+    }
+
+    (async function() {
+        const context = {
+            req: req,
+            res: res,
+            next: next,
+            remoteIp: remoteIp
+        };
+
+        let results = await Promise.all([
+            limiter(userAgent, captchaPolicy, context),
+            limiter(ip, captchaPolicy, context),
+
+            // limiter(asynchronous, captchaPolicy, context)
+            // limiter(fakerLimiter, reporterPolicy, context)
+        ]);
+
+        let allPass = true, exclusion = false, policy = null;
+
+        logger.debug('limiter result: ' + JSON.stringify(results));
+
+        _.forEach(results, (result) => {
+            if (typeof result === 'object' && !exclusion) {
+                exclusion = result.exclusion;
+            }
+
+            if (typeof result === 'function') {
+                allPass = false;
+                policy = result;
+            }
+        });
+
+        if (exclusion) {
+            return next();
+        } else if (!allPass && policy) {
+            policy(req, res, next);
+        } else {
+            return next();
+        }
+    }()).catch((err) => {
+        logger.error(err);
+        return next();
+    });
+};

+'use strict';
+
+const cache = global.yoho.cache.master;
+const config = global.yoho.config;
+const logger = global.yoho.logger;
+
+const limitKey = 'limit2';
+
+module.exports = (limiter, policy) => {
+    const ipBlackKey = `pc:limiter:${limiter.remoteIp}`; // ci ip黑名单
+    const ipLimitKey = `${config.app}:${limitKey}:${limiter.remoteIp}`; // 业务黑名单
+
+    return Promise.all([
+        cache.getAsync(ipBlackKey),
+        cache.getAsync(ipLimitKey)
+    ]).then(result => {
+        let ipBlackRes = result[0];
+        let ipLimitRes = result[1];
+
+        logger.debug(ipBlackKey, ipBlackRes);
+        logger.debug(ipLimitKey, ipLimitRes);
+
+        if ((ipBlackRes && +ipBlackRes > 0) || (ipLimitRes && +ipLimitRes > 0)) {
+            return Promise.resolve(policy);
+        } else {
+            return Promise.resolve(true);
+        }
+    });
+};

@@ -12,7 +12,8 @@ const DEFAULT_PATH_WHITE_LIST = [
     '/common/getbanner',
     '/common/suggestfeedback',
     '/product/search/history',
-    '/product/search/suggest'
+    '/product/search/suggest',
+    '/3party/captcha.png'
 ];
 
 const cacheWhiteList = {

+'use strict';
+
+const cache = global.yoho.cache.master;
+const _ = require('lodash');
+const logger = global.yoho.logger;
+
+
+module.exports = (limiter, policy) => {
+    const blackKey = 'pc:limiter:ua:black';
+
+    const ua = limiter.req.header('User-Agent');
+
+    cache.getAsync(blackKey).then((args) => {
+        let blacklist = [];
+
+        try {
+            blacklist = JSON.parse(args);
+        } catch (error) {
+            logger.error(error);
+        }
+
+        blacklist = blacklist || [];
+
+        if (blacklist.length === 0) {
+            return Promise.resolve(true);
+        }
+
+        const test = (list) => {
+            let result = false;
+
+            _.each(list, (item) => {
+                let regexp;
+
+                try {
+                    regexp = new RegExp(item);
+                } catch (e) {
+                    logger.error(e);
+                }
+
+                if (regexp.test(ua)) {
+                    result = true;
+                }
+            });
+
+            return result;
+        };
+
+        if (test(blacklist)) {
+            return Promise.resolve(policy);
+        } else {
+            return Promise.resolve(true);
+        }
+    });
+
+};

+/**
+ * 控制路由请求次数
+ * @date: 2018/03/05
+ */
+'use strict';
+
+const _ = require('lodash');
+const cache = global.yoho.cache.master;
+const helpers = global.yoho.helpers;
+const pathToRegexp = require('path-to-regexp');
+const logger = global.yoho.logger;
+const md5 = require('md5');
+
+const statusCode = {
+    code: 4403,
+    data: {},
+    message: '亲，您的访问次数过多，请稍后再试哦...'
+};
+
+const limitKey = 'limit2';
+
+const _jumpUrl = (req, res, next, result) => {
+    if (result.code === 4403) {
+        if (req.xhr) {
+            res.set({
+                'Cache-Control': 'no-cache',
+                Pragma: 'no-cache',
+                Expires: (new Date(1900, 0, 1, 0, 0, 0, 0)).toUTCString()
+            });
+            return res.status(403).json(result);
+        }
+        return res.redirect(`${result.data.url}&refer=${req.originalUrl}`);
+    }
+
+    return next();
+};
+
+module.exports = () => {
+    return (req, res, next) => {
+        // default open
+        if (_.get(req.app.locals.pc, 'close.risk', false)) {
+            return next();
+        }
+
+        let ip = _.get(req.yoho, 'clientIp', '');
+        let path = req.path || '';
+        let risks = _.get(req.app.locals.pc, 'json.risk', []);
+        let router = {};
+
+        logger.debug(`risk => risks: ${JSON.stringify(risks)}, path: ${path}, ip: ${ip}`); // eslint-disable-line
+        if (_.isEmpty(path) || _.isEmpty(risks)) {
+            return next();
+        }
+
+        _.isArray(risks) && risks.some(item => {
+            if (item.state === 'off') {
+                return false;
+            }
+
+            if (!item.regRoute) {
+                item.regRoute = pathToRegexp(item.route);
+            }
+
+            if (item.regRoute.test(path)) {
+                router = item;
+                return true;
+            }
+
+            return false;
+        });
+
+        logger.debug(`risk => router: ${JSON.stringify(router)}, path: ${path}`); // eslint-disable-line
+        if (_.isEmpty(router)) {
+            return next();
+        }
+
+        let keyPath = md5(`${router.regRoute}`);
+        let limitEnable = `pc:risk:${limitKey}:${keyPath}:${ip}`;
+        let checkUrl = helpers.urlFormat('/3party/check', {
+            pid: `pc:risk:${limitKey}:${keyPath}`
+        });
+
+        cache.getAsync(limitEnable)
+            .then(result => {
+                if (result) {
+                    logger.info(`risk => getCache: ${JSON.stringify(result)}, path: ${path}`); // eslint-disable-line
+                    return Object.assign({}, statusCode, {
+                        data: {
+                            url: checkUrl
+                        }
+                    });
+                } else {
+                    return {code: 200};
+                }
+            }).then(result => {
+                logger.debug(`risk => result: ${JSON.stringify(result)}, path: ${path}`); // eslint-disable-line
+                return _jumpUrl(req, res, next, result);
+            }).catch(e => {
+                console.log(`risk => path: ${path}, err: ${e.message}`);
+                return next();
+            });
+    };
+};

 {
   "name": "yohobuy-node",
-  "version": "6.6.15",
+  "version": "6.6.17",
   "private": true,
   "description": "A New Yohobuy Project With Express",
   "repository": {
@@ -57,7 +57,7 @@
     "urlencode": "^1.1.0",
     "uuid": "^2.0.2",
     "yoho-express-session": "^2.0.0",
-    "yoho-node-lib": "=0.6.20",
+    "yoho-node-lib": "=0.6.32",
     "yoho-zookeeper": "^1.0.10",
     "yo-cli": "=1.0.7"
   },

@@ -1023,7 +1023,7 @@ $captchaSmsBtn.on('click', function() {
                 }
 
                 return $.Deferred().reject().promise(); // eslint-disable-line
-            }).then(disable60sSendSmsBtn);
+            }).then(disable60sSendSmsBtn).fail(disable60sSendSmsBtn);
         });
 });
 

@@ -2124,17 +2124,14 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
   version "1.0.2"
   resolved "http://npm.yohops.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
 
-cosmiconfig@^2.1.0, cosmiconfig@^2.1.1:
-  version "2.2.2"
-  resolved "http://npm.yohops.com/cosmiconfig/-/cosmiconfig-2.2.2.tgz#6173cebd56fac042c1f4390edf7af6c07c7cb892"
+cosmiconfig@^4.0.0:
+  version "4.0.0"
+  resolved "http://npm.yohops.com/cosmiconfig/-/cosmiconfig-4.0.0.tgz#760391549580bbd2df1e562bc177b13c290972dc"
   dependencies:
     is-directory "^0.3.1"
-    js-yaml "^3.4.3"
-    minimist "^1.2.0"
-    object-assign "^4.1.0"
-    os-homedir "^1.0.1"
-    parse-json "^2.2.0"
-    require-from-string "^1.1.0"
+    js-yaml "^3.9.0"
+    parse-json "^4.0.0"
+    require-from-string "^2.0.1"
 
 cosmiconfig@^5.0.0:
   version "5.0.5"
@@ -4328,6 +4325,18 @@ ignore@^3.3.3, ignore@^3.3.5:
   version "3.3.8"
   resolved "http://npm.yohops.com/ignore/-/ignore-3.3.8.tgz#3f8e9c35d38708a3a7e0e9abb6c73e7ee7707b2b"
 
+import-cwd@^2.0.0:
+  version "2.1.0"
+  resolved "http://npm.yohops.com/import-cwd/-/import-cwd-2.1.0.tgz#aa6cf36e722761285cb371ec6519f53e2435b0a9"
+  dependencies:
+    import-from "^2.1.0"
+
+import-from@^2.1.0:
+  version "2.1.0"
+  resolved "http://npm.yohops.com/import-from/-/import-from-2.1.0.tgz#335db7f2a7affd53aaa471d4b8021dee36b7f3b1"
+  dependencies:
+    resolve-from "^3.0.0"
+
 import-lazy@^2.1.0:
   version "2.1.0"
   resolved "http://npm.yohops.com/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43"
@@ -4372,9 +4381,9 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-influx-batch-sender@^0.1.5:
-  version "0.1.5"
-  resolved "http://npm.yohops.com/influx-batch-sender/-/influx-batch-sender-0.1.5.tgz#865a5eb9a446f4e8ed442ab44e67c3532c27a5d7"
+influx-batch-sender@^0.1.6:
+  version "0.1.6"
+  resolved "http://npm.yohops.com/influx-batch-sender/-/influx-batch-sender-0.1.6.tgz#fe4fb11ac80ece3ca791e40c8de098ce0181d499"
   dependencies:
     debug "^3.0.1"
     lodash "^4.17.4"
@@ -4855,7 +4864,7 @@ js-tokens@^3.0.0, js-tokens@^3.0.2:
   version "3.0.2"
   resolved "http://npm.yohops.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b"
 
-js-yaml@^3.11.0, js-yaml@^3.4.3, js-yaml@^3.9.0, js-yaml@^3.9.1:
+js-yaml@^3.11.0, js-yaml@^3.9.0, js-yaml@^3.9.1:
   version "3.12.0"
   resolved "http://npm.yohops.com/js-yaml/-/js-yaml-3.12.0.tgz#eaed656ec8344f10f527c6bfa1b6e2244de167d1"
   dependencies:
@@ -6275,7 +6284,7 @@ os-browserify@^0.3.0:
   version "0.3.0"
   resolved "http://npm.yohops.com/os-browserify/-/os-browserify-0.3.0.tgz#854373c7f5c2315914fc9bfc6bd8238fdda1ec27"
 
-os-homedir@^1.0.0, os-homedir@^1.0.1:
+os-homedir@^1.0.0:
   version "1.0.2"
   resolved "http://npm.yohops.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3"
 
@@ -6941,36 +6950,20 @@ postcss-less@^2.0.0:
   dependencies:
     postcss "^5.2.16"
 
-postcss-load-config@^1.2.0:
-  version "1.2.0"
-  resolved "http://npm.yohops.com/postcss-load-config/-/postcss-load-config-1.2.0.tgz#539e9afc9ddc8620121ebf9d8c3673e0ce50d28a"
-  dependencies:
-    cosmiconfig "^2.1.0"
-    object-assign "^4.1.0"
-    postcss-load-options "^1.2.0"
-    postcss-load-plugins "^2.3.0"
-
-postcss-load-options@^1.2.0:
-  version "1.2.0"
-  resolved "http://npm.yohops.com/postcss-load-options/-/postcss-load-options-1.2.0.tgz#b098b1559ddac2df04bc0bb375f99a5cfe2b6d8c"
-  dependencies:
-    cosmiconfig "^2.1.0"
-    object-assign "^4.1.0"
-
-postcss-load-plugins@^2.3.0:
-  version "2.3.0"
-  resolved "http://npm.yohops.com/postcss-load-plugins/-/postcss-load-plugins-2.3.0.tgz#745768116599aca2f009fad426b00175049d8d92"
+postcss-load-config@^2.0.0:
+  version "2.0.0"
+  resolved "http://npm.yohops.com/postcss-load-config/-/postcss-load-config-2.0.0.tgz#f1312ddbf5912cd747177083c5ef7a19d62ee484"
   dependencies:
-    cosmiconfig "^2.1.1"
-    object-assign "^4.1.0"
+    cosmiconfig "^4.0.0"
+    import-cwd "^2.0.0"
 
-postcss-loader@^2.1.4:
-  version "2.1.5"
-  resolved "http://npm.yohops.com/postcss-loader/-/postcss-loader-2.1.5.tgz#3c6336ee641c8f95138172533ae461a83595e788"
+postcss-loader@^2.1.6:
+  version "2.1.6"
+  resolved "http://npm.yohops.com/postcss-loader/-/postcss-loader-2.1.6.tgz#1d7dd7b17c6ba234b9bed5af13e0bea40a42d740"
   dependencies:
     loader-utils "^1.1.0"
     postcss "^6.0.0"
-    postcss-load-config "^1.2.0"
+    postcss-load-config "^2.0.0"
     schema-utils "^0.4.0"
 
 postcss-markdown@^0.28.0:
@@ -8068,9 +8061,9 @@ require-directory@^2.1.1:
   version "2.1.1"
   resolved "http://npm.yohops.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
 
-require-from-string@^1.1.0:
-  version "1.2.1"
-  resolved "http://npm.yohops.com/require-from-string/-/require-from-string-1.2.1.tgz#529c9ccef27380adfec9a2f965b649bbee636418"
+require-from-string@^2.0.1:
+  version "2.0.2"
+  resolved "http://npm.yohops.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909"
 
 require-main-filename@^1.0.1:
   version "1.0.1"
@@ -10308,13 +10301,13 @@ yoho-jquery@^1.12.4:
   version "1.12.4"
   resolved "http://npm.yohops.com/yoho-jquery/-/yoho-jquery-1.12.4.tgz#22499b325f293ee8b1d60559777348156494926d"
 
-yoho-node-lib@=0.6.20:
-  version "0.6.20"
-  resolved "http://npm.yohops.com/yoho-node-lib/-/yoho-node-lib-0.6.20.tgz#682dbad0c5d0cbe79987dfddb872bc16c6cfb6a3"
+yoho-node-lib@=0.6.30:
+  version "0.6.30"
+  resolved "http://npm.yohops.com/yoho-node-lib/-/yoho-node-lib-0.6.30.tgz#d22c63ca3da65c35161395c30054feab09483253"
   dependencies:
     dnscache "^1.0.1"
     handlebars "^4.0.5"
-    influx-batch-sender "^0.1.5"
+    influx-batch-sender "^0.1.6"
     lodash "^4.13.1"
     lru-cache "^4.1.1"
     md5 "^2.1.0"