merge master

@@ -76,9 +76,9 @@ module.exports = {
     zookeeperServer: '127.0.0.1:2181',
     redis: {
         connect: {
-            // host: '127.0.0.1',
+            host: '127.0.0.1',
 
-            host: '192.168.102.49',
+            // host: '192.168.102.49',
             port: '6379',
             enable_offline_queue: false,
             retry_strategy(options) {

@@ -15,4 +15,4 @@ function init() {
     });
 }
 
-module.exports = init();
\ No newline at end of file
+module.exports = init();

@@ -28,7 +28,7 @@ class MySqlSender {
         const bulk = this.batchMessages.splice(0, len);
 
         for (let i of bulk) {
-            logger.info('[db] insert db [%s]', JSON.stringify(i));
+            logger.debug('[db] insert db [%s]', JSON.stringify(i));
             await client(this.table).insert(i).catch((err) => {
                 logger.error('[db] insert data=[%s] error=[%s]', JSON.stringify(i), err);
             });

@@ -206,4 +206,4 @@ module.exports = {
     handleClientFirst,
     handleClientTiming,
     handleClientResource
-};
\ No newline at end of file
+};

@@ -110,7 +110,7 @@ module.exports = (req, res, next) => {
 
         res.send();
 
-        logger.info('[client] handle OK [%s]', req.query.l);
+        logger.debug('[client] handle OK [%s]', req.query.l);
     } catch (e) {
         logger.error('[client] handle ERROR [%s]', e);
         return next(e);

+
+const zk = require('./zk');
+const _ = require('lodash');
+const logger = global.yoho.logger;
+
+module.exports = async({user}, next) => {
+    const disable = _.get(zk, `${user.app}.sys.noLimiter`, false);
+
+    logger.debug('disable==>', disable);
+
+    if (disable) {
+        return;
+    }
+
+    return next();
+};

+
+const zk = require('./zk');
+const _ = require('lodash');
+const logger = global.yoho.logger;
+
+module.exports = async({user}, next) => {
+    const enable = _.get(zk, `${user.app}.open.apmrisk`, false);
+
+    logger.debug('enable all ==>', enable);
+
+    if (!enable) {
+        return;
+    }
+
+    return next();
+};

 const compose = require('koa-compose');
 const _ = require('lodash');
+
+const enable = require('./enable');
 const qpsPath = require('./qps-path');
+
+const disableBelow = require('./disable');
+const ipFilter = require('./ip');
+const userFilter = require('./user');
+const userAgentFilter = require('./useragent');
+const xhrFilter = require('./xhr');
+const whitelistIpFilter = require('./whitelist-ip');
+const whitelistPathFilter = require('./whitelist-path');
 const qps = require('./qps');
 
 const APP_NAME = {
@@ -9,17 +19,43 @@ const APP_NAME = {
     UNKNOWN: ''
 };
 
+const APP_TYPE = {
+    'yohobuy-node': 'web',
+    'yohobuywap-node': 'h5',
+    UNKNOWN: ''
+};
+
 module.exports = () => {
-    const handlers = compose([qpsPath, qps]);
+    const handlers = compose([
+        enable,
+        qpsPath,
+
+        disableBelow,
+        userFilter,
+        xhrFilter,
+        whitelistIpFilter,
+        whitelistPathFilter,
+        ipFilter,
+        userAgentFilter,
+        qps
+    ]);
 
     return async(m) => {
         const user = {
-            uid: _.get(m, 'fields.uid', ''),
-            ip: _.get(m, 'fields.ip', ''),
+            uid: _.parseInt(_.get(m, 'fields.uid', '0').replace(/\"/g, ''), 10),
+            ip: _.get(m, 'fields.ip', '').replace(/\"/g, ''),
             app: APP_NAME[_.get(m, 'tags.app', 'UNKNOWN')],
-            path: _.get(m, 'fields.path', '')
+            appName: _.get(m, 'tags.app', ''),
+            appType: APP_TYPE[_.get(m, 'tags.app', 'UNKNOWN')],
+            path: decodeURIComponent(_.get(m, 'fields.path', '').replace(/\"/g, '')),
+            userAgent: decodeURIComponent(_.get(m, 'fields.userAgent', '').replace(/\"/g, '')),
+            ajax: _.parseInt(_.get(m, 'fields.ajax', 0))
         };
 
+        if (!user.ip || !user.app) {
+            return;
+        }
+
         await handlers({user});
     };
 };

+const cache = global.yoho.cache.master;
+const logger = global.yoho.logger;
+
+const {limitKey} = require('./vars');
+
+module.exports = async({user}, next) => {
+    const ipBlackKey = `pc:limiter:${user.ip}`; // ci ip黑名单
+    const ipLimitKey = `${user.appType}:${limitKey}:${user.ip}`; // 业务黑名单
+
+    const result = await Promise.all([
+        cache.getAsync(ipBlackKey),
+        cache.getAsync(ipLimitKey)
+    ]);
+
+    let ipBlackRes = result[0];
+    let ipLimitRes = result[1];
+
+    logger.debug('ip==>', ipBlackKey, ipBlackRes);
+    logger.debug('ip==>', ipLimitKey, ipLimitRes);
+
+    if ((ipBlackRes && +ipBlackRes > 0) || (ipLimitRes && +ipLimitRes > 0)) {
+        return;
+    } else {
+        return next();
+    }
+};

@@ -6,18 +6,13 @@
 
 const _ = require('lodash');
 const cache = global.yoho.cache.master;
-const config = global.yoho.config;
 const logger = global.yoho.logger;
 const md5 = require('md5');
 const pathToRegexp = require('path-to-regexp');
 const Promise = require('bluebird');
 
-const zk = {};
-
-if (config.zookeeperServer) {
-    require('yoho-zookeeper')(config.zookeeperServer, 'pc', zk.pc = {}, global.yoho.cache);
-    require('yoho-zookeeper')(config.zookeeperServer, 'wap', zk.wap = {}, global.yoho.cache);
-}
+const zk = require('./zk');
+const {limitKey} = require('./vars');
 
 const INVALIDTIME = 3600 * 24; // 24h
 const IP_WHITE_LIST = [
@@ -29,8 +24,6 @@ const IP_WHITE_LIST = [
     '218.94.77.166'
 ];
 
-const key = 'risk1';
-
 module.exports = async({user}, next) => {
     if (!user.app || !user.path || !user.ip) {
         return next();
@@ -44,10 +37,10 @@ module.exports = async({user}, next) => {
 
     const ip = user.ip;
     const path = user.path;
+    // const risks = _.get(zk, `${app}.json.risk`, [{route: '/product/(.*).html', interval: 5000, requests: 10}]);
     const risks = _.get(zk, `${app}.json.risk`, []);
     let router = {};
 
-    logger.debug(`risk => risks: ${JSON.stringify(risks)}, path: ${path}, ip: ${ip}`); // eslint-disable-line
     if (_.isEmpty(path) || _.isEmpty(risks) || IP_WHITE_LIST.indexOf(ip) > -1) {
         return next();
     }
@@ -71,47 +64,43 @@ module.exports = async({user}, next) => {
         return false;
     });
 
-    logger.debug(`risk => router: ${JSON.stringify(router)}, path: ${path}`); // eslint-disable-line
+    logger.debug(`risk==> router: ${JSON.stringify(router)}, path: ${path}`); // eslint-disable-line
+
     if (_.isEmpty(router)) {
         return next();
     }
 
     let keyPath = md5(`${router.regRoute}`);
-    let limitKey = `${app}:${key}:limit:${keyPath}:${ip}`; // 查询这个key是否生效
-    let configKey = `${app}:${key}:${keyPath}:${ip}`;
+    let limitEnable = `${app}:risk:${limitKey}:${keyPath}:${ip}`; // 查询这个key是否生效
+    let configKey = `${app}:risk:count:${keyPath}:${ip}`;
 
-    await Promise.all([
-        cache.getAsync(limitKey),
+    const inters = await Promise.all([
+        cache.getAsync(limitEnable),
         cache.getAsync(configKey),
-    ]).then(inters => {
-        logger.debug(`risk => getCache: ${JSON.stringify(inters)}, path: ${path}`); // eslint-disable-line
-        if (inters[0]) {
-            logger.info('[qps:route] this user[%o] has rejected', user);
-            return;
-        }
+    ]);
 
-        if (typeof inters[1] === 'undefined') {
-            cache.setAsync(configKey, 1, router.interval || 300);
-            return;
-        }
+    logger.debug(`risk==> cache: %s %d`, limitEnable, inters[0], configKey, inters[1]); // eslint-disable-line
 
-        inters[1] = parseInt(`0${inters[1]}`, 10);
-        if (inters[1] <= router.requests) {
-            router = [];
-            cache.incrAsync(configKey, 1);
-            return;
-        }
+    if (inters[0]) {
+        logger.info('[qps:route] this user[%j] has rejected', user);
+        return;
+    }
 
-        logger.warn('[qps:route] this user[%o] is being marked as rejected', user);
-        return Promise.all([
-            cache.setAsync(limitKey, 1, INVALIDTIME),
-            cache.delAsync(configKey)
-        ]);
-    }).then(result => {
-        logger.debug('[qps:route] user[%o] result[%o]', user, result); // eslint-disable-line
-    }).catch(e => {
-        logger.error(`risk => path: ${path}, err: ${e.message}`);
-    }).finally(() => {
+    if (typeof inters[1] === 'undefined') {
+        cache.setAsync(configKey, 1, router.interval || 300);
         return next();
-    });
+    }
+
+    inters[1] = parseInt(`0${inters[1]}`, 10);
+    if (inters[1] <= router.requests) {
+        router = [];
+        cache.incrAsync(configKey, 1);
+        return next();
+    }
+
+    logger.info('[qps:route] this user[%j] is being marked as rejected', user);
+    await Promise.all([
+        cache.setAsync(limitEnable, 1, INVALIDTIME),
+        cache.delAsync(configKey)
+    ]);
 };

@@ -5,6 +5,7 @@ const cache = global.yoho.cache.master;
 const config = global.yoho.config;
 const Promise = require('bluebird');
 const _ = require('lodash');
+const {limitKey} = require('./vars');
 
 // 超出访问限制ip限制访问1小时
 const limiterIpTime = 3600;
@@ -12,51 +13,46 @@ const limiterIpTime = 3600;
 // 页面访问限制
 const MAX_TIMES = config.REQUEST_LIMIT;
 
-const limiterKey = 'limiter2';
-
 module.exports = async({user}, next) => {
-    if (!user.app || !user.ip) {
-        return next();
-    }
+    const appType = user.appType;
 
     // 存储规则的cache keys
     let ruleKeys = {};
     let getOp = {};
 
     _.forEach(MAX_TIMES, (val, key) => {
-        ruleKeys[key] = `${user.app}:${limiterKey}:${key}:max:${user.ip}`;
+        ruleKeys[key] = `${appType}:${limitKey}:${key}:max:${user.ip}`;
         getOp[key] = cache.getAsync(ruleKeys[key]);
     });
 
-    getOp.human = cache.getAsync(`${user.app}:${limiterKey}:ishuman:${user.ip}`);
+    getOp.human = cache.getAsync(`${appType}:${limitKey}:ishuman:${user.ip}`);
 
     return Promise.props(getOp).then((results) => {
         if (results.human) { // 经过验证码之后1小时有效期内不再验证qps
-            logger.warn('[qps] this user[%o] is being marked as human', user);
+            logger.info('[qps] this user[%j] is being marked as human', user);
             return {};
         }
 
         // 遍历限制规则，若满足返回相应处理策略， 否则页面访问次数加1
-        let operation = [];
+        let operation = {};
 
         _.forEach(MAX_TIMES, (val, key) => {
             let cacheKey = ruleKeys[key];
 
             if (!results[key]) {
-                operation.push(cache.setAsync(cacheKey, 1, +key));
+                operation[cacheKey] = cache.setAsync(cacheKey, 1, +key);
             } else if (+results[key] > +val) {
-                logger.warn('[qps] this user[%o] is being marked as rejected', user);
+                logger.info('[qps] this user[%j] is being marked as rejected', user);
 
-                // ip限制1小时
-                operation.push(cache.setAsync(`${user.app}:${limiterKey}:${user.ip}`, 1, limiterIpTime));
+                operation[`${appType}:${limitKey}:${user.ip}`] = cache.setAsync(`${appType}:${limitKey}:${user.ip}`, 1, limiterIpTime);
             } else {
-                operation.push(cache.incrAsync(cacheKey, 1));
+                operation[cacheKey] = cache.incrAsync(cacheKey, 1);
             }
         });
 
-        return Promise.all(operation);
+        return Promise.props(operation);
     }).then((result) => {
-        logger.debug('[qps] user[%o] result[%o]', user, result); // eslint-disable-line
+        logger.debug('[qps] user[%j] result[%j]', user, result); // eslint-disable-line
     }).catch(err=>{
         logger.error(err);
     }).finally(() => {

+const logger = global.yoho.logger;
+
+module.exports = ({user}, next) => {
+    logger.debug('user==>', user.uid);
+
+    if (user.uid) {
+        return;
+    }
+
+    return next();
+};

+'use strict';
+
+const cache = global.yoho.cache.master;
+const _ = require('lodash');
+const logger = global.yoho.logger;
+const Promise = require('bluebird');
+
+module.exports = async({user}, next) => {
+    const blackKey = `${user.app}:limiter:ua:black`;
+    const whiteKey = `${user.app}:limiter:ua:white`;
+
+    const ua = user.userAgent;
+
+    logger.debug('userAgent==>%s', user.userAgent);
+
+    Promise.all([
+        cache.getAsync(blackKey),
+        cache.getAsync(whiteKey)
+    ]).then((args) => {
+        let blacklist = [];
+        let whitelist = [];
+
+        try {
+            blacklist = JSON.parse(args[0]);
+        } catch (error) {
+            logger.error(error);
+        }
+
+        try {
+            whitelist = JSON.parse(args[1]);
+        } catch (error) {
+            logger.error(error);
+        }
+
+        blacklist = blacklist || [];
+        whitelist = whitelist || [];
+
+        if (blacklist.length === 0 && whitelist.length === 0) {
+            return next();
+        }
+
+        const test = (list) => {
+            let result = false;
+
+            _.each(list, (item) => {
+                let regexp;
+
+                try {
+                    regexp = new RegExp(item);
+                } catch (e) {
+                    logger.error(e);
+                }
+
+                if (regexp.test(ua)) {
+                    result = true;
+                }
+            });
+
+            return result;
+        };
+
+        if (test(blacklist) || test(whitelist)) {
+            return;
+        }
+        return next();
+    }).catch(() => {
+        return next();
+    });
+};

+module.exports = {
+    limitKey: 'limit2'
+};

+const _ = require('lodash');
+const logger = global.yoho.logger;
+const cache = global.yoho.cache.master;
+const WHITE_LIST_KEY = 'whitelist:ip:';
+
+const IP_WHITE_LIST = [
+    '106.38.38.146', // 北京办公区域
+    '106.38.38.147', // 北京办公区域
+    '106.39.86.227', // 北京办公区域
+    '218.94.75.58', // 南京办公区域
+    '218.94.75.50', // 南京办公区域
+    '218.94.77.166', // 南京办公区域
+
+    // '222.73.196.18', // B站合作方单击次数快加白名单
+    '123.206.73.107', // 腾讯云出口IP
+    '139.199.35.21', // 腾讯云出口IP
+    '139.199.29.44', // 腾讯云出口IP
+    '123.206.21.19' // 腾讯云出口IP
+];
+
+const IP_WHITE_SEGMENT = [
+    '10.66.', // 内网IP段
+    '192.168.' // 内网IP段
+];
+
+module.exports = async({user}, next) => {
+    const remoteIp = user.ip;
+    let key = `${WHITE_LIST_KEY}${remoteIp}`;
+    let remoteIpSegment = `${remoteIp.split('.').slice(0, 2).join('.')}.`;
+
+    logger.debug('whitelist-ip==>%s', user.path);
+    if (_.includes(IP_WHITE_LIST, remoteIp) || _.includes(IP_WHITE_SEGMENT, remoteIpSegment)) {
+        return;
+    }
+
+    let result = Boolean(await cache.getAsync(key));
+
+    if (result) {
+        return;
+    }
+
+    logger.debug(key, result);
+
+    return next();
+};

+const _ = require('lodash');
+const logger = global.yoho.logger;
+const cache = global.yoho.cache.master;
+const get_WHITE_LIST_KEY = (t) => `${t}:limiter:whitelist:path`;
+
+const DEFAULT_PATH_WHITE_LIST = {
+    pc: [
+        '/3party/check',
+        '/passport/images-risk.png',
+        '/passport/images.png',
+        '/passport/cert/headerTip',
+        '/common/getbanner',
+        '/common/suggestfeedback',
+        '/product/search/history',
+        '/product/search/suggest'
+    ],
+    wap: [
+        '/3party/check',
+        '/3party/check/submit',
+        '/passport/captcha/get',
+        '/passport/img-check.jpg',
+        '/passport/img-check-risk.jpg',
+        '/passport/geetest/register',
+        '/activity/individuation',
+        '/activity/individuation/coupon',
+        '/activity/share',
+        '/activity/wechat/share',
+        '/activity/wechat/1111',
+        '/api/switch',
+        '/passport/login/user',
+        '/sw.js',
+        '/manifest.json',
+        '/activity/sw.js',
+        '/activity/manifest.json',
+        '/hfxRaNY27L.txt',
+        '/activity/hfxRaNY27L.txt',
+        '/product/shop/hfxRaNY27L.txt',
+        '/product/hfxRaNY27L.txt',
+        '/.well-known/apple-app-site-association',
+        '/service/sitemap.xml',
+        '/node/status.html',
+        '/product/seckill',
+        '/product/seckill/list',
+        '/product/seckill/remind',
+        '/product/seckill/get-product-list'
+    ]
+};
+
+const cacheWhiteList = {
+    nowTime() {
+        return Date.parse(new Date()) / 1000;
+    },
+    async getValue(app) {
+        if (this.updateTime || this.nowTime() - this.updateTime > 60 * 10) {
+            await this.syncRemoteConfig(app);
+        }
+
+        return _.uniq(_.concat([], DEFAULT_PATH_WHITE_LIST[app], _.toArray(this.whiteList)));
+    },
+    async syncRemoteConfig(app) {
+        if (this.syncing) {
+            return;
+        }
+
+        this.syncing = true;
+
+        await cache.getAsync(get_WHITE_LIST_KEY(app)).then(res => {
+            this.updateTime = this.nowTime();
+
+            if (!res) {
+                return;
+            }
+
+            this.whiteList = JSON.parse(res);
+            this.syncing = false;
+        }).catch((e) => {
+            this.syncing = false;
+            logger.debug('whitelist path parse error. ' + JSON.stringfy(e));
+        });
+    }
+};
+
+module.exports = async({user}, next) => {
+    const paths = await cacheWhiteList.getValue(user.app);
+
+    logger.debug('whitelist-path==>', user.path);
+    if (paths.includes(user.path)) {
+        return;
+    }
+
+    return next();
+};

+const logger = global.yoho.logger;
+
+module.exports = ({user}, next) => {
+    logger.debug('ajax==>%s %d', user.path, user.ajax);
+
+    if (user.ajax) {
+        return;
+    }
+
+    return next();
+};

+const {config} = global.yoho;
+
+const zk = {};
+
+if (config.zookeeperServer) {
+    require('yoho-zookeeper')(config.zookeeperServer, 'pc', zk.pc = {}, global.yoho.cache);
+    require('yoho-zookeeper')(config.zookeeperServer, 'wap', zk.wap = {}, global.yoho.cache);
+}
+
+module.exports = zk;

@@ -3,6 +3,7 @@ const MysqlSender = require('../lib/mysql-sender');
 const config = require('../common/config');
 const msg2row = require('./msg2row');
 
+
 const logger = global.yoho.logger;
 const errorSqlSender = new MysqlSender(config.table.error);
 const slowRouterSqlSender = new MysqlSender(config.table.slow);
@@ -15,7 +16,7 @@ const API_BLACK_LIST = [
     'app.shop.banner'
 ];
 
-function handleWebServerDuration(m) {
+async function handleWebServerDuration(m) {
     let duration = _.parseInt(m.fields.duration);
 
     if (duration > config.slowRoute.min / 10 && duration < config.slowRoute.max) {

@@ -9,9 +9,8 @@ const {
     handleWebServerDuration
 } = require('./serverapm-service');
 
-//const riskService = require('./risk-service');
-
-//const handleRisk = riskService();
+const riskService = require('./risk-service');
+const handleRisk = riskService();
 
 const server = {
     async handle(data) {
@@ -26,7 +25,10 @@ const server = {
                 switch (m.measurement) {
                     case 'web-server-duration': {
                         handleWebServerDuration(m);
-                        //await handleRisk(m);
+
+                        if (m.tags.type === 'route') {
+                            await handleRisk(m);
+                        }
                         break;
                     }
                     case 'error-report': {
@@ -43,8 +45,9 @@ const server = {
                     }
                 }
 
-                logger.debug('[server] handle OK [%s]', data);
             }, {concurrency: 2});
+
+            logger.debug('[server] handle OK [%j]', data);
         } catch (e) {
             logger.error('[server] handle ERROR [%s]', e);
         }