fix

htoooth
Commit b65a257f71b18faca80d959429e75a6d8ee6490a 1 parent 24f41efe
Showing 15 changed files with 50 additions and 32 deletions
lib/mysql-client.js
lib/mysql-sender.js
middleware/clientapm.js
middleware/risk-service/disable.js
middleware/risk-service/index.js
middleware/risk-service/ip.js
middleware/risk-service/qps-path.js
middleware/risk-service/qps.js
middleware/risk-service/user.js
middleware/risk-service/useragent.js
middleware/risk-service/whitelist-ip.js
middleware/risk-service/whitelist-path.js
middleware/risk-service/xhr.js
middleware/serverapm-service.js
middleware/serverapm.js
--- a/lib/mysql-client.js
View file @b65a257
+++ b/lib/mysql-client.js
View file @b65a257
@@ -15,4 +15,4 @@ function init() {
     });
 }
-module.exports = init();
+module.exports = init();
--- a/lib/mysql-sender.js
View file @b65a257
+++ b/lib/mysql-sender.js
View file @b65a257
@@ -21,14 +21,14 @@ class MySqlSender {
         const len = this.batchMessages.length;
         if (len < 1) {
-            logger.debug('[db] insert list is empty');
+            // logger.debug('[db] insert list is empty');
             return;
         }
         const bulk = this.batchMessages.splice(0, len);
         for (let i of bulk) {
-            logger.info('[db] insert db [%s]', JSON.stringify(i));
+            logger.debug('[db] insert db [%s]', JSON.stringify(i));
             await client(this.table).insert(i).catch((err) => {
                 logger.error('[db] insert data=[%s] error=[%s]', JSON.stringify(i), err);
             });
--- a/middleware/clientapm.js
View file @b65a257
+++ b/middleware/clientapm.js
View file @b65a257
@@ -110,7 +110,7 @@ module.exports = (req, res, next) => {
         res.send();
-        logger.info('[client] handle OK [%s]', req.query.l);
+        logger.debug('[client] handle OK [%s]', req.query.l);
     } catch (e) {
         logger.error('[client] handle ERROR [%s]', e);
         return next(e);
--- a/middleware/risk-service/disable.js
View file @b65a257
+++ b/middleware/risk-service/disable.js
View file @b65a257
 const zk = require('./zk');
 const _ = require('lodash');
+const logger = global.yoho.logger;
 module.exports = async({user}, next) => {
     const disable = _.get(zk, `${user.app}.sys.noLimiter`, false);
+    logger.debug('disable==>', disable);
+
     if (disable) {
         return;
     }
--- a/middleware/risk-service/index.js
View file @b65a257
+++ b/middleware/risk-service/index.js
View file @b65a257
@@ -20,29 +20,31 @@ const APP_NAME = {
 module.exports = () => {
     const handlers = compose([
-        // qpsPath,
+        qpsPath,
         disableBelow,
         xhrFilter,
-        // whitelistIpFilter,
-        // userFilter,
-        // whitelistPathFilter,
-        // userAgentFilter,
-        // ipFilter,
-        // qps
+        whitelistIpFilter,
+        userFilter,
+        whitelistPathFilter,
+        userAgentFilter,
+        ipFilter,
+        qps
     ]);
     return async(m) => {
         const user = {
-            uid: _.get(m, 'fields.uid', ''),
+            uid: _.parseInt(_.get(m, 'fields.uid', '0'), 10),
             ip: _.get(m, 'fields.ip', '').replace(/\"/g, ''),
             app: APP_NAME[_.get(m, 'tags.app', 'UNKNOWN')],
             path: decodeURIComponent(_.get(m, 'fields.path', '').replace(/\"/g, '')),
             userAgent: decodeURIComponent(_.get(m, 'fields.userAgent', '').replace(/\"/g, '')),
-            ajax: _.get(m, 'fields.ajax', 0)
+            ajax: _.parseInt(_.get(m, 'fields.ajax', 0))
         };
-        console.log(user);
+        if (!user.ip || !user.app) {
+            return;
+        }
         await handlers({user});
     };
--- a/middleware/risk-service/ip.js
View file @b65a257
+++ b/middleware/risk-service/ip.js
View file @b65a257
@@ -13,8 +13,8 @@ module.exports = async({user}, next) => {
     let ipBlackRes = result[0];
     let ipLimitRes = result[1];
-    logger.debug(ipBlackKey, ipBlackRes);
-    logger.debug(ipLimitKey, ipLimitRes);
+    logger.debug('ip==>', ipBlackKey, ipBlackRes);
+    logger.debug('ip==>', ipLimitKey, ipLimitRes);
     if ((ipBlackRes && +ipBlackRes > 0) || (ipLimitRes && +ipLimitRes > 0)) {
         return;
--- a/middleware/risk-service/qps-path.js
View file @b65a257
+++ b/middleware/risk-service/qps-path.js
View file @b65a257
@@ -38,10 +38,9 @@ module.exports = async({user}, next) => {
     const ip = user.ip;
     const path = user.path;
-    const risks = _.get(zk, `${app}.json.risk`, []);
+    const risks = _.get(zk, `${app}.json.risk`, [{route: '/product/(.*).html', interval: 5000, requests: 10}]);
     let router = {};
-    logger.debug(`risk => risks: ${JSON.stringify(risks)}, path: ${path}, ip: ${ip}`); // eslint-disable-line
     if (_.isEmpty(path) || _.isEmpty(risks) || IP_WHITE_LIST.indexOf(ip) > -1) {
         return next();
     }
@@ -65,7 +64,8 @@ module.exports = async({user}, next) => {
         return false;
     });
-    logger.debug(`risk => router: ${JSON.stringify(router)}, path: ${path}`); // eslint-disable-line
+    logger.debug(`risk==> router: ${JSON.stringify(router)}, path: ${path}`); // eslint-disable-line
+
     if (_.isEmpty(router)) {
         return next();
     }
@@ -79,9 +79,8 @@ module.exports = async({user}, next) => {
         cache.getAsync(configKey),
     ]);
-    logger.debug(`risk => getCache: ${JSON.stringify(inters)}, path: ${path}`); // eslint-disable-line
     if (inters[0]) {
-        logger.info('[qps:route] this user[%o] has rejected', user);
+        logger.info('[qps:route] this user[%j] has rejected', user);
         return;
     }
@@ -97,7 +96,7 @@ module.exports = async({user}, next) => {
         return next();
     }
-    logger.warn('[qps:route] this user[%o] is being marked as rejected', user);
+    logger.info('[qps:route] this user[%j] is being marked as rejected', user);
     await Promise.all([
         cache.setAsync(limitKey, 1, INVALIDTIME),
         cache.delAsync(configKey)
--- a/middleware/risk-service/qps.js
View file @b65a257
+++ b/middleware/risk-service/qps.js
View file @b65a257
@@ -32,7 +32,7 @@ module.exports = async({user}, next) => {
     return Promise.props(getOp).then((results) => {
         if (results.human) { // 经过验证码之后1小时有效期内不再验证qps
-            logger.warn('[qps] this user[%o] is being marked as human', user);
+            logger.info('[qps] this user[%o] is being marked as human', user);
             return {};
         }
@@ -45,7 +45,7 @@ module.exports = async({user}, next) => {
             if (!results[key]) {
                 operation[cacheKey] = cache.setAsync(cacheKey, 1, +key);
             } else if (+results[key] > +val) {
-                logger.warn('[qps] this user[%o] is being marked as rejected', user);
+                logger.info('[qps] this user[%j] is being marked as rejected', user);
                 operation[`${user.app}:${limiterKey}:${user.ip}`] = cache.setAsync(`${user.app}:${limiterKey}:${user.ip}`, 1, limiterIpTime);
             } else {
@@ -55,7 +55,7 @@ module.exports = async({user}, next) => {
         return Promise.props(operation);
     }).then((result) => {
-        logger.debug('[qps] user[%j] result[%j]', user, result); // eslint-disable-line
+        // logger.debug('[qps] user[%j] result[%j]', user, result); // eslint-disable-line
     }).catch(err=>{
         logger.error(err);
     }).finally(() => {
--- a/middleware/risk-service/user.js
View file @b65a257
+++ b/middleware/risk-service/user.js
View file @b65a257
+const logger = global.yoho.logger;
+
 module.exports = ({user}, next) => {
+    logger.debug('user==>', user.uid);
+
     if (user.uid) {
         return;
     }
--- a/middleware/risk-service/useragent.js
View file @b65a257
+++ b/middleware/risk-service/useragent.js
View file @b65a257
@@ -7,11 +7,13 @@ const Promise = require('bluebird');
 module.exports = async({user}, next) => {
-    const blackKey = `${user.app}:limiter:ua:black`,
-        whiteKey = `${user.app}:limiter:ua:white`;
+    const blackKey = `${user.app}:limiter:ua:black`;
+    const whiteKey = `${user.app}:limiter:ua:white`;
     const ua = user.userAgent;
+    logger.debug('userAgent==>%s', user.userAgent);
+
     Promise.all([
         cache.getAsync(blackKey),
         cache.getAsync(whiteKey)
--- a/middleware/risk-service/whitelist-ip.js
View file @b65a257
+++ b/middleware/risk-service/whitelist-ip.js
View file @b65a257
 const _ = require('lodash');
-const co = Promise.coroutine;
 const logger = global.yoho.logger;
 const cache = global.yoho.cache.master;
 const WHITE_LIST_KEY = 'whitelist:ip:';
@@ -29,6 +28,7 @@ module.exports = async({user}, next) => {
     let key = `${WHITE_LIST_KEY}${remoteIp}`;
     let remoteIpSegment = `${remoteIp.split('.').slice(0, 2).join('.')}.`;
+    logger.debug('whitelist-ip==>%s', user.path);
     if (_.includes(IP_WHITE_LIST, remoteIp) || _.includes(IP_WHITE_SEGMENT, remoteIpSegment)) {
         return;
     }
--- a/middleware/risk-service/whitelist-path.js
View file @b65a257
+++ b/middleware/risk-service/whitelist-path.js
View file @b65a257
@@ -51,6 +51,7 @@ const cacheWhiteList = {
 module.exports = async({user}, next) => {
     const paths = await cacheWhiteList.getValue();
+    logger.debug('whitelist-path==>', user.path);
     if (paths.includes(user.path)) {
         return;
     }
--- a/middleware/risk-service/xhr.js
View file @b65a257
+++ b/middleware/risk-service/xhr.js
View file @b65a257
+const logger = global.yoho.logger;
+
 module.exports = ({user}, next) => {
+    logger.debug('ajax==>%s %d', user.path, user.ajax);
+
     if (user.ajax) {
         return;
     }
--- a/middleware/serverapm-service.js
View file @b65a257
+++ b/middleware/serverapm-service.js
View file @b65a257
@@ -3,6 +3,7 @@ const MysqlSender = require('../lib/mysql-sender');
 const config = require('../common/config');
 const msg2row = require('./msg2row');
+
 const logger = global.yoho.logger;
 const errorSqlSender = new MysqlSender(config.table.error);
 const slowRouterSqlSender = new MysqlSender(config.table.slow);
@@ -15,7 +16,7 @@ const API_BLACK_LIST = [
     'app.shop.banner'
 ];
-function handleWebServerDuration(m) {
+async function handleWebServerDuration(m) {
     let duration = _.parseInt(m.fields.duration);
     if (duration > config.slowRoute.min / 10 && duration < config.slowRoute.max) {
--- a/middleware/serverapm.js
View file @b65a257
+++ b/middleware/serverapm.js
View file @b65a257
@@ -10,7 +10,6 @@ const {
 } = require('./serverapm-service');
 const riskService = require('./risk-service');
-
 const handleRisk = riskService();
 const server = {
@@ -25,8 +24,11 @@ const server = {
                 switch (m.measurement) {
                     case 'web-server-duration': {
-                        handleWebServerDuration(m);
-                        await handleRisk(m);
+                        // handleWebServerDuration(m);
+
+                        if (m.tags.type === 'route') {
+                            await handleRisk(m);
+                        }
                         break;
                     }
                     case 'error-report': {