fe / yoho-apm · Commits

Go to a project

Authored by htoooth
Commit b65a257f71b18faca80d959429e75a6d8ee6490a 1 parent 24f41efe

fix

Inline Side-by-side
Showing 15 changed files with 48 additions and 30 deletions
@@ -21,14 +21,14 @@ class MySqlSender { @@ -21,14 +21,14 @@ class MySqlSender {
21 const len = this.batchMessages.length; 21 const len = this.batchMessages.length;
22 22
23 if (len < 1) { 23 if (len < 1) {
24 - logger.debug('[db] insert list is empty'); 24 + // logger.debug('[db] insert list is empty');
25 return; 25 return;
26 } 26 }
27 27
28 const bulk = this.batchMessages.splice(0, len); 28 const bulk = this.batchMessages.splice(0, len);
29 29
30 for (let i of bulk) { 30 for (let i of bulk) {
31 - logger.info('[db] insert db [%s]', JSON.stringify(i)); 31 + logger.debug('[db] insert db [%s]', JSON.stringify(i));
32 await client(this.table).insert(i).catch((err) => { 32 await client(this.table).insert(i).catch((err) => {
33 logger.error('[db] insert data=[%s] error=[%s]', JSON.stringify(i), err); 33 logger.error('[db] insert data=[%s] error=[%s]', JSON.stringify(i), err);
34 }); 34 });
@@ -110,7 +110,7 @@ module.exports = (req, res, next) => { @@ -110,7 +110,7 @@ module.exports = (req, res, next) => {
110 110
111 res.send(); 111 res.send();
112 112
113 - logger.info('[client] handle OK [%s]', req.query.l); 113 + logger.debug('[client] handle OK [%s]', req.query.l);
114 } catch (e) { 114 } catch (e) {
115 logger.error('[client] handle ERROR [%s]', e); 115 logger.error('[client] handle ERROR [%s]', e);
116 return next(e); 116 return next(e);
1 1
2 const zk = require('./zk'); 2 const zk = require('./zk');
3 const _ = require('lodash'); 3 const _ = require('lodash');
  4 +const logger = global.yoho.logger;
4 5
5 module.exports = async({user}, next) => { 6 module.exports = async({user}, next) => {
6 const disable = _.get(zk, `${user.app}.sys.noLimiter`, false); 7 const disable = _.get(zk, `${user.app}.sys.noLimiter`, false);
7 8
  9 + logger.debug('disable==>', disable);
  10 +
8 if (disable) { 11 if (disable) {
9 return; 12 return;
10 } 13 }
@@ -20,29 +20,31 @@ const APP_NAME = { @@ -20,29 +20,31 @@ const APP_NAME = {
20 20
21 module.exports = () => { 21 module.exports = () => {
22 const handlers = compose([ 22 const handlers = compose([
23 - // qpsPath, 23 + qpsPath,
24 24
25 disableBelow, 25 disableBelow,
26 xhrFilter, 26 xhrFilter,
27 - // whitelistIpFilter,  
28 - // userFilter,  
29 - // whitelistPathFilter,  
30 - // userAgentFilter,  
31 - // ipFilter,  
32 - // qps 27 + whitelistIpFilter,
  28 + userFilter,
  29 + whitelistPathFilter,
  30 + userAgentFilter,
  31 + ipFilter,
  32 + qps
33 ]); 33 ]);
34 34
35 return async(m) => { 35 return async(m) => {
36 const user = { 36 const user = {
37 - uid: _.get(m, 'fields.uid', ''), 37 + uid: _.parseInt(_.get(m, 'fields.uid', '0'), 10),
38 ip: _.get(m, 'fields.ip', '').replace(/\"/g, ''), 38 ip: _.get(m, 'fields.ip', '').replace(/\"/g, ''),
39 app: APP_NAME[_.get(m, 'tags.app', 'UNKNOWN')], 39 app: APP_NAME[_.get(m, 'tags.app', 'UNKNOWN')],
40 path: decodeURIComponent(_.get(m, 'fields.path', '').replace(/\"/g, '')), 40 path: decodeURIComponent(_.get(m, 'fields.path', '').replace(/\"/g, '')),
41 userAgent: decodeURIComponent(_.get(m, 'fields.userAgent', '').replace(/\"/g, '')), 41 userAgent: decodeURIComponent(_.get(m, 'fields.userAgent', '').replace(/\"/g, '')),
42 - ajax: _.get(m, 'fields.ajax', 0) 42 + ajax: _.parseInt(_.get(m, 'fields.ajax', 0))
43 }; 43 };
44 44
45 - console.log(user); 45 + if (!user.ip || !user.app) {
  46 + return;
  47 + }
46 48
47 await handlers({user}); 49 await handlers({user});
48 }; 50 };
@@ -13,8 +13,8 @@ module.exports = async({user}, next) => { @@ -13,8 +13,8 @@ module.exports = async({user}, next) => {
13 let ipBlackRes = result[0]; 13 let ipBlackRes = result[0];
14 let ipLimitRes = result[1]; 14 let ipLimitRes = result[1];
15 15
16 - logger.debug(ipBlackKey, ipBlackRes);  
17 - logger.debug(ipLimitKey, ipLimitRes); 16 + logger.debug('ip==>', ipBlackKey, ipBlackRes);
  17 + logger.debug('ip==>', ipLimitKey, ipLimitRes);
18 18
19 if ((ipBlackRes && +ipBlackRes > 0) || (ipLimitRes && +ipLimitRes > 0)) { 19 if ((ipBlackRes && +ipBlackRes > 0) || (ipLimitRes && +ipLimitRes > 0)) {
20 return; 20 return;
@@ -38,10 +38,9 @@ module.exports = async({user}, next) => { @@ -38,10 +38,9 @@ module.exports = async({user}, next) => {
38 38
39 const ip = user.ip; 39 const ip = user.ip;
40 const path = user.path; 40 const path = user.path;
41 - const risks = _.get(zk, `${app}.json.risk`, []); 41 + const risks = _.get(zk, `${app}.json.risk`, [{route: '/product/(.*).html', interval: 5000, requests: 10}]);
42 let router = {}; 42 let router = {};
43 43
44 - logger.debug(`risk => risks: ${JSON.stringify(risks)}, path: ${path}, ip: ${ip}`); // eslint-disable-line  
45 if (_.isEmpty(path) || _.isEmpty(risks) || IP_WHITE_LIST.indexOf(ip) > -1) { 44 if (_.isEmpty(path) || _.isEmpty(risks) || IP_WHITE_LIST.indexOf(ip) > -1) {
46 return next(); 45 return next();
47 } 46 }
@@ -65,7 +64,8 @@ module.exports = async({user}, next) => { @@ -65,7 +64,8 @@ module.exports = async({user}, next) => {
65 return false; 64 return false;
66 }); 65 });
67 66
68 - logger.debug(`risk => router: ${JSON.stringify(router)}, path: ${path}`); // eslint-disable-line 67 + logger.debug(`risk==> router: ${JSON.stringify(router)}, path: ${path}`); // eslint-disable-line
  68 +
69 if (_.isEmpty(router)) { 69 if (_.isEmpty(router)) {
70 return next(); 70 return next();
71 } 71 }
@@ -79,9 +79,8 @@ module.exports = async({user}, next) => { @@ -79,9 +79,8 @@ module.exports = async({user}, next) => {
79 cache.getAsync(configKey), 79 cache.getAsync(configKey),
80 ]); 80 ]);
81 81
82 - logger.debug(`risk => getCache: ${JSON.stringify(inters)}, path: ${path}`); // eslint-disable-line  
83 if (inters[0]) { 82 if (inters[0]) {
84 - logger.info('[qps:route] this user[%o] has rejected', user); 83 + logger.info('[qps:route] this user[%j] has rejected', user);
85 return; 84 return;
86 } 85 }
87 86
@@ -97,7 +96,7 @@ module.exports = async({user}, next) => { @@ -97,7 +96,7 @@ module.exports = async({user}, next) => {
97 return next(); 96 return next();
98 } 97 }
99 98
100 - logger.warn('[qps:route] this user[%o] is being marked as rejected', user); 99 + logger.info('[qps:route] this user[%j] is being marked as rejected', user);
101 await Promise.all([ 100 await Promise.all([
102 cache.setAsync(limitKey, 1, INVALIDTIME), 101 cache.setAsync(limitKey, 1, INVALIDTIME),
103 cache.delAsync(configKey) 102 cache.delAsync(configKey)
@@ -32,7 +32,7 @@ module.exports = async({user}, next) => { @@ -32,7 +32,7 @@ module.exports = async({user}, next) => {
32 32
33 return Promise.props(getOp).then((results) => { 33 return Promise.props(getOp).then((results) => {
34 if (results.human) { // 经过验证码之后1小时有效期内不再验证qps 34 if (results.human) { // 经过验证码之后1小时有效期内不再验证qps
35 - logger.warn('[qps] this user[%o] is being marked as human', user); 35 + logger.info('[qps] this user[%o] is being marked as human', user);
36 return {}; 36 return {};
37 } 37 }
38 38
@@ -45,7 +45,7 @@ module.exports = async({user}, next) => { @@ -45,7 +45,7 @@ module.exports = async({user}, next) => {
45 if (!results[key]) { 45 if (!results[key]) {
46 operation[cacheKey] = cache.setAsync(cacheKey, 1, +key); 46 operation[cacheKey] = cache.setAsync(cacheKey, 1, +key);
47 } else if (+results[key] > +val) { 47 } else if (+results[key] > +val) {
48 - logger.warn('[qps] this user[%o] is being marked as rejected', user); 48 + logger.info('[qps] this user[%j] is being marked as rejected', user);
49 49
50 operation[`${user.app}:${limiterKey}:${user.ip}`] = cache.setAsync(`${user.app}:${limiterKey}:${user.ip}`, 1, limiterIpTime); 50 operation[`${user.app}:${limiterKey}:${user.ip}`] = cache.setAsync(`${user.app}:${limiterKey}:${user.ip}`, 1, limiterIpTime);
51 } else { 51 } else {
@@ -55,7 +55,7 @@ module.exports = async({user}, next) => { @@ -55,7 +55,7 @@ module.exports = async({user}, next) => {
55 55
56 return Promise.props(operation); 56 return Promise.props(operation);
57 }).then((result) => { 57 }).then((result) => {
58 - logger.debug('[qps] user[%j] result[%j]', user, result); // eslint-disable-line 58 + // logger.debug('[qps] user[%j] result[%j]', user, result); // eslint-disable-line
59 }).catch(err=>{ 59 }).catch(err=>{
60 logger.error(err); 60 logger.error(err);
61 }).finally(() => { 61 }).finally(() => {
  1 +const logger = global.yoho.logger;
  2 +
1 module.exports = ({user}, next) => { 3 module.exports = ({user}, next) => {
  4 + logger.debug('user==>', user.uid);
  5 +
2 if (user.uid) { 6 if (user.uid) {
3 return; 7 return;
4 } 8 }
@@ -7,11 +7,13 @@ const Promise = require('bluebird'); @@ -7,11 +7,13 @@ const Promise = require('bluebird');
7 7
8 8
9 module.exports = async({user}, next) => { 9 module.exports = async({user}, next) => {
10 - const blackKey = `${user.app}:limiter:ua:black`,  
11 - whiteKey = `${user.app}:limiter:ua:white`; 10 + const blackKey = `${user.app}:limiter:ua:black`;
  11 + const whiteKey = `${user.app}:limiter:ua:white`;
12 12
13 const ua = user.userAgent; 13 const ua = user.userAgent;
14 14
  15 + logger.debug('userAgent==>%s', user.userAgent);
  16 +
15 Promise.all([ 17 Promise.all([
16 cache.getAsync(blackKey), 18 cache.getAsync(blackKey),
17 cache.getAsync(whiteKey) 19 cache.getAsync(whiteKey)
1 const _ = require('lodash'); 1 const _ = require('lodash');
2 -const co = Promise.coroutine;  
3 const logger = global.yoho.logger; 2 const logger = global.yoho.logger;
4 const cache = global.yoho.cache.master; 3 const cache = global.yoho.cache.master;
5 const WHITE_LIST_KEY = 'whitelist:ip:'; 4 const WHITE_LIST_KEY = 'whitelist:ip:';
@@ -29,6 +28,7 @@ module.exports = async({user}, next) => { @@ -29,6 +28,7 @@ module.exports = async({user}, next) => {
29 let key = `${WHITE_LIST_KEY}${remoteIp}`; 28 let key = `${WHITE_LIST_KEY}${remoteIp}`;
30 let remoteIpSegment = `${remoteIp.split('.').slice(0, 2).join('.')}.`; 29 let remoteIpSegment = `${remoteIp.split('.').slice(0, 2).join('.')}.`;
31 30
  31 + logger.debug('whitelist-ip==>%s', user.path);
32 if (_.includes(IP_WHITE_LIST, remoteIp) || _.includes(IP_WHITE_SEGMENT, remoteIpSegment)) { 32 if (_.includes(IP_WHITE_LIST, remoteIp) || _.includes(IP_WHITE_SEGMENT, remoteIpSegment)) {
33 return; 33 return;
34 } 34 }
@@ -51,6 +51,7 @@ const cacheWhiteList = { @@ -51,6 +51,7 @@ const cacheWhiteList = {
51 module.exports = async({user}, next) => { 51 module.exports = async({user}, next) => {
52 const paths = await cacheWhiteList.getValue(); 52 const paths = await cacheWhiteList.getValue();
53 53
  54 + logger.debug('whitelist-path==>', user.path);
54 if (paths.includes(user.path)) { 55 if (paths.includes(user.path)) {
55 return; 56 return;
56 } 57 }
  1 +const logger = global.yoho.logger;
  2 +
1 module.exports = ({user}, next) => { 3 module.exports = ({user}, next) => {
  4 + logger.debug('ajax==>%s %d', user.path, user.ajax);
  5 +
2 if (user.ajax) { 6 if (user.ajax) {
3 return; 7 return;
4 } 8 }
@@ -3,6 +3,7 @@ const MysqlSender = require('../lib/mysql-sender'); @@ -3,6 +3,7 @@ const MysqlSender = require('../lib/mysql-sender');
3 const config = require('../common/config'); 3 const config = require('../common/config');
4 const msg2row = require('./msg2row'); 4 const msg2row = require('./msg2row');
5 5
  6 +
6 const logger = global.yoho.logger; 7 const logger = global.yoho.logger;
7 const errorSqlSender = new MysqlSender(config.table.error); 8 const errorSqlSender = new MysqlSender(config.table.error);
8 const slowRouterSqlSender = new MysqlSender(config.table.slow); 9 const slowRouterSqlSender = new MysqlSender(config.table.slow);
@@ -15,7 +16,7 @@ const API_BLACK_LIST = [ @@ -15,7 +16,7 @@ const API_BLACK_LIST = [
15 'app.shop.banner' 16 'app.shop.banner'
16 ]; 17 ];
17 18
18 -function handleWebServerDuration(m) { 19 +async function handleWebServerDuration(m) {
19 let duration = _.parseInt(m.fields.duration); 20 let duration = _.parseInt(m.fields.duration);
20 21
21 if (duration > config.slowRoute.min / 10 && duration < config.slowRoute.max) { 22 if (duration > config.slowRoute.min / 10 && duration < config.slowRoute.max) {
@@ -10,7 +10,6 @@ const { @@ -10,7 +10,6 @@ const {
10 } = require('./serverapm-service'); 10 } = require('./serverapm-service');
11 11
12 const riskService = require('./risk-service'); 12 const riskService = require('./risk-service');
13 -  
14 const handleRisk = riskService(); 13 const handleRisk = riskService();
15 14
16 const server = { 15 const server = {
@@ -25,8 +24,11 @@ const server = { @@ -25,8 +24,11 @@ const server = {
25 24
26 switch (m.measurement) { 25 switch (m.measurement) {
27 case 'web-server-duration': { 26 case 'web-server-duration': {
28 - handleWebServerDuration(m); 27 + // handleWebServerDuration(m);
  28 +
  29 + if (m.tags.type === 'route') {
29 await handleRisk(m); 30 await handleRisk(m);
  31 + }
30 break; 32 break;
31 } 33 }
32 case 'error-report': { 34 case 'error-report': {