fe / yohobuy-node · Commits

Go to a project

Authored by htoooth
Commit 0dea998c834bb519f42934ab12ad76af5ffbcc30 2 parents 0a15d138 c2854864

Merge branch 'master' into feature/captcha-change

Inline Side-by-side
Showing 7 changed files with 126 additions and 158 deletions
@@ -59,11 +59,6 @@ if (config.zookeeperServer) { @@ -59,11 +59,6 @@ if (config.zookeeperServer) {
59 59
60 app.enable('trust proxy'); 60 app.enable('trust proxy');
61 61
62 -// 请求限制中间件  
63 -if (!app.locals.devEnv) {  
64 - app.use(require('./doraemon/middleware/limiter'));  
65 -}  
66 -  
67 app.set('subdomain offset', 2); 62 app.set('subdomain offset', 2);
68 63
69 // 添加请求上下文 64 // 添加请求上下文
@@ -149,6 +144,12 @@ try { @@ -149,6 +144,12 @@ try {
149 app.use(mobileRefer()); 144 app.use(mobileRefer());
150 app.use(mobileCheck()); 145 app.use(mobileCheck());
151 app.use(user()); 146 app.use(user());
  147 +
  148 + // 请求限制中间件
  149 + if (!app.locals.devEnv) {
  150 + app.use(require('./doraemon/middleware/limiter'));
  151 + }
  152 +
152 app.use(seo()); 153 app.use(seo());
153 app.use(setPageInfo()); 154 app.use(setPageInfo());
154 app.use(layoutTools()); 155 app.use(layoutTools());
@@ -3,6 +3,8 @@ @@ -3,6 +3,8 @@
3 const cache = global.yoho.cache.master; 3 const cache = global.yoho.cache.master;
4 const Promise = require('bluebird'); 4 const Promise = require('bluebird');
5 const co = Promise.coroutine; 5 const co = Promise.coroutine;
  6 +const config = global.yoho.config;
  7 +const _ = require('lodash');
6 8
7 const HeaderModel = require('../../../doraemon/models/header'); 9 const HeaderModel = require('../../../doraemon/models/header');
8 10
@@ -15,24 +17,13 @@ const index = co(function* (channel) { @@ -15,24 +17,13 @@ const index = co(function* (channel) {
15 }); 17 });
16 18
17 const removeBlack = (remoteIp) => { 19 const removeBlack = (remoteIp) => {
18 - let key = `pc:limiter:${remoteIp}`,  
19 - key10m = `pc:limiter:10m:${remoteIp}`,  
20 - keyMax = `pc:limiter:max:${remoteIp}`,  
21 - key10mMax = `pc:limiter:10m:max:${remoteIp}`,  
22 - synchronizeKey = `pc:limiter:synchronize:${remoteIp}`,  
23 - asynchronousKey = `pc:limiter:asynchronous:${remoteIp}`,  
24 - spiderKey = `pc:limiter:spider:${remoteIp}`;  
25 -  
26 -  
27 - return Promise.all([  
28 - cache.delAsync(key),  
29 - cache.delAsync(key10m),  
30 - cache.delAsync(keyMax),  
31 - cache.delAsync(key10mMax),  
32 - cache.delAsync(synchronizeKey),  
33 - cache.delAsync(asynchronousKey),  
34 - cache.delAsync(spiderKey)  
35 - ]); 20 + let operations = [cache.delAsync(`${config.app}:limiter:${remoteIp}`)];
  21 +
  22 + _.forEach(config.REQUEST_LIMIT, (val, key) => {
  23 + operations.push(cache.delAsync(`${config.app}:limiter:${key}:max:${remoteIp}`));
  24 + });
  25 +
  26 + return Promise.all(operations);
36 }; 27 };
37 28
38 module.exports = { 29 module.exports = {
@@ -141,8 +141,6 @@ module.exports = { @@ -141,8 +141,6 @@ module.exports = {
141 cache: false 141 cache: false
142 }, 142 },
143 zookeeperServer: '192.168.102.168:2188', 143 zookeeperServer: '192.168.102.168:2188',
144 - maxQps: 1200,  
145 - maxQps10m: 2500,  
146 sessionMemcachedPrefix: 'yohobuy_session:', 144 sessionMemcachedPrefix: 'yohobuy_session:',
147 redis: { 145 redis: {
148 connect: { 146 connect: {
@@ -162,6 +160,17 @@ module.exports = { @@ -162,6 +160,17 @@ module.exports = {
162 return Math.min(options.attempt * 100, 1000); 160 return Math.min(options.attempt * 100, 1000);
163 } 161 }
164 } 162 }
  163 + },
  164 + REQUEST_LIMIT: {
  165 +
  166 + // 30s 最多访问15次
  167 + 30: 15,
  168 +
  169 + // 60s 最多访问15次
  170 + 60: 20,
  171 +
  172 + // 100s 最多访问15次
  173 + 600: 100
165 } 174 }
166 }; 175 };
167 176
@@ -174,7 +183,7 @@ if (isProduction) { @@ -174,7 +183,7 @@ if (isProduction) {
174 service: 'http://api.yoho.yohoops.org/', 183 service: 'http://api.yoho.yohoops.org/',
175 search: 'http://search.yohoops.org/yohosearch/', 184 search: 'http://search.yohoops.org/yohosearch/',
176 global: 'http://api-global.yohobuy.com/', 185 global: 'http://api-global.yohobuy.com/',
177 - serviceNotify: 'http://api.yoho.yohoops.org/', 186 + serviceNotify: 'http://service.yoho.cn/',
178 imSocket: 'wss://imsocket.yohobuy.com:443', 187 imSocket: 'wss://imsocket.yohobuy.com:443',
179 imCs: 'https://imhttp.yohobuy.com/api', 188 imCs: 'https://imhttp.yohobuy.com/api',
180 platformApi: 'http://api.platform.yohoops.org', 189 platformApi: 'http://api.platform.yohoops.org',
@@ -29,56 +29,62 @@ const sortMap = { @@ -29,56 +29,62 @@ const sortMap = {
29 boys: [ 29 boys: [
30 {misort: 16, viewNum: 5}, 30 {misort: 16, viewNum: 5},
31 {misort: 21, viewNum: 5}, 31 {misort: 21, viewNum: 5},
32 - {misort: 12, viewNum: 5}, 32 + {misort: 1900, viewNum: 5},
  33 + {misort: 1904, viewNum: 5},
  34 + {misort: 226, viewNum: 5},
  35 + {misort: 1896, viewNum: 5},
33 {misort: 26, viewNum: 5}, 36 {misort: 26, viewNum: 5},
  37 + {misort: 27, viewNum: 5},
34 {misort: 44, viewNum: 5}, 38 {misort: 44, viewNum: 5},
  39 + {misort: 45, viewNum: 5},
35 {misort: 49, viewNum: 5}, 40 {misort: 49, viewNum: 5},
36 - {misort: 27, viewNum: 5},  
37 {misort: 60, viewNum: 5}, 41 {misort: 60, viewNum: 5},
38 - {misort: 59, viewNum: 5},  
39 - {misort: 65, viewNum: 5},  
40 - {misort: 50, viewNum: 5},  
41 - {misort: 237, viewNum: 5},  
42 - {misort: 45, viewNum: 5},  
43 - {misort: 345, viewNum: 5},  
44 - {misort: 226, viewNum: 5},  
45 - {misort: 39, viewNum: 5},  
46 - {misort: 66, viewNum: 5} 42 + {misort: 39, viewNum: 5}
47 ], 43 ],
48 girls: [ 44 girls: [
49 -  
50 - {misort: 16, viewNum: 4},  
51 {misort: 21, viewNum: 4}, 45 {misort: 21, viewNum: 4},
52 - {misort: 12, viewNum: 4},  
53 - {misort: 27, viewNum: 4},  
54 - {misort: 26, viewNum: 4},  
55 - {misort: 44, viewNum: 4}, 46 + {misort: 16, viewNum: 4},
56 {misort: 1900, viewNum: 4}, 47 {misort: 1900, viewNum: 4},
  48 + {misort: 1904, viewNum: 4},
  49 + {misort: 1896, viewNum: 4},
  50 + {misort: 1892, viewNum: 4},
  51 + {misort: 20, viewNum: 4},
  52 + {misort: 44, viewNum: 4},
  53 + {misort: 26, viewNum: 4},
  54 + {misort: 27, viewNum: 4},
  55 + {misort: 45, viewNum: 4},
  56 + {misort: 226, viewNum: 4},
  57 + {misort: 172, viewNum: 4},
57 {misort: 31, viewNum: 4}, 58 {misort: 31, viewNum: 4},
58 {misort: 49, viewNum: 4}, 59 {misort: 49, viewNum: 4},
59 {misort: 50, viewNum: 4}, 60 {misort: 50, viewNum: 4},
60 - {misort: 65, viewNum: 4},  
61 {misort: 60, viewNum: 4}, 61 {misort: 60, viewNum: 4},
62 - {misort: 48, viewNum: 4},  
63 - {misort: 1896, viewNum: 4},  
64 - {misort: 32, viewNum: 4},  
65 - {misort: 66, viewNum: 4},  
66 - {misort: 39, viewNum: 4},  
67 - {misort: 1180, viewNum: 4}  
68 - 62 + {misort: 65, viewNum: 4}
69 ], 63 ],
70 kids: [ 64 kids: [
71 - {misort: 405, viewNum: 4},  
72 {misort: 396, viewNum: 4}, 65 {misort: 396, viewNum: 4},
73 - {misort: 369, viewNum: 4},  
74 - {misort: 384, viewNum: 4},  
75 - {misort: 367, viewNum: 4} 66 + {misort: 404, viewNum: 4},
  67 + {misort: 400, viewNum: 4},
  68 + {misort: 368, viewNum: 4},
  69 + {misort: 406, viewNum: 4},
  70 + {misort: 390, viewNum: 4},
  71 + {misort: 414, viewNum: 4},
  72 + {misort: 448, viewNum: 4},
  73 + {misort: 429, viewNum: 4},
  74 + {misort: 408, viewNum: 4},
  75 + {misort: 470, viewNum: 4},
  76 + {misort: 406, viewNum: 4},
  77 + {misort: 388, viewNum: 4}
76 ], 78 ],
77 lifestyle: [ 79 lifestyle: [
78 - {msort: 1172, viewNum: 5},  
79 - {msort: 1178, viewNum: 5}, 80 + {msort: 1170, viewNum: 5},
80 {msort: 1174, viewNum: 5}, 81 {msort: 1174, viewNum: 5},
81 - {msort: 1182, viewNum: 5} 82 + {msort: 1178, viewNum: 5},
  83 + {msort: 1180, viewNum: 5},
  84 + {msort: 1182, viewNum: 5},
  85 + {msort: 1184, viewNum: 5},
  86 + {msort: 1176, viewNum: 5},
  87 + {msort: 1186, viewNum: 5}
82 ] 88 ]
83 }; 89 };
84 90
@@ -5,8 +5,8 @@ const logger = global.yoho.logger; @@ -5,8 +5,8 @@ const logger = global.yoho.logger;
5 const ip = require('./rules/ip-list'); 5 const ip = require('./rules/ip-list');
6 const userAgent = require('./rules/useragent'); 6 const userAgent = require('./rules/useragent');
7 const qpsLimiter = require('./rules/qps-limit'); 7 const qpsLimiter = require('./rules/qps-limit');
8 -const asynchronous = require('./rules/asynchronous');  
9 8
  9 +// const asynchronous = require('./rules/asynchronous');
10 // const fakerLimiter = require('./rules/faker-limit'); 10 // const fakerLimiter = require('./rules/faker-limit');
11 const captchaPolicy = require('./policies/captcha'); 11 const captchaPolicy = require('./policies/captcha');
12 12
@@ -15,7 +15,16 @@ const captchaPolicy = require('./policies/captcha'); @@ -15,7 +15,16 @@ const captchaPolicy = require('./policies/captcha');
15 const IP_WHITE_LIST = [ 15 const IP_WHITE_LIST = [
16 '106.38.38.146', 16 '106.38.38.146',
17 '218.94.75.58', 17 '218.94.75.58',
18 - '218.94.75.50' 18 + '218.94.75.50',
  19 + '218.94.77.166'
  20 +];
  21 +
  22 +const PATH_WHITE_LIST = [
  23 + '/3party/check',
  24 + '/passport/images.png',
  25 + '/passport/cert/headerTip',
  26 + '/common/getbanner',
  27 + '/common/suggestfeedback'
19 ]; 28 ];
20 29
21 const limiter = (rule, policy, context) => { 30 const limiter = (rule, policy, context) => {
@@ -37,7 +46,9 @@ module.exports = (req, res, next) => { @@ -37,7 +46,9 @@ module.exports = (req, res, next) => {
37 remoteIp = req.get('X-Real-IP'); 46 remoteIp = req.get('X-Real-IP');
38 } 47 }
39 48
40 - const excluded = _.includes(IP_WHITE_LIST, remoteIp); 49 + // 排除条件：ip白名单/路径白名单/异步请求/登录用户
  50 + const excluded = _.includes(IP_WHITE_LIST, remoteIp) ||
  51 + _.includes(PATH_WHITE_LIST, req.path) || req.xhr || !_.isEmpty(_.get(req, 'user.uid'));
41 const enabled = !_.get(req.app.locals, 'pc.sys.noLimiter'); 52 const enabled = !_.get(req.app.locals, 'pc.sys.noLimiter');
42 53
43 logger.info(`request remote ip: ${remoteIp}; excluded: ${excluded}; enabled: ${enabled}`); 54 logger.info(`request remote ip: ${remoteIp}; excluded: ${excluded}; enabled: ${enabled}`);
@@ -54,9 +65,9 @@ module.exports = (req, res, next) => { @@ -54,9 +65,9 @@ module.exports = (req, res, next) => {
54 Promise.all([ 65 Promise.all([
55 limiter(userAgent, captchaPolicy, context), 66 limiter(userAgent, captchaPolicy, context),
56 limiter(ip, captchaPolicy, context), 67 limiter(ip, captchaPolicy, context),
57 - limiter(qpsLimiter, captchaPolicy, context),  
58 - limiter(asynchronous, captchaPolicy, context) 68 + limiter(qpsLimiter, captchaPolicy, context)
59 69
  70 + // limiter(asynchronous, captchaPolicy, context)
60 // limiter(fakerLimiter, reporterPolicy, context) 71 // limiter(fakerLimiter, reporterPolicy, context)
61 ]).then((results) => { 72 ]).then((results) => {
62 let allPass = true, exclusion = false, policy = null; 73 let allPass = true, exclusion = false, policy = null;
@@ -68,11 +79,8 @@ module.exports = (req, res, next) => { @@ -68,11 +79,8 @@ module.exports = (req, res, next) => {
68 exclusion = result.exclusion; 79 exclusion = result.exclusion;
69 } 80 }
70 81
71 - if (!excluded && typeof result === 'function') {  
72 - allPass = false;  
73 - }  
74 -  
75 if (typeof result === 'function') { 82 if (typeof result === 'function') {
  83 + allPass = false;
76 policy = result; 84 policy = result;
77 } 85 }
78 }); 86 });
@@ -2,15 +2,17 @@ @@ -2,15 +2,17 @@
2 2
3 const cache = global.yoho.cache.master; 3 const cache = global.yoho.cache.master;
4 const _ = require('lodash'); 4 const _ = require('lodash');
  5 +const config = global.yoho.config;
  6 +const logger = global.yoho.logger;
5 7
6 -module.exports = (limiter) => {  
7 - const key = `pc:limiter:${limiter.remoteIp}`; 8 +module.exports = (limiter, policy) => {
  9 + const key = `${config.app}:limiter:${limiter.remoteIp}`;
8 10
9 return cache.getAsync(key).then((result) => { 11 return cache.getAsync(key).then((result) => {
  12 + logger.debug(key, result);
  13 +
10 if (result && _.isNumber(result)) { 14 if (result && _.isNumber(result)) {
11 - return Promise.resolve({  
12 - exclusion: result === -1  
13 - }); 15 + return Promise.resolve(policy);
14 } else { 16 } else {
15 return Promise.resolve(true); 17 return Promise.resolve(true);
16 } 18 }
  1 +/**
  2 + * 限制页面访问次数，如超过限制次数，返回相应策略（目前是ip加入黑名单，跳转图形验证码页面，解除限制）
  3 + * 当前规则只针对未登录用户
  4 + */
  5 +
1 'use strict'; 6 'use strict';
2 7
3 const logger = global.yoho.logger; 8 const logger = global.yoho.logger;
4 const cache = global.yoho.cache.master; 9 const cache = global.yoho.cache.master;
5 const config = global.yoho.config; 10 const config = global.yoho.config;
6 -const ONE_DAY = 60 * 60 * 24;  
7 -const MAX_QPS = config.maxQps;  
8 -const MAX_QPS_10m = config.maxQps10m; // eslint-disable-line  
9 const _ = require('lodash'); 11 const _ = require('lodash');
10 12
11 -const PAGES = {  
12 - '/product/^\\/([\\d]+)(.*)/': 5,  
13 - '/product/list/index': 5,  
14 - '/product/search/index': 5  
15 -}; 13 +// 超出访问限制ip限制访问1小时
  14 +const limiterIpTime = 3600;
16 15
17 -function urlJoin(a, b) {  
18 - if (_.endsWith(a, '/') && _.startsWith(b, '/')) {  
19 - return a + b.substring(1, b.length);  
20 - } else if (!_.endsWith(a, '/') && !_.startsWith(b, '/')) {  
21 - return a + '/' + b;  
22 - } else {  
23 - return a + b;  
24 - }  
25 -} 16 +// 页面访问限制
  17 +const MAX_TIMES = config.REQUEST_LIMIT;
26 18
27 module.exports = (limiter, policy) => { 19 module.exports = (limiter, policy) => {
28 - const req = limiter.req,  
29 - res = limiter.res,  
30 - next = limiter.next; // eslint-disable-line  
31 -  
32 - const key = `pc:limiter:${limiter.remoteIp}`;  
33 - const keyMax = `pc:limiter:max:${limiter.remoteIp}`;  
34 - const key10m = `pc:limiter:10m:${limiter.remoteIp}`;  
35 - const key10mMax = `pc:limiter:10m:max:${limiter.remoteIp}`;  
36 -  
37 - res.on('render', function() {  
38 - let route = req.route ? req.route.path : '';  
39 - let appPath = req.app.mountpath;  
40 -  
41 - if (_.isArray(route) && route.length > 0) {  
42 - route = route[0];  
43 - }  
44 20
45 - let pageKey = urlJoin(appPath, route.toString()); // route may be a regexp  
46 - let pageIncr = PAGES[pageKey] || 0; 21 + // 存储规则的cache keys
  22 + let ruleKeys = {};
  23 + let getOp = {};
47 24
48 - if (pageIncr > 0) {  
49 - cache.incr(key, pageIncr, (err) => {}); // eslint-disable-line  
50 - cache.incr(key10m, pageIncr, (err) => {}); // eslint-disable-line  
51 - } 25 + _.forEach(MAX_TIMES, (val, key) => {
  26 + ruleKeys[key] = `${config.app}:limiter:${key}:max:${limiter.remoteIp}`;
  27 + getOp[key] = cache.getAsync(ruleKeys[key]);
52 }); 28 });
53 29
54 - return cache.getMultiAsync([key, key10m, keyMax, key10mMax]).then((results) => {  
55 - let result = results[key];  
56 - let result10m = results[key10m];  
57 -  
58 - logger.debug('qps limiter: ' + key + '@' + result + ' max: ' + MAX_QPS);  
59 - logger.debug('qps limiter:10m ' + key10m + '@' + result10m + ' max: ' + MAX_QPS_10m); // eslint-disable-line  
60 -  
61 - // 达到1分钟或是10分钟的访问限制，禁止访问  
62 - if (results[keyMax] === 1 || results[key10mMax] === 1) {  
63 - return Promise.resolve(policy);  
64 - }  
65 -  
66 - // 默认数据设置  
67 - if (!result && !_.isNumber(result)) {  
68 - cache.setAsync(key, 1, 60); // 设置key，1m失效  
69 - }  
70 -  
71 - if (!result10m && !_.isNumber(result10m)) {  
72 - cache.setAsync(key10m, 1, 600); // 设置key，10m失效  
73 - } 30 + return Promise.props(getOp).then((results) => {
74 31
75 - // 第一次访问，都没计数，直接过  
76 - if (!result && !_.isNumber(result) && !result10m && !_.isNumber(result10m)) {  
77 - return Promise.resolve(true);  
78 - } 32 + logger.debug(MAX_TIMES);
  33 + logger.debug(_.values(ruleKeys));
  34 + logger.debug(results);
79 35
80 - if (result === -1 || result10m === -1) {  
81 - return Promise.resolve(true);  
82 - } 36 + // 遍历限制规则，若满足返回相应处理策略， 否则页面访问次数加1
  37 + let operation = [];
83 38
84 - // 判断 qps 10分钟  
85 - if (result10m === 9999) {  
86 - res.statusCode = 403;  
87 - return Promise.resolve(policy);  
88 - } else if (result10m > MAX_QPS_10m) { // eslint-disable-line  
89 - cache.setAsync(key10mMax, 1, ONE_DAY);  
90 - logger.debug('req limit', key10m); 39 + _.forEach(MAX_TIMES, (val, key) => {
  40 + let cacheKey = ruleKeys[key];
91 41
92 - return Promise.resolve(policy);  
93 - } 42 + if (!results[key]) {
  43 + operation.push(cache.setAsync(cacheKey, 1, +key));
  44 + } else if (+results[key] > +val) {
94 45
95 - // 判断 qps 1分钟  
96 - if (result === 9999) {  
97 - res.statusCode = 403;  
98 - return Promise.resolve(policy);  
99 - } else if (result > MAX_QPS) { // 判断 qps  
100 - cache.setAsync(keyMax, 1, ONE_DAY);  
101 - logger.debug('req limit', key); 46 + // ip限制1小时
  47 + operation.push(cache.setAsync(`${config.app}:limiter:${limiter.remoteIp}`, 1, limiterIpTime));
  48 + return Promise.resolve(policy);
  49 + } else {
  50 + operation.push(cache.incrAsync(cacheKey, 1));
  51 + }
  52 + });
102 53
103 - return Promise.resolve(policy);  
104 - } 54 + Promise.all(operation);
105 55
106 - cache.incrAsync(key, 1); // qps + 1  
107 - cache.incrAsync(key10m, 1); // qps + 1 56 + // 不满足任何限制规则，继续访问
108 return Promise.resolve(true); 57 return Promise.resolve(true);
  58 + }).catch(err=>{
  59 + logger.error(err);
109 }); 60 });
110 }; 61 };