fe / yohobuy-node · Commits

Go to a project

Authored by htoooth
Commit 0dea998c834bb519f42934ab12ad76af5ffbcc30 2 parents 0a15d138 c2854864

Merge branch 'master' into feature/captcha-change

Inline Side-by-side
Showing 7 changed files with 126 additions and 158 deletions
... ... @@ -59,11 +59,6 @@ if (config.zookeeperServer) {
app.enable('trust proxy');
// 请求限制中间件
if (!app.locals.devEnv) {
app.use(require('./doraemon/middleware/limiter'));
}
app.set('subdomain offset', 2);
// 添加请求上下文
... ... @@ -149,6 +144,12 @@ try {
app.use(mobileRefer());
app.use(mobileCheck());
app.use(user());
// 请求限制中间件
if (!app.locals.devEnv) {
app.use(require('./doraemon/middleware/limiter'));
}
app.use(seo());
app.use(setPageInfo());
app.use(layoutTools());
... ...
... ... @@ -3,6 +3,8 @@
const cache = global.yoho.cache.master;
const Promise = require('bluebird');
const co = Promise.coroutine;
const config = global.yoho.config;
const _ = require('lodash');
const HeaderModel = require('../../../doraemon/models/header');
... ... @@ -15,24 +17,13 @@ const index = co(function* (channel) {
});
const removeBlack = (remoteIp) => {
let key = `pc:limiter:${remoteIp}`,
key10m = `pc:limiter:10m:${remoteIp}`,
keyMax = `pc:limiter:max:${remoteIp}`,
key10mMax = `pc:limiter:10m:max:${remoteIp}`,
synchronizeKey = `pc:limiter:synchronize:${remoteIp}`,
asynchronousKey = `pc:limiter:asynchronous:${remoteIp}`,
spiderKey = `pc:limiter:spider:${remoteIp}`;
return Promise.all([
cache.delAsync(key),
cache.delAsync(key10m),
cache.delAsync(keyMax),
cache.delAsync(key10mMax),
cache.delAsync(synchronizeKey),
cache.delAsync(asynchronousKey),
cache.delAsync(spiderKey)
]);
let operations = [cache.delAsync(`${config.app}:limiter:${remoteIp}`)];
_.forEach(config.REQUEST_LIMIT, (val, key) => {
operations.push(cache.delAsync(`${config.app}:limiter:${key}:max:${remoteIp}`));
});
return Promise.all(operations);
};
module.exports = {
... ...
... ... @@ -141,8 +141,6 @@ module.exports = {
cache: false
},
zookeeperServer: '192.168.102.168:2188',
maxQps: 1200,
maxQps10m: 2500,
sessionMemcachedPrefix: 'yohobuy_session:',
redis: {
connect: {
... ... @@ -162,6 +160,17 @@ module.exports = {
return Math.min(options.attempt * 100, 1000);
}
}
},
REQUEST_LIMIT: {
// 30s 最多访问15次
30: 15,
// 60s 最多访问15次
60: 20,
// 100s 最多访问15次
600: 100
}
};
... ... @@ -174,7 +183,7 @@ if (isProduction) {
service: 'http://api.yoho.yohoops.org/',
search: 'http://search.yohoops.org/yohosearch/',
global: 'http://api-global.yohobuy.com/',
serviceNotify: 'http://api.yoho.yohoops.org/',
serviceNotify: 'http://service.yoho.cn/',
imSocket: 'wss://imsocket.yohobuy.com:443',
imCs: 'https://imhttp.yohobuy.com/api',
platformApi: 'http://api.platform.yohoops.org',
... ...
... ... @@ -29,56 +29,62 @@ const sortMap = {
boys: [
{misort: 16, viewNum: 5},
{misort: 21, viewNum: 5},
{misort: 12, viewNum: 5},
{misort: 1900, viewNum: 5},
{misort: 1904, viewNum: 5},
{misort: 226, viewNum: 5},
{misort: 1896, viewNum: 5},
{misort: 26, viewNum: 5},
{misort: 27, viewNum: 5},
{misort: 44, viewNum: 5},
{misort: 45, viewNum: 5},
{misort: 49, viewNum: 5},
{misort: 27, viewNum: 5},
{misort: 60, viewNum: 5},
{misort: 59, viewNum: 5},
{misort: 65, viewNum: 5},
{misort: 50, viewNum: 5},
{misort: 237, viewNum: 5},
{misort: 45, viewNum: 5},
{misort: 345, viewNum: 5},
{misort: 226, viewNum: 5},
{misort: 39, viewNum: 5},
{misort: 66, viewNum: 5}
{misort: 39, viewNum: 5}
],
girls: [
{misort: 16, viewNum: 4},
{misort: 21, viewNum: 4},
{misort: 12, viewNum: 4},
{misort: 27, viewNum: 4},
{misort: 26, viewNum: 4},
{misort: 44, viewNum: 4},
{misort: 16, viewNum: 4},
{misort: 1900, viewNum: 4},
{misort: 1904, viewNum: 4},
{misort: 1896, viewNum: 4},
{misort: 1892, viewNum: 4},
{misort: 20, viewNum: 4},
{misort: 44, viewNum: 4},
{misort: 26, viewNum: 4},
{misort: 27, viewNum: 4},
{misort: 45, viewNum: 4},
{misort: 226, viewNum: 4},
{misort: 172, viewNum: 4},
{misort: 31, viewNum: 4},
{misort: 49, viewNum: 4},
{misort: 50, viewNum: 4},
{misort: 65, viewNum: 4},
{misort: 60, viewNum: 4},
{misort: 48, viewNum: 4},
{misort: 1896, viewNum: 4},
{misort: 32, viewNum: 4},
{misort: 66, viewNum: 4},
{misort: 39, viewNum: 4},
{misort: 1180, viewNum: 4}
{misort: 65, viewNum: 4}
],
kids: [
{misort: 405, viewNum: 4},
{misort: 396, viewNum: 4},
{misort: 369, viewNum: 4},
{misort: 384, viewNum: 4},
{misort: 367, viewNum: 4}
{misort: 404, viewNum: 4},
{misort: 400, viewNum: 4},
{misort: 368, viewNum: 4},
{misort: 406, viewNum: 4},
{misort: 390, viewNum: 4},
{misort: 414, viewNum: 4},
{misort: 448, viewNum: 4},
{misort: 429, viewNum: 4},
{misort: 408, viewNum: 4},
{misort: 470, viewNum: 4},
{misort: 406, viewNum: 4},
{misort: 388, viewNum: 4}
],
lifestyle: [
{msort: 1172, viewNum: 5},
{msort: 1178, viewNum: 5},
{msort: 1170, viewNum: 5},
{msort: 1174, viewNum: 5},
{msort: 1182, viewNum: 5}
{msort: 1178, viewNum: 5},
{msort: 1180, viewNum: 5},
{msort: 1182, viewNum: 5},
{msort: 1184, viewNum: 5},
{msort: 1176, viewNum: 5},
{msort: 1186, viewNum: 5}
]
};
... ...
... ... @@ -5,8 +5,8 @@ const logger = global.yoho.logger;
const ip = require('./rules/ip-list');
const userAgent = require('./rules/useragent');
const qpsLimiter = require('./rules/qps-limit');
const asynchronous = require('./rules/asynchronous');
// const asynchronous = require('./rules/asynchronous');
// const fakerLimiter = require('./rules/faker-limit');
const captchaPolicy = require('./policies/captcha');
... ... @@ -15,7 +15,16 @@ const captchaPolicy = require('./policies/captcha');
const IP_WHITE_LIST = [
'106.38.38.146',
'218.94.75.58',
'218.94.75.50'
'218.94.75.50',
'218.94.77.166'
];
const PATH_WHITE_LIST = [
'/3party/check',
'/passport/images.png',
'/passport/cert/headerTip',
'/common/getbanner',
'/common/suggestfeedback'
];
const limiter = (rule, policy, context) => {
... ... @@ -37,7 +46,9 @@ module.exports = (req, res, next) => {
remoteIp = req.get('X-Real-IP');
}
const excluded = _.includes(IP_WHITE_LIST, remoteIp);
// 排除条件：ip白名单/路径白名单/异步请求/登录用户
const excluded = _.includes(IP_WHITE_LIST, remoteIp) ||
_.includes(PATH_WHITE_LIST, req.path) || req.xhr || !_.isEmpty(_.get(req, 'user.uid'));
const enabled = !_.get(req.app.locals, 'pc.sys.noLimiter');
logger.info(`request remote ip: ${remoteIp}; excluded: ${excluded}; enabled: ${enabled}`);
... ... @@ -54,9 +65,9 @@ module.exports = (req, res, next) => {
Promise.all([
limiter(userAgent, captchaPolicy, context),
limiter(ip, captchaPolicy, context),
limiter(qpsLimiter, captchaPolicy, context),
limiter(asynchronous, captchaPolicy, context)
limiter(qpsLimiter, captchaPolicy, context)
// limiter(asynchronous, captchaPolicy, context)
// limiter(fakerLimiter, reporterPolicy, context)
]).then((results) => {
let allPass = true, exclusion = false, policy = null;
... ... @@ -68,11 +79,8 @@ module.exports = (req, res, next) => {
exclusion = result.exclusion;
}
if (!excluded && typeof result === 'function') {
allPass = false;
}
if (typeof result === 'function') {
allPass = false;
policy = result;
}
});
... ...
... ... @@ -2,15 +2,17 @@
const cache = global.yoho.cache.master;
const _ = require('lodash');
const config = global.yoho.config;
const logger = global.yoho.logger;
module.exports = (limiter) => {
const key = `pc:limiter:${limiter.remoteIp}`;
module.exports = (limiter, policy) => {
const key = `${config.app}:limiter:${limiter.remoteIp}`;
return cache.getAsync(key).then((result) => {
logger.debug(key, result);
if (result && _.isNumber(result)) {
return Promise.resolve({
exclusion: result === -1
});
return Promise.resolve(policy);
} else {
return Promise.resolve(true);
}
... ...
/**
* 限制页面访问次数，如超过限制次数，返回相应策略（目前是ip加入黑名单，跳转图形验证码页面，解除限制）
* 当前规则只针对未登录用户
*/
'use strict';
const logger = global.yoho.logger;
const cache = global.yoho.cache.master;
const config = global.yoho.config;
const ONE_DAY = 60 * 60 * 24;
const MAX_QPS = config.maxQps;
const MAX_QPS_10m = config.maxQps10m; // eslint-disable-line
const _ = require('lodash');
const PAGES = {
'/product/^\\/([\\d]+)(.*)/': 5,
'/product/list/index': 5,
'/product/search/index': 5
};
// 超出访问限制ip限制访问1小时
const limiterIpTime = 3600;
function urlJoin(a, b) {
if (_.endsWith(a, '/') && _.startsWith(b, '/')) {
return a + b.substring(1, b.length);
} else if (!_.endsWith(a, '/') && !_.startsWith(b, '/')) {
return a + '/' + b;
} else {
return a + b;
}
}
// 页面访问限制
const MAX_TIMES = config.REQUEST_LIMIT;
module.exports = (limiter, policy) => {
const req = limiter.req,
res = limiter.res,
next = limiter.next; // eslint-disable-line
const key = `pc:limiter:${limiter.remoteIp}`;
const keyMax = `pc:limiter:max:${limiter.remoteIp}`;
const key10m = `pc:limiter:10m:${limiter.remoteIp}`;
const key10mMax = `pc:limiter:10m:max:${limiter.remoteIp}`;
res.on('render', function() {
let route = req.route ? req.route.path : '';
let appPath = req.app.mountpath;
if (_.isArray(route) && route.length > 0) {
route = route[0];
}
let pageKey = urlJoin(appPath, route.toString()); // route may be a regexp
let pageIncr = PAGES[pageKey] || 0;
// 存储规则的cache keys
let ruleKeys = {};
let getOp = {};
if (pageIncr > 0) {
cache.incr(key, pageIncr, (err) => {}); // eslint-disable-line
cache.incr(key10m, pageIncr, (err) => {}); // eslint-disable-line
}
_.forEach(MAX_TIMES, (val, key) => {
ruleKeys[key] = `${config.app}:limiter:${key}:max:${limiter.remoteIp}`;
getOp[key] = cache.getAsync(ruleKeys[key]);
});
return cache.getMultiAsync([key, key10m, keyMax, key10mMax]).then((results) => {
let result = results[key];
let result10m = results[key10m];
logger.debug('qps limiter: ' + key + '@' + result + ' max: ' + MAX_QPS);
logger.debug('qps limiter:10m ' + key10m + '@' + result10m + ' max: ' + MAX_QPS_10m); // eslint-disable-line
// 达到1分钟或是10分钟的访问限制，禁止访问
if (results[keyMax] === 1 || results[key10mMax] === 1) {
return Promise.resolve(policy);
}
// 默认数据设置
if (!result && !_.isNumber(result)) {
cache.setAsync(key, 1, 60); // 设置key，1m失效
}
if (!result10m && !_.isNumber(result10m)) {
cache.setAsync(key10m, 1, 600); // 设置key，10m失效
}
return Promise.props(getOp).then((results) => {
// 第一次访问，都没计数，直接过
if (!result && !_.isNumber(result) && !result10m && !_.isNumber(result10m)) {
return Promise.resolve(true);
}
logger.debug(MAX_TIMES);
logger.debug(_.values(ruleKeys));
logger.debug(results);
if (result === -1 || result10m === -1) {
return Promise.resolve(true);
}
// 遍历限制规则，若满足返回相应处理策略， 否则页面访问次数加1
let operation = [];
// 判断 qps 10分钟
if (result10m === 9999) {
res.statusCode = 403;
return Promise.resolve(policy);
} else if (result10m > MAX_QPS_10m) { // eslint-disable-line
cache.setAsync(key10mMax, 1, ONE_DAY);
logger.debug('req limit', key10m);
_.forEach(MAX_TIMES, (val, key) => {
let cacheKey = ruleKeys[key];
return Promise.resolve(policy);
}
if (!results[key]) {
operation.push(cache.setAsync(cacheKey, 1, +key));
} else if (+results[key] > +val) {
// 判断 qps 1分钟
if (result === 9999) {
res.statusCode = 403;
return Promise.resolve(policy);
} else if (result > MAX_QPS) { // 判断 qps
cache.setAsync(keyMax, 1, ONE_DAY);
logger.debug('req limit', key);
// ip限制1小时
operation.push(cache.setAsync(`${config.app}:limiter:${limiter.remoteIp}`, 1, limiterIpTime));
return Promise.resolve(policy);
} else {
operation.push(cache.incrAsync(cacheKey, 1));
}
});
return Promise.resolve(policy);
}
Promise.all(operation);
cache.incrAsync(key, 1); // qps + 1
cache.incrAsync(key10m, 1); // qps + 1
// 不满足任何限制规则，继续访问
return Promise.resolve(true);
}).catch(err=>{
logger.error(err);
});
};
... ...