fe / yohobuywap-node · Commits

Go to a project

Authored by 郭成尧
Commit 87c2d7805afabd1fcf018286f4e42aa986399023 2 parents 3167b82b f56be33f

Merge remote-tracking branch 'origin/master' into hotfix/httponly

Inline Side-by-side
Showing 15 changed files with 324 additions and 104 deletions
... ... @@ -44,7 +44,7 @@ const logger = global.yoho.logger;
// zookeeper
if (config.zookeeperServer) {
require('yoho-zookeeper')(config.zookeeperServer, 'wap', app.locals.wap = {}, global.yoho.cache);
require('yoho-zookeeper')(config.zookeeperServer, 'wap', app.locals.wap = {});
}
// 访问域名层级
... ... @@ -149,6 +149,7 @@ try {
const pageCache = require('./doraemon/middleware/page-cache');
const downloadBar = require('./doraemon/middleware/download-bar');
const routeEncode = require('./doraemon/middleware/route-encode');
const riskManagement = require('./doraemon/middleware/risk-management');
// YOHO 前置中间件
app.use(tdkUrl());
... ... @@ -169,6 +170,7 @@ try {
app.use(pageCache());
app.use(routeEncode.md);
app.use(downloadBar());
app.use(riskManagement());
require('./dispatch')(app);
app.all('*', errorHanlder.notFound()); // 404
... ...
'use strict';
const _ = require('lodash');
const decodeURIComponent = require('../../../utils/string-process').decodeURIComponent;
const logger = global.yoho.logger;
const Geetest = require('geetest');
const config = global.yoho.config;
const co = Promise.coroutine;
const cache = global.yoho.cache.master;
const captcha = new Geetest({
... ... @@ -65,13 +67,19 @@ exports.submit = (req, res) => {
remoteIp = arr[0];
}
let key = `pc:limiter:${remoteIp}`;
let key10m = `pc:limiter:10m:${remoteIp}`;
let operations = [cache.delAsync(`${config.app}:limiter:${remoteIp}`)];
yield Promise.all([
cache.delAsync(key),
cache.delAsync(key10m)
]);
if (req.body.pid) {
let riskPid = decodeURIComponent(req.body.pid) + ':' + _.get(req.yoho, 'clientIp', '');
operations.push(cache.delAsync(riskPid));
}
_.forEach(config.REQUEST_LIMIT, (val, key) => {
operations.push(cache.delAsync(`${config.app}:limiter:${key}:max:${remoteIp}`));
});
yield Promise.all(operations);
return res.json({
code: 200
... ...
... ... @@ -126,8 +126,6 @@ module.exports = {
appSecret: 'ce21ae4a3f93852279175a167e54509b',
notifyUrl: domains.service + 'payment/weixin_notify',
},
maxQps: 1200,
maxQps10m: 2500,
geetestJs: '//static.geetest.com/static/tools/gt.js',
jsSdk: '//cdn.yoho.cn/js-sdk/1.3.0/jssdk.js',
redis: {
... ... @@ -149,6 +147,19 @@ module.exports = {
}
}
},
REQUEST_LIMIT: {
// 10s 最多访问5次
10: 10,
// 30s 最多访问15次
30: 20,
// 60s 最多访问15次
60: 30,
// 100s 最多访问15次
600: 100
},
superCapture: '93c70db61fe276f93ce781ad17dc47cd',
from: from
};
... ...
... ... @@ -5,15 +5,33 @@ const logger = global.yoho.logger;
const ip = require('./rules/ip-list');
const userAgent = require('./rules/useragent');
const qpsLimiter = require('./rules/qps-limit');
// const asynchronous = require('./rules/asynchronous');
// const fakerLimiter = require('./rules/faker-limit');
const captchaPolicy = require('./policies/captcha');
// const reporterPolicy = require('./policies/reporter');
const IP_WHITE_LIST = [
'106.38.38.146',
'106.38.38.147',
'106.39.86.227',
'218.94.75.58',
'218.94.75.50',
'218.94.77.166'
];
const PATH_WHITE_LIST = [
'/3party/check',
'/3party/check/submit',
'/passport/imagesNode',
'/passport/cert/headerTip',
'/passport/captcha/get',
'/passport/images',
'/passport/img-check.jpg',
'/passport/geetest/register'
];
const limiter = (rule, policy, context) => {
return rule(context, policy);
};
... ... @@ -33,8 +51,12 @@ module.exports = (req, res, next) => {
remoteIp = req.get('X-Real-IP');
}
const excluded = _.includes(IP_WHITE_LIST, remoteIp);
const enabled = !_.get(req.app.locals, 'wap.sys.noLimiter');
// 排除条件：ip白名单/路径白名单/异步请求/登录用户
const excluded = _.includes(IP_WHITE_LIST, remoteIp) ||
_.includes(PATH_WHITE_LIST, req.path) || req.xhr || !_.isEmpty(_.get(req, 'user.uid'));
const enabled = !_.get(req.app.locals, 'wap.sys.noLimite');
logger.debug(`request remote ip: ${remoteIp}; excluded: ${excluded}; enabled: ${enabled}`);
// 判断获取remoteIp成功，并且开关未关闭
if (enabled && remoteIp && !excluded) {
... ... @@ -48,8 +70,9 @@ module.exports = (req, res, next) => {
Promise.all([
limiter(userAgent, captchaPolicy, context),
limiter(ip, captchaPolicy, context),
limiter(qpsLimiter, captchaPolicy, context),
limiter(qpsLimiter, captchaPolicy, context)
// limiter(asynchronous, captchaPolicy, context)
// limiter(fakerLimiter, reporterPolicy, context)
]).then((results) => {
let allPass = true, exclusion = false, policy = null;
... ... @@ -61,11 +84,8 @@ module.exports = (req, res, next) => {
exclusion = result.exclusion;
}
if (!excluded && typeof result === 'function') {
allPass = false;
}
if (typeof result === 'function') {
allPass = false;
policy = result;
}
});
... ...
... ... @@ -2,7 +2,6 @@
const helpers = global.yoho.helpers;
const _ = require('lodash');
const WHITE_LIST = [
'/3party/check',
'/3party/check/submit',
... ... @@ -23,7 +22,7 @@ module.exports = (req, res, next) => {
return next();
}
if (_.toNumber(res.statusCode) === 403) {
if (res.statusCode === 403) {
return res.end();
}
... ...
'use strict';
const cache = global.yoho.cache.master;
const _ = require('lodash');
const logger = global.yoho.logger;
const ASYNCHRONOUSPAGES = {
'/product/index/isFavoriteShop': 1,
'/common/suggestfeedback': 1,
'/product/detail/hotarea': 1,
'/common/getbanner': 1,
'/passport/cert/headerTip': 1
};
function isNormalSpider(userAgent) {
let normalReg = /(spider)|(bot.html)/i;
if (normalReg.test(userAgent)) {
return true;
} else {
return false;
}
}
module.exports = (limiter, policy) => {
const ua = limiter.req.header('User-Agent');
const synchronizeKey = `wap:limiter:synchronize:${limiter.remoteIp}`; // 同步
const asynchronousKey = `wap:limiter:asynchronous:${limiter.remoteIp}`; // 异步
const spiderKey = `wap:limiter:spider:${limiter.remoteIp}`; // 异步
// 正常蜘蛛直接过
if (isNormalSpider(ua)) {
return Promise.resolve(true);
}
const req = limiter.req,
res = limiter.res;
res.on('render', function() {
cache.incrAsync(synchronizeKey, 1).catch(e=>console.log(e)); // eslint-disable-line
});
return cache.getMultiAsync([synchronizeKey, asynchronousKey, spiderKey]).then((results) => {
logger.debug(results);
if (results[spiderKey]) {
logger.info(results);
return Promise.resolve(policy);
}
// 默认数据设置
if (!results[synchronizeKey] && !_.isNumber(results[synchronizeKey])) {
cache.setAsync(synchronizeKey, 1, 600);
}
// 默认数据设置
if (ASYNCHRONOUSPAGES[req.path] > 0) {
cache.setAsync(asynchronousKey, 1, 600);
cache.setAsync(synchronizeKey, 1, 600);
}
if (results[synchronizeKey] > 10 && !results[asynchronousKey]) {
cache.setAsync(spiderKey, 1, 60 * 60 * 24);
logger.info(results);
return Promise.resolve(policy);
}
return Promise.resolve(true);
});
};
... ...
'use strict';
const logger = global.yoho.logger; // eslint-disable-line
const cache = global.yoho.cache.master;
const ONE_DAY = 60 * 60 * 24;
module.exports = (limiter, policy) => {
const req = limiter.req,
res = limiter.res;
res = limiter.res,
next = limiter.next; // eslint-disable-line
const key = `pc:limiter:faker:${limiter.remoteIp}`;
const key = `wap:limiter:faker:${limiter.remoteIp}`;
if (req.header('X-Requested-With') === 'XMLHttpRequest') {
cache.decrAsync(key, 1);
... ... @@ -20,7 +22,7 @@ module.exports = (limiter, policy) => {
return cache.getAsync(key).then((result) => {
if (result) {
if (result > 100) {
return Promise.resolve(policy);// policy(req, res, next);
return Promise.resolve(policy); // policy(req, res, next);
} else {
return Promise.resolve(true);
}
... ...
... ... @@ -2,15 +2,17 @@
const cache = global.yoho.cache.master;
const _ = require('lodash');
const config = global.yoho.config;
const logger = global.yoho.logger;
module.exports = (limiter) => {
const key = `pc:limiter:${limiter.remoteIp}`;
module.exports = (limiter, policy) => {
const key = `${config.app}:limiter:${limiter.remoteIp}`;
return cache.getAsync(key).then((result) => {
logger.debug(key, result);
if (result && _.isNumber(result)) {
return Promise.resolve({
exclusion: result === -1
});
return Promise.resolve(policy);
} else {
return Promise.resolve(true);
}
... ...
/**
* 限制页面访问次数，如超过限制次数，返回相应策略（目前是ip加入黑名单，跳转图形验证码页面，解除限制）
* 当前规则只针对未登录用户
*/
'use strict';
const logger = global.yoho.logger;
const cache = global.yoho.cache.master;
const config = global.yoho.config;
const ONE_DAY = 60 * 60 * 24;
const MAX_QPS = config.maxQps;
const MAX_QPS_10m = config.maxQps10m;
const _ = require('lodash');
const PAGES = {
'/product/^\\/(\\d+)\\.html/': 5,
'/product/list/index': 5,
'/product/index/index': 5,
'/product/search/list': 5
};
// 超出访问限制ip限制访问1小时
const limiterIpTime = 3600;
function urlJoin(a, b) {
if (_.endsWith(a, '/') && _.startsWith(b, '/')) {
return a + b.substring(1, b.length);
} else if (!_.endsWith(a, '/') && !_.startsWith(b, '/')) {
return a + '/' + b;
} else {
return a + b;
}
}
// 页面访问限制
const MAX_TIMES = config.REQUEST_LIMIT;
module.exports = (limiter, policy) => {
const req = limiter.req,
res = limiter.res;
const key = `pc:limiter:${limiter.remoteIp}`;
const key10m = `pc:limiter:10m:${limiter.remoteIp}`;
res.on('render', function() {
let route = req.route ? req.route.path : '';
let appPath = req.app.mountpath;
if (_.isArray(route) && route.length > 0) {
route = route[0];
}
// 存储规则的cache keys
let ruleKeys = {};
let getOp = {};
let pageKey = urlJoin(appPath, route.toString()); // route may be a regexp
let pageIncr = PAGES[pageKey] || 0;
if (pageIncr > 0) {
cache.incrAsync(key, pageIncr);
cache.incrAsync(key10m, pageIncr);
}
_.forEach(MAX_TIMES, (val, key) => {
ruleKeys[key] = `${config.app}:limiter:${key}:max:${limiter.remoteIp}`;
getOp[key] = cache.getAsync(ruleKeys[key]);
});
return cache.getMultiAsync([key, key10m]).then((results) => {
let result = results[key];
let result10m = results[key10m];
logger.debug('qps limiter: ' + key + '@' + result + ' max: ' + MAX_QPS);
logger.debug('qps limiter 10m: ' + key10m + '@' + result10m + ' max: ' + MAX_QPS_10m);
// 默认数据设置
if (!result && !_.isNumber(result)) {
cache.setAsync(key, 1, 60); // 设置key，1m失效
}
if (!result10m && !_.isNumber(result10m)) {
cache.setAsync(key10m, 1, 600); // 设置key，10m失效
}
return Promise.props(getOp).then((results) => {
// 第一次访问，都没计数，直接过
if (!result && !_.isNumber(result) && !result10m && !_.isNumber(result10m)) {
return Promise.resolve(true);
}
logger.debug(MAX_TIMES);
logger.debug(_.values(ruleKeys));
logger.debug(results);
if (result === -1 || result10m === -1) {
return Promise.resolve(true);
}
// 遍历限制规则，若满足返回相应处理策略， 否则页面访问次数加1
let operation = [];
// 判断 qps 10分钟
if (result10m > MAX_QPS_10m) {
cache.touch(key10m, ONE_DAY);
logger.debug('req limit', key10m);
_.forEach(MAX_TIMES, (val, key) => {
let cacheKey = ruleKeys[key];
return Promise.resolve(policy);
}
if (!results[key]) {
operation.push(cache.setAsync(cacheKey, 1, +key));
} else if (+results[key] > +val) {
// 判断 qps 1分钟
if (result > MAX_QPS) {
cache.touch(key, ONE_DAY);
logger.debug('req limit', key);
// ip限制1小时
operation.push(cache.setAsync(`${config.app}:limiter:${limiter.remoteIp}`, 1, limiterIpTime));
return Promise.resolve(policy);
} else {
operation.push(cache.incrAsync(cacheKey, 1));
}
});
return Promise.resolve(policy);
}
Promise.all(operation);
cache.incrAsync(key, 1); // qps + 1
cache.incrAsync(key10m, 1); // qps + 1
// 不满足任何限制规则，继续访问
return Promise.resolve(true);
}).catch(err=>{
logger.error(err);
});
};
... ...
... ... @@ -6,8 +6,8 @@ const logger = global.yoho.logger;
module.exports = (limiter, policy) => {
const blackKey = 'pc:limiter:ua:black',
whiteKey = 'pc:limiter:ua:white';
const blackKey = 'wap:limiter:ua:black',
whiteKey = 'wap:limiter:ua:white';
const ua = limiter.req.header('User-Agent');
... ... @@ -15,7 +15,8 @@ module.exports = (limiter, policy) => {
cache.getAsync(blackKey),
cache.getAsync(whiteKey)
]).then((args) => {
const blacklist = args[0] || [], whitelist = args[1] || [];
const blacklist = args[0] || [],
whitelist = args[1] || [];
if (blacklist.length === 0 && whitelist.length === 0) {
return Promise.resolve(true);
... ...
/**
* 控制路由请求次数
* @date: 2018/03/05
*/
'use strict';
const _ = require('lodash');
const cache = global.yoho.cache.master;
const helpers = global.yoho.helpers;
const pathToRegexp = require('path-to-regexp');
const logger = global.yoho.logger;
const statusCode = {
code: 4403,
data: {},
message: '亲，您的访问次数过多，请稍后再试哦...'
};
const INVALIDTIME = 3600 * 24; // 24h
const IP_WHITE_LIST = [
'106.38.38.146',
'106.38.38.147',
'106.39.86.227',
'218.94.75.58',
'218.94.75.50',
'218.94.77.166'
];
const _jumpUrl = (req, res, next, result) => {
if (result.code === 4403) {
if (req.xhr) {
res.set({
'Cache-Control': 'no-cache',
Pragma: 'no-cache',
Expires: (new Date(1900, 0, 1, 0, 0, 0, 0)).toUTCString()
});
return res.status(403).json(result);
}
return res.redirect(`${result.data.url}&refer=${req.originalUrl}`);
}
return next();
};
module.exports = () => {
return (req, res, next) => {
// default open
if (_.get(req.app.locals.wap, 'close.risk', false)) {
return next();
}
let ip = _.get(req.yoho, 'clientIp', '');
let path = req.path || '';
let risks = _.get(req.app.locals.wap, 'json.risk', []);
let router = {};
logger.debug(`risk => risks: ${JSON.stringify(risks)}, path: ${path}, ip: ${ip}`); // eslint-disable-line
if (_.isEmpty(path) || _.isEmpty(risks) || IP_WHITE_LIST.indexOf(ip) > -1) {
return next();
}
_.isArray(risks) && risks.some(item => {
if (item.state === 'off') {
return false;
}
if (!item.regRoute) {
item.regRoute = pathToRegexp(item.route);
item.interval = parseInt(item.interval, 10);
item.requests = parseInt(item.requests, 10);
}
if (item.regRoute.test(path)) {
router = item;
return true;
}
return false;
});
logger.debug(`risk => router: ${JSON.stringify(router)}, path: ${path}`); // eslint-disable-line
if (_.isEmpty(router)) {
return next();
}
let keyPath = `${_.trim(path, '/').replace(/\//g, ':')}`;
let limitKey = `wap:risk:limit:${keyPath}:${ip}`;
let configKey = `wap:risk:${keyPath}:${ip}`;
let checkUrl = helpers.urlFormat('/3party/check', {
pid: `wap:risk:limit:${keyPath}`
});
return Promise.all([
cache.getAsync(limitKey),
cache.getAsync(configKey),
]).then(inters => {
logger.debug(`risk => getCache: ${JSON.stringify(inters)}, path: ${path}`); // eslint-disable-line
if (inters[0]) {
return Object.assign({}, statusCode, {data: {url: checkUrl}});
}
if (typeof inters[1] === 'undefined') {
cache.setAsync(configKey, 1, router.interval || 300);
return Object.assign({}, statusCode, {code: 200, message: ''});
}
inters[1] = parseInt(`0${inters[1]}`, 10);
if (inters[1] <= router.requests) {
router = [];
cache.incrAsync(configKey, 1);
return Object.assign({}, statusCode, {code: 200, message: ''});
}
return Promise.all([
cache.setAsync(limitKey, 1, INVALIDTIME),
cache.delAsync(configKey)
]).then(() => {
return Object.assign({}, statusCode, {data: {url: checkUrl}});
});
}).then(result => {
logger.debug(`risk => result: ${JSON.stringify(result)}, path: ${path}`); // eslint-disable-line
return _jumpUrl(req, res, next, result);
}).catch(e => {
console.log(`risk => path: ${path}, err: ${e.message}`);
return next();
});
};
};
... ...
{
"name": "yohobuywap-node",
"version": "6.5.20",
"version": "6.5.21",
"private": true,
"description": "A New Yohobuy Project With Express",
"repository": {
... ... @@ -63,6 +63,7 @@
"passport-sina": "^0.1.0",
"passport-strategy": "^1.0.0",
"passport-weixin": "^0.2.0",
"path-to-regexp": "^2.1.0",
"redis": "^2.8.0",
"request": "^2.81.0",
"request-promise": "^4.2.1",
... ... @@ -74,7 +75,7 @@
"yoho-express-session": "^2.0.0",
"yoho-md5": "^2.0.0",
"yoho-node-lib": "=0.6.2",
"yoho-zookeeper": "^1.0.8"
"yoho-zookeeper": "^1.0.9"
},
"devDependencies": {
"@mapbox/stylelint-processor-arbitrary-tags": "^0.2.0",
... ...
... ... @@ -16,6 +16,8 @@ validate.init();
$(function() {
$('.submit').on('click', function() {
validate.getResults().then((result) => {
$.extend(result, {pid: window.queryString.pid});
$.ajax({
method: 'POST',
url: '/3party/check/submit',
... ...
... ... @@ -49,5 +49,10 @@ $(document).ajaxError((event, xhr) => {
window.location.href = `/signin.html?refer=${encodeURIComponent(window.location.href)}`;
}
}
} else if (xhr.status === 403 && xhr.responseJSON.code === 4403) {
tip.show(xhr.responseJSON.message);
setTimeout(function() {
window.location.href = `${xhr.responseJSON.data.url}&refer=${encodeURIComponent(window.location.href)}`;
}, 2000);
}
});
... ...
... ... @@ -8,6 +8,7 @@
.status-c {
float: left;
width: 640px;
position: relative;
}
.status-title {
... ...