OPENTECH / yoho-node-ci · Commits

Go to a project

Authored by 徐炜
Commit c68c8d0fafb95d3c6d6c70dabc67f88d0842390b 1 parent da8e2f9b

滥用防护

Inline Side-by-side
Showing 5 changed files with 249 additions and 17 deletions
... ... @@ -11,6 +11,18 @@ class ApiCache {
this.memcached = new Memcached(host);
}
setKey(key, value, ttl) {
this._log(`setting ${key}`);
this.memcached.set(key, value, ttl, (err) => {
if (err) {
this._log(`set ${key} fail`)
} else {
this._log(`set ${key} success`)
}
});
}
delKey(key) {
this._log(`deleting ${key}`)
... ... @@ -34,8 +46,73 @@ class ApiCache {
});
}
find(condition) {
let count = 0;
return new Promise((resolve, reject) => {
this.memcached.items((err, result) => {
if (err) console.error(err);
if (result.length === 0) {
this._log('empty items')
}
// for each server...
result.forEach(itemSet => {
var keys = Object.keys(itemSet);
keys.pop(); // we don't need the "server" key, but the other indicate the slab id's
var len = keys.length;
if (keys.length === 0) {
this._log('empty item set');
}
const keyList = [];
keys.forEach(stats => {
// get a cachedump for each slabid and slab.number
this.memcached.cachedump(itemSet.server, parseInt(stats, 10), itemSet[stats].number, (err, response) => {
if (response) {
if (_.isArray(response)) {
_.each(response, (item) => {
count++;
if (condition(item.key)) {
keyList.push(item.key);
}
});
}
else {
count++;
if (condition(response.key)) {
keyList.push(response.key);
}
}
}
if (--len === 0) {
this.memcached.getMulti(keyList, (err, data) => {
this.memcached.end();
if (err) {
reject(err);
}
resolve(data);
});
}
});
})
})
});
});
}
stats() {
this.memcached.stats( (err, stats) => {
this.memcached.stats((err, stats) => {
if (err) {
} else {
... ... @@ -43,7 +120,7 @@ class ApiCache {
stats = stats[0];
}
stats.hits_percent = (stats.get_hits / (stats.get_hits + stats.get_misses) * 100).toFixed(2) + '%';
stats.hits_percent = (stats.get_hits / (stats.get_hits + stats.get_misses) * 100).toFixed(2) + '%';
stats.heap_info = (stats.bytes / 1024 / 1024).toFixed(4) + 'mb' + ' / ' + stats.limit_maxbytes / 1024 / 1024 + 'mb';
ws.broadcast(`/api_cache/monit`, {
... ... @@ -55,7 +132,7 @@ class ApiCache {
}
restart() {
this.memcached.stats( (err, stats) => {
this.memcached.stats((err, stats) => {
if (err) {
} else {
... ... @@ -73,7 +150,7 @@ class ApiCache {
conn.on('ready', async() => {
try {
conn.exec(`sudo kill ${pid}`, (err, stream) => {
if(err) {
if (err) {
console.log(err);
} else {
stream.on('close', () => {
... ... @@ -110,8 +187,8 @@ class ApiCache {
let count = 0;
this.memcached.items( ( err, result ) => {
if( err ) console.error( err );
this.memcached.items((err, result) => {
if (err) console.error(err);
if (result.length === 0) {
this._log('empty items')
... ... @@ -119,25 +196,25 @@ class ApiCache {
// for each server...
result.forEach(itemSet => {
var keys = Object.keys( itemSet );
keys.pop(); // we don't need the "server" key, but the other indicate the slab id's
var keys = Object.keys(itemSet);
keys.pop(); // we don't need the "server" key, but the other indicate the slab id's
var len = keys.length;
if (keys.length === 0) {
this._log('empty item set');
}
keys.forEach(stats => {
// get a cachedump for each slabid and slab.number
this.memcached.cachedump( itemSet.server, parseInt(stats, 10), itemSet[stats].number, ( err, response ) => {
this.memcached.cachedump(itemSet.server, parseInt(stats, 10), itemSet[stats].number, (err, response) => {
// dump the shizzle
if (response) {
if (_.isArray(response)) {
_.each(response, keyObj => {
count ++ ;
count++;
if (keyObj.key && (keyObj.key.indexOf(key1) >= 0 || keyObj.key.indexOf(key2) >= 0 || keyObj.key.indexOf(key) >= 0)) {
this.delKey(keyObj.key);
} else {
... ... @@ -145,7 +222,7 @@ class ApiCache {
}
});
} else {
count ++;
count++;
if (response.key && (response.key.indexOf(key1) >= 0 || response.key.indexOf(key2) >= 0 || response.key.indexOf(key) >= 0)) {
this.delKey(response.key);
} else {
... ... @@ -154,7 +231,7 @@ class ApiCache {
}
}
len --;
len--;
if (len === 0) {
this.memcached.end();
... ...
'use strict';
const Router = require('koa-router');
const ApiCache = require('../../ci/api_cache');
const _ = require('lodash');
const {
MemcachedHost
} = require('../../models');
let r = new Router();
const defensive = {
index: async(ctx, next) => {
const regexp = /pc:limiter:faker:(.*)/;
const threshold = ctx.request.query.threshold || 100;
const limit = ctx.request.query.limit || 10;
let hosts = await MemcachedHost.findAll();
let results = await Promise.all(_.map(hosts, (h) => {
return (new ApiCache(h.host)).find((key) => {
return regexp.test(key);
});
}));
let list = [];
if (results && results[0]) {
Object.keys(results[0]).forEach((key) => {
const index = results[0][key];
if (index > threshold) {
list.push({
ip: ((key) => {
const m = key.match(regexp);
return m && m.length > 0 ? m[1] : 'Unknown';
})(key),
index: index
})
}
});
}
list = _.orderBy(list, (item) => {
return item.index;
}, 'desc').slice(0, limit);
await ctx.render('action/abuse_protection', {
list: list,
threshold: threshold,
limit: limit
});
},
lock: async(ctx, next) => {
let hosts = await MemcachedHost.findAll();
await Promise.all(_.map(hosts, (h) => {
const key = `pc:limiter:${ctx.request.body.remoteIp}`,
value = 9999,
ttl = 60 * 60 * 8; // 封停8小时
return (new ApiCache(h.host)).setKey(key, value, ttl);
}));
return ctx.body = {
code: 200
};
},
unlock: async(ctx, next) => {
let hosts = await MemcachedHost.findAll();
await Promise.all(_.map(hosts, (h) => {
const key = `pc:limiter:${ctx.request.body.remoteIp}`;
return (new ApiCache(h.host)).delKey(key);
}));
return ctx.body = {
code: 200
};
}
};
r.get('/abuse_protection', defensive.index);
r.post('/lock', defensive.lock);
r.post('/unlock', defensive.unlock);
module.exports = r;
\ No newline at end of file
... ...
... ... @@ -17,6 +17,7 @@ const apiCache = require('./actions/api_cache');
const degrade = require('./actions/degrade');
const deploy = require('./actions/deploy');
const api = require('./actions/api');
const abuseProtection = require('./actions/abuse_protection');
const noAuth = new Router();
const base = new Router();
... ... @@ -49,8 +50,7 @@ module.exports = function (app) {
base.use('/api_cache', apiCache.routes(), apiCache.allowedMethods());
base.use('/degrade', degrade.routes(), degrade.allowedMethods());
base.use('/deploys', deploy.routes(), deploy.allowedMethods());
base.use('/abuse_protection', abuseProtection.routes(), degrade.allowedMethods());
base.use('', index.routes(), index.allowedMethods());
... ...
<div class="pageheader">
<div class="media">
<div class="pageicon pull-left">
<i class="fa fa-th-list"></i>
</div>
<div class="media-body">
<ul class="breadcrumb">
<li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
<li><a href="/abuse_protection">滥用防护</a></li>
</ul>
<h4>疑似爬虫监控</h4>
</div>
</div>
<!-- media -->
</div>
<!-- pageheader -->
<div class="contentpanel servers-page">
<form action="/abuse_protection/abuse_protection" class="form-inline" method="get">
<div class="form-group input-group">
<input id="threshold" type="text" class="form-control" name="threshold" value="{{threshold}}"/>
<div class="input-group-addon">/天</div>
</div>
<div class="form-group input-group">
<input id="threshold" type="text" class="form-control" name="limit" value="{{limit}}"/>
<div class="input-group-addon">限制</div>
</div>
<button class="btn btn-primary">刷新</button>
</form>
<table class="table table-striped table-bordered building-table" style="margin-top: 2rem">
<tr>
<th>IP</th>
<th>爬虫指数</th>
<th>操作</th>
</tr>
{{#list}}
<tr>
<td>{{ip}}</td>
<td>{{index}}</td>
<td>
<button class="access-deny btn btn-primary btn-sm btn-danger" data-ip="{{ip}}">禁止访问</button>
<button class="access-allow btn btn-success btn-sm" data-ip="{{ip}}">允许访问</button>
</td>
</tr>
{{/list}}
</table>
</div>
<script>
$(function() {
$('.access-deny').click(function() {
$.post('/abuse_protection/lock', {remoteIp: $(this).data('ip')}, function() {
});
});
$('.access-allow').click(function() {
$.post('/abuse_protection/unlock', {remoteIp: $(this).data('ip')}, function() {
});
});
});
</script>
\ No newline at end of file
... ...
... ... @@ -43,6 +43,9 @@
</li>
{{/if}}
<li><a href="/degrade"><i class="fa fa-hand-o-down"></i> <span>降级配置</span></a></li>
<li><a href="/abuse_protection/abuse_protection"><i class="fa fa-shield"></i> <span>滥用防护</span></a></li>
</ul>
</div>
\ No newline at end of file
... ...