Authored by 周奇琪

Merge branch 'master' of git.yoho.cn:OPENTECH/yoho-node-ci

Conflicts:
	apps/web/views/partials/common/menu.hbs
... ... @@ -37,6 +37,12 @@ app.use(convert(body({
multipart: true,
queryString: {
plainObjects: true
},
jsonLimit: '10mb',
formLimit: '10mb',
textLimit: '10mb',
formidable: {
maxFieldsSize: 10 * 1024 * 1024
}
})));
app.use(mount('/', webApp));
... ...
... ... @@ -11,6 +11,18 @@ class ApiCache {
this.memcached = new Memcached(host);
}
setKey(key, value, ttl) {
this._log(`setting ${key}`);
this.memcached.set(key, value, ttl, (err) => {
if (err) {
this._log(`set ${key} fail`)
} else {
this._log(`set ${key} success`)
}
});
}
delKey(key) {
this._log(`deleting ${key}`)
... ... @@ -34,8 +46,75 @@ class ApiCache {
});
}
find(condition) {
let count = 0;
return new Promise((resolve, reject) => {
this.memcached.items((err, result) => {
if (err) console.error(err);
if (result.length === 0) {
this._log('empty items');
resolve([]);
}
// for each server...
result.forEach(itemSet => {
var keys = Object.keys(itemSet);
keys.pop(); // we don't need the "server" key, but the other indicate the slab id's
var len = keys.length;
if (keys.length === 0) {
this._log('empty item set');
resolve([]);
}
const keyList = [];
keys.forEach(stats => {
// get a cachedump for each slabid and slab.number
this.memcached.cachedump(itemSet.server, parseInt(stats, 10), itemSet[stats].number, (err, response) => {
if (response) {
if (_.isArray(response)) {
_.each(response, (item) => {
count++;
if (condition(item.key)) {
keyList.push(item.key);
}
});
}
else {
count++;
if (condition(response.key)) {
keyList.push(response.key);
}
}
}
if (--len === 0) {
this.memcached.getMulti(keyList, (err, data) => {
this.memcached.end();
if (err) {
reject(err);
}
resolve(data);
});
}
});
})
})
});
});
}
stats() {
this.memcached.stats( (err, stats) => {
this.memcached.stats((err, stats) => {
if (err) {
} else {
... ... @@ -43,7 +122,7 @@ class ApiCache {
stats = stats[0];
}
stats.hits_percent = (stats.get_hits / (stats.get_hits + stats.get_misses) * 100).toFixed(2) + '%';
stats.hits_percent = (stats.get_hits / (stats.get_hits + stats.get_misses) * 100).toFixed(2) + '%';
stats.heap_info = (stats.bytes / 1024 / 1024).toFixed(4) + 'mb' + ' / ' + stats.limit_maxbytes / 1024 / 1024 + 'mb';
ws.broadcast(`/api_cache/monit`, {
... ... @@ -55,7 +134,7 @@ class ApiCache {
}
restart() {
this.memcached.stats( (err, stats) => {
this.memcached.stats((err, stats) => {
if (err) {
} else {
... ... @@ -73,7 +152,7 @@ class ApiCache {
conn.on('ready', async() => {
try {
conn.exec(`sudo kill ${pid}`, (err, stream) => {
if(err) {
if (err) {
console.log(err);
} else {
stream.on('close', () => {
... ... @@ -110,8 +189,8 @@ class ApiCache {
let count = 0;
this.memcached.items( ( err, result ) => {
if( err ) console.error( err );
this.memcached.items((err, result) => {
if (err) console.error(err);
if (result.length === 0) {
this._log('empty items')
... ... @@ -119,25 +198,25 @@ class ApiCache {
// for each server...
result.forEach(itemSet => {
var keys = Object.keys( itemSet );
keys.pop(); // we don't need the "server" key, but the other indicate the slab id's
var keys = Object.keys(itemSet);
keys.pop(); // we don't need the "server" key, but the other indicate the slab id's
var len = keys.length;
if (keys.length === 0) {
this._log('empty item set');
}
keys.forEach(stats => {
// get a cachedump for each slabid and slab.number
this.memcached.cachedump( itemSet.server, parseInt(stats, 10), itemSet[stats].number, ( err, response ) => {
this.memcached.cachedump(itemSet.server, parseInt(stats, 10), itemSet[stats].number, (err, response) => {
// dump the shizzle
if (response) {
if (_.isArray(response)) {
_.each(response, keyObj => {
count ++ ;
count++;
if (keyObj.key && (keyObj.key.indexOf(key1) >= 0 || keyObj.key.indexOf(key2) >= 0 || keyObj.key.indexOf(key) >= 0)) {
this.delKey(keyObj.key);
} else {
... ... @@ -145,7 +224,7 @@ class ApiCache {
}
});
} else {
count ++;
count++;
if (response.key && (response.key.indexOf(key1) >= 0 || response.key.indexOf(key2) >= 0 || response.key.indexOf(key) >= 0)) {
this.delKey(response.key);
} else {
... ... @@ -154,7 +233,7 @@ class ApiCache {
}
}
len --;
len--;
if (len === 0) {
this.memcached.end();
... ...
... ... @@ -52,6 +52,26 @@ const defaultDegrades = [
path: '/pc/cart/removeMerge',
name: '【移除】购物车>>> 凑单商品'
},
{
path: '/pc/clientService/new',
name: '【开关】开启新客服系统'
},
{
path: '/pc/qcloud_cdn',
name: '【开关】启动腾讯云备份CDN'
},
{
path: '/pc/user/removeStudentIdentification',
name: '【移除】学生认证开关'
},
{
path: '/pc/sys/noLimiter',
name: '【开关】关闭请求限制'
},
{
path: '/pc/pay/oldCart',
name: '【开关】开启老版购物车'
},
//wap
{
path: '/wap/plustar/removeCollect',
... ... @@ -76,6 +96,30 @@ const defaultDegrades = [
{
path: '/wap/cart/removePrefer',
name: '【移除】购物车>>> 为您优选新品'
},
{
path: '/wap/clientService/new',
name: '【开关】开启新客服系统'
},
{
path: '/wap/qcloud_cdn',
name: '【开关】启动腾讯云备份CDN'
},
{
path: '/wap/user/removeStudentIdentification',
name: '【移除】学生认证开关'
},
{
path: '/wap/sys/noLimiter',
name: '【开关】关闭请求限制'
},
{
path: '/wap/pay/newCart',
name: '【开关】开启新版购物车'
},
{
path: '/wap/pay/newPay',
name: '【开关】开启新版支付'
}
];
... ...
'use strict';
const Router = require('koa-router');
const ApiCache = require('../../ci/api_cache');
const _ = require('lodash');
const {
MemcachedHost
} = require('../../models');
let r = new Router();
const defensive = {
index: async(ctx, next) => {
const regexp = /pc:limiter:faker:(.*)/;
const threshold = ctx.request.query.threshold || 100;
const limit = ctx.request.query.limit || 10;
let hosts = await MemcachedHost.findAll();
const selectedHosts = _.filter(hosts, host => {
const isCurrent = host.host === ctx.request.query.node;
if (isCurrent) {
host.isCurrent = true;
}
return isCurrent;
});
let results = await Promise.all(_.map(selectedHosts, (h) => {
return (new ApiCache(h.host)).find((key) => {
return regexp.test(key);
});
}));
let list = [];
if (results && results[0]) {
Object.keys(results[0]).forEach((key) => {
const index = results[0][key];
if (index > threshold) {
list.push({
ip: ((key) => {
const m = key.match(regexp);
return m && m.length > 0 ? m[1] : 'Unknown';
})(key),
index: index
})
}
});
}
list = _.orderBy(list, (item) => {
return item.index;
}, 'desc').slice(0, limit);
await ctx.render('action/abuse_protection', {
hosts: hosts,
list: list,
noData: list.length === 0,
threshold: threshold,
limit: limit
});
},
lock: async(ctx, next) => {
let hosts = await MemcachedHost.findAll();
await Promise.all(_.map(hosts, (h) => {
const key = `pc:limiter:${ctx.request.body.remoteIp}`,
value = 9999,
ttl = 60 * 60 * 8; // 封停8小时
return (new ApiCache(h.host)).setKey(key, value, ttl);
}));
return ctx.body = {
code: 200
};
},
unlock: async(ctx, next) => {
let hosts = await MemcachedHost.findAll();
await Promise.all(_.map(hosts, (h) => {
const key = `pc:limiter:${ctx.request.body.remoteIp}`;
return (new ApiCache(h.host)).delKey(key);
}));
return ctx.body = {
code: 200
};
}
};
r.get('/abuse_protection', defensive.index);
r.post('/lock', defensive.lock);
r.post('/unlock', defensive.unlock);
module.exports = r;
\ No newline at end of file
... ...
... ... @@ -26,6 +26,8 @@ module.exports = {
hosts = target;
}
hosts = hosts || [];
hosts.forEach((host) => {
if (!servers[host]) {
servers[host] = new Collect(host, p.name, p.cloud);
... ...
'use strict';
const Router = require('koa-router');
const {
MemcachedHost
} = require('../../models');
const Operation = require('../../logger/operation');
const {
Server
} = require('../../models');
const {DegradeServer} = require('../../models');
const zookeeper = require('node-zookeeper-client');
const tester = require('../../zookeeper/tester');
const getter = require('../../zookeeper/getter');
const Model = require('../../models/model');
const _ = require('lodash');
const ApiCache = require('../../ci/api_cache');
const envs = {
p1oduction: '线上环境',
preview: '灰度环境',
test: '测试环境'
};
class Store extends Model {
constructor() {
super('abuse_protection');
}
}
const store = new Store();
const makeServer = ((ipKey, uaKey, listName, black) => {
const r = new Router;
const servers = {
ua: async(ctx, next) => {
let list = await servers.getLists(uaKey);
await ctx.render('action/crawler', {
listName: listName,
list: (list ? list.map((item) => {
return {name: item}
}) : ''),
main_name: 'UA'
});
},
ip: async(ctx, next) => {
let list = await servers.getLists(ipKey);
await ctx.render('action/crawler', {
listName: listName,
list: (list ? list.map((item) => {
return {name: item}
}) : ''),
main_name: 'IP'
});
},
change_ua: async(ctx, next) => {
const doUpdate = async(ua) => {
console.log('include ua:' + ua);
let hosts = await MemcachedHost.findAll();
await Promise.all(_.map(hosts, (h) => {
const key = `pc:limiter:ua:${black ? 'black' : 'white'}`,
value = JSON.parse(ua || '[]');
return (new ApiCache(h.host)).setKey(key, value, 0);
}));
};
let result = await servers.setLists(ctx);
await doUpdate(ctx.query.val);
if (result) {
ctx.body = {
listName: listName,
code: 200,
message: 'update success'
};
} else {
ctx.body = {
listName: listName,
code: 500,
message: 'update fail,Please retry'
}
}
},
change_ip: async(ctx, next) => {
let oldList = await servers.getLists(ipKey);
const newList = JSON.parse(ctx.query.val || '[]');
const exclude = async(ip) => {
console.log('exclude:' + ip);
let hosts = await MemcachedHost.findAll();
await Promise.all(_.map(hosts, (h) => {
const key = `pc:limiter:${ip}`,
value = -1,
ttl = 0;
return (new ApiCache(h.host)).setKey(key, value, ttl);
}));
};
const lock = async(ip) => {
console.log('lock:' + ip);
let hosts = await MemcachedHost.findAll();
await Promise.all(_.map(hosts, (h) => {
const key = `pc:limiter:${ip}`,
value = 9999,
ttl = 60 * 60 * 8; // 封停8小时
return (new ApiCache(h.host)).setKey(key, value, ttl);
}));
};
const unlock = async(ip) => {
console.log('unlock:' + ip);
let hosts = await MemcachedHost.findAll();
await Promise.all(_.map(hosts, (h) => {
const key = `pc:limiter:${ip}`;
return (new ApiCache(h.host)).delKey(key);
}));
};
const unlockList = [];
_.each(oldList, (item) => {
if (_.indexOf(newList, item) < 0) {
unlockList.push(item);
}
});
_.each(newList, (ip) => {
if (black) {
lock(ip);
} else {
exclude(ip);
}
});
_.each(unlockList, (ip) => {
unlock(ip);
});
let result = await servers.setLists(ctx);
if (result) {
ctx.body = {
listName: listName,
code: 200,
message: 'update success'
};
} else {
ctx.body = {
code: 500,
message: 'update fail,Please retry'
}
}
},
getLists: async(path) => {
const result = await store.findOne({
path: path
});
return result && result.val ? JSON.parse(result.val) : [];
},
async setLists(ctx, type) {
let {path, val} = ctx.query;
const rec = await store.findOne({
path: path
});
if (rec) {
store.update({
path: path
}, {
$set: {
val: val
}
})
} else {
store.insert({
path: path,
val: val
})
}
}
};
r.get('/ua', servers.ua);
r.get('/ip', servers.ip);
r.get('/change_ua', servers.change_ua);
r.get('/change_ip', servers.change_ip);
return r;
});
module.exports = makeServer;
\ No newline at end of file
... ...
... ... @@ -8,27 +8,29 @@ const servers = require('./actions/servers');
const login = require('./actions/login');
const monitor = require('./actions/monitor');
const users = require('./actions/users');
const hotfix = require( './actions/hotfix');
const hotfix = require('./actions/hotfix');
const operationLog = require('./actions/operation_log');
const pageCache = require( './actions/page_cache');
const cdnCache = require( './actions/cdn_cache');
const productCache = require( './actions/product_cache');
const pageCache = require('./actions/page_cache');
const cdnCache = require('./actions/cdn_cache');
const productCache = require('./actions/product_cache');
const apiCache = require('./actions/api_cache');
const degrade = require('./actions/degrade');
const deploy = require('./actions/deploy');
const api = require('./actions/api');
const abuseProtection = require('./actions/abuse_protection');
const crawler = require('./actions/crawler');
const noAuth = new Router();
const base = new Router();
module.exports = function (app) {
module.exports = function(app) {
noAuth.use('', login.routes(), login.allowedMethods());
noAuth.use('', common.routes(), common.allowedMethods());
noAuth.use('/api', api.routes(), api.allowedMethods());
app.use(noAuth.routes(), noAuth.allowedMethods());
app.use(async (ctx, next) => {
app.use(async(ctx, next) => {
if (ctx.session && ctx.session.user) {
await next();
... ... @@ -41,7 +43,7 @@ module.exports = function (app) {
base.use('/servers', servers.routes(), servers.allowedMethods());
base.use('/monitor', monitor.routes(), monitor.allowedMethods());
base.use('/users', users.routes(), users.allowedMethods());
base.use('/hotfix', hotfix.routes(), hotfix.allowedMethods());
// base.use('/hotfix', hotfix.routes(), hotfix.allowedMethods());
base.use('/operation', operationLog.routes(), operationLog.allowedMethods());
base.use('/page_cache', pageCache.routes(), pageCache.allowedMethods());
base.use('/cdn_cache', cdnCache.routes(), cdnCache.allowedMethods());
... ... @@ -50,7 +52,13 @@ module.exports = function (app) {
base.use('/degrade', degrade.routes(), degrade.allowedMethods());
base.use('/deploys', deploy.routes(), deploy.allowedMethods());
const white = crawler('/crawler/ip_whitelists', '/crawler/ua_whitelists', '白名单', false);
const black = crawler('/crawler/ip_blacklists', '/crawler/ua_blacklists', '黑名单', true);
base.use('/crawler_white', white.routes(), white.allowedMethods());
base.use('/crawler_black', black.routes(), black.allowedMethods());
base.use('/abuse_protection', abuseProtection.routes(), degrade.allowedMethods());
base.use('', index.routes(), index.allowedMethods());
... ...
<div class="pageheader">
<div class="media">
<div class="pageicon pull-left">
<i class="fa fa-th-list"></i>
</div>
<div class="media-body">
<ul class="breadcrumb">
<li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
<li><a href="/abuse_protection">滥用防护</a></li>
</ul>
<h4>疑似爬虫监控</h4>
</div>
</div>
<!-- media -->
</div>
<!-- pageheader -->
<div class="contentpanel servers-page">
<form action="/abuse_protection/abuse_protection" class="form-inline" method="get">
<div class="form-group">
<select name="node" class="form-control" style="height: 40px">
<option value="">Memcached节点</option>
{{#hosts}}
<option value="{{host}}" {{#if isCurrent}}selected="selected"{{/if}}>{{host}}</option>
{{/hosts}}
</select>
</div>
<div class="form-group input-group">
<input id="threshold" type="text" class="form-control" size="3" name="threshold" value="{{threshold}}"/>
<div class="input-group-addon">/天</div>
</div>
<div class="form-group input-group">
<div class="input-group-addon">显示</div>
<input id="threshold" type="text" class="form-control" size="3" name="limit" value="{{limit}}"/>
<div class="input-group-addon"></div>
</div>
<button class="btn btn-primary">应用</button>
</form>
<table class="table table-striped table-bordered building-table" style="margin-top: 2rem">
<tr>
<th>IP</th>
<th>爬虫指数</th>
<th>操作</th>
</tr>
{{#list}}
<tr>
<td>{{ip}}</td>
<td>{{index}}</td>
<td>
<button class="access-deny btn btn-primary btn-sm btn-danger" data-ip="{{ip}}">禁止访问</button>
<button class="access-allow btn btn-success btn-sm" data-ip="{{ip}}">允许访问</button>
</td>
</tr>
{{/list}}
{{#if noData}}
<tr>
<td colspan="4" class="text-center">无数据</td>
</tr>
{{/if}}
</table>
</div>
<script>
$(function() {
$('.access-deny').click(function() {
$.post('/abuse_protection/lock', {remoteIp: $(this).data('ip')}, function() {
});
});
$('.access-allow').click(function() {
$.post('/abuse_protection/unlock', {remoteIp: $(this).data('ip')}, function() {
});
});
});
</script>
\ No newline at end of file
... ...
<div class="pageheader">
<div class="media">
<div class="pageicon pull-left">
<i class="fa fa-th-list"></i>
</div>
<div class="media-body">
<ul class="breadcrumb">
<li><a href=""><i class="glyphicon glyphicon-home"></i></a></li>
<li><a href="">防爬虫设置</a></li>
<li>{{main_name}}{{listName}}</li>
</ul>
<h4>{{main_name}}{{listName}}设置</h4>
</div>
</div>
<!-- media -->
</div>
<!-- pageheader -->
<div class="contentpanel page-servers">
<div class="panel panel-primary-head">
<!-- panel-heading -->
<div class="input-group" style="margin: 20px 20px 20px 0">
<input id="val" type="text" class="form-control" placeholder="请输入..." style="width: 500px;">
<span class="input-group-btn" style="float: left;">
<button class="btn btn-default" type="button" id="add_black">添加至{{listName}}</button>
</span>
</div>
<table id="table-servers" class="table table-striped table-bordered responsive" style="border: 1px solid #ddd;">
<thead class="">
<tr>
<th>{{main_name}}</th>
<th>操作</th>
</tr>
</thead>
<tbody>
{{#each list}}
<tr>
<td>{{name}}</td>
<td>
<button class="btn btn-danger btn-xs server-del">删除</button>
</td>
</tr>
{{/each}}
</tbody>
</table>
</div>
<!-- panel -->
</div>
<script>
var isBlack = '{{listName}}' === '黑名单';
var path = location.pathname.match(/\/crawler_(white|black)\/(.*)/)[2];
$('#add_black').on('click', function() {
var val = $('#val').val();
if (!val)return;
var dataId = $($("tbody").find("tr")[$("tbody").find("tr").length - 1]).find('[data-id]').attr('data-id');
$('tbody').append('<tr><td>' + val + '</td><td><button class="btn btn-danger btn-xs server-del">删除</button></td></tr>');
$('#val').val('');
var vallists = [];
$('tr td:first-child').each(function() {
vallists.push($(this).html());
});
vallists = JSON.stringify(vallists);
if (isBlack) {
$.get('/crawler_black/change_' + path + '?path=/crawler/' + path + '_blacklists&val=' + vallists);
} else {
$.get('/crawler_white/change_' + path + '?path=/crawler/' + path + '_whitelists&val=' + vallists);
}
});
$('tbody').on('click', '.server-del', function() {
$(this).parent().parent().remove();
var vallists = [];
$('tr td:first-child').each(function() {
vallists.push($(this).html());
});
vallists = (vallists.length ? JSON.stringify(vallists) : '');
if (isBlack) {
$.get('/crawler_black/change_' + path + '?path=/crawler/' + path + '_blacklists&val=' + vallists);
} else {
$.get('/crawler_white/change_' + path + '?path=/crawler/' + path + '_whitelists&val=' + vallists);
}
});
</script>
\ No newline at end of file
... ...
... ... @@ -18,12 +18,12 @@
{{/if}}
{{#if not_business}}
<li><a href="/projects"><i class="glyphicon glyphicon-th"></i> <span>项目</span></a></li>
<li class="parent"><a href=""><i class="glyphicon glyphicon-wrench"></i> <span>APP Hotfix</span></a>
{{!--<li class="parent"><a href=""><i class="glyphicon glyphicon-wrench"></i> <span>APP Hotfix</span></a>
<ul class="children">
<li><a href="/hotfix/Android">Android</a></li>
<li><a href="/hotfix/iOS">iOS</a></li>
</ul>
</li>
</li>--}}
<li class="parent"><a href=""><i class="fa fa-eye"></i> <span>监控中心</span></a>
<ul class="children">
<li><a href="/monitor/log">实时日志</a></li>
... ... @@ -38,15 +38,29 @@
</ul>
</li>
{{#if is_master}}
<li class="parent"><a href=""><i class="fa fa-gears"></i> <span>系统管理</span></a>
<li class="parent"><a href=""><i class="fa fa-gears"></i> <span>系统管理</span></a>
<ul class="children">
<li><a href="/servers/setting">服务器配置</a></li>
<li><a href="/users/setting">用户管理</a></li>
<li><a href="/operation/log">操作记录</a></li>
</ul>
</li>
{{/if}}
<li><a href="/degrade"><i class="fa fa-hand-o-down"></i> <span>降级配置</span></a></li>
<li class="parent"><a><i class="fa fa-shield"></i> <span>滥用防护</span></a>
<ul class="children">
<li><a href="/servers/setting">服务器配置</a></li>
<li><a href="/users/setting">用户管理</a></li>
<li><a href="/operation/log">操作记录</a></li>
<li><a href="/crawler_black/ip">IP黑名单</a></li>
<li><a href="/crawler_white/ip">IP白名单</a></li>
<li><a href="/crawler_black/ua">UA黑名单</a></li>
<li><a href="/crawler_white/ua">UA白名单</a></li>
<li>
<a href="/abuse_protection/abuse_protection">
<span>滥用防护</span>
</a>
</li>
</ul>
</li>
{{/if}}
<li><a href="/degrade"><i class="fa fa-hand-o-down"></i> <span>降级配置</span></a></li>
{{/if}}
</ul>
... ...