OPENTECH / yoho-node-ci · Commits

Go to a project

Authored by 周奇琪
Commit 3c8d0c056a7900c9a734b94a62c3f408fb6cd401 2 parents 9ea50e97 928465a7

Merge branch 'master' of git.yoho.cn:OPENTECH/yoho-node-ci 

Conflicts:
	apps/web/views/partials/common/menu.hbs
Inline Side-by-side
Showing 10 changed files with 663 additions and 31 deletions
@@ -37,6 +37,12 @@ app.use(convert(body({ @@ -37,6 +37,12 @@ app.use(convert(body({
37 multipart: true, 37 multipart: true,
38 queryString: { 38 queryString: {
39 plainObjects: true 39 plainObjects: true
  40 + },
  41 + jsonLimit: '10mb',
  42 + formLimit: '10mb',
  43 + textLimit: '10mb',
  44 + formidable: {
  45 + maxFieldsSize: 10 * 1024 * 1024
40 } 46 }
41 }))); 47 })));
42 app.use(mount('/', webApp)); 48 app.use(mount('/', webApp));
@@ -11,6 +11,18 @@ class ApiCache { @@ -11,6 +11,18 @@ class ApiCache {
11 this.memcached = new Memcached(host); 11 this.memcached = new Memcached(host);
12 } 12 }
13 13
  14 + setKey(key, value, ttl) {
  15 + this._log(`setting ${key}`);
  16 +
  17 + this.memcached.set(key, value, ttl, (err) => {
  18 + if (err) {
  19 + this._log(`set ${key} fail`)
  20 + } else {
  21 + this._log(`set ${key} success`)
  22 + }
  23 + });
  24 + }
  25 +
14 delKey(key) { 26 delKey(key) {
15 this._log(`deleting ${key}`) 27 this._log(`deleting ${key}`)
16 28
@@ -34,8 +46,75 @@ class ApiCache { @@ -34,8 +46,75 @@ class ApiCache {
34 }); 46 });
35 } 47 }
36 48
  49 + find(condition) {
  50 + let count = 0;
  51 +
  52 + return new Promise((resolve, reject) => {
  53 + this.memcached.items((err, result) => {
  54 + if (err) console.error(err);
  55 +
  56 + if (result.length === 0) {
  57 + this._log('empty items');
  58 + resolve([]);
  59 + }
  60 + // for each server...
  61 + result.forEach(itemSet => {
  62 + var keys = Object.keys(itemSet);
  63 + keys.pop(); // we don't need the "server" key, but the other indicate the slab id's
  64 +
  65 + var len = keys.length;
  66 +
  67 + if (keys.length === 0) {
  68 + this._log('empty item set');
  69 + resolve([]);
  70 + }
  71 + const keyList = [];
  72 +
  73 + keys.forEach(stats => {
  74 + // get a cachedump for each slabid and slab.number
  75 + this.memcached.cachedump(itemSet.server, parseInt(stats, 10), itemSet[stats].number, (err, response) => {
  76 +
  77 + if (response) {
  78 + if (_.isArray(response)) {
  79 + _.each(response, (item) => {
  80 + count++;
  81 +
  82 + if (condition(item.key)) {
  83 + keyList.push(item.key);
  84 + }
  85 + });
  86 + }
  87 + else {
  88 + count++;
  89 +
  90 + if (condition(response.key)) {
  91 + keyList.push(response.key);
  92 + }
  93 + }
  94 + }
  95 +
  96 +
  97 + if (--len === 0) {
  98 + this.memcached.getMulti(keyList, (err, data) => {
  99 + this.memcached.end();
  100 +
  101 + if (err) {
  102 + reject(err);
  103 + }
  104 +
  105 + resolve(data);
  106 + });
  107 + }
  108 + });
  109 + })
  110 + })
  111 + });
  112 + });
  113 +
  114 + }
  115 +
37 stats() { 116 stats() {
38 - this.memcached.stats( (err, stats) => { 117 + this.memcached.stats((err, stats) => {
39 if (err) { 118 if (err) {
40 119
41 } else { 120 } else {
@@ -43,7 +122,7 @@ class ApiCache { @@ -43,7 +122,7 @@ class ApiCache {
43 stats = stats[0]; 122 stats = stats[0];
44 } 123 }
45 124
46 - stats.hits_percent = (stats.get_hits / (stats.get_hits + stats.get_misses) * 100).toFixed(2) + '%'; 125 + stats.hits_percent = (stats.get_hits / (stats.get_hits + stats.get_misses) * 100).toFixed(2) + '%';
47 stats.heap_info = (stats.bytes / 1024 / 1024).toFixed(4) + 'mb' + ' / ' + stats.limit_maxbytes / 1024 / 1024 + 'mb'; 126 stats.heap_info = (stats.bytes / 1024 / 1024).toFixed(4) + 'mb' + ' / ' + stats.limit_maxbytes / 1024 / 1024 + 'mb';
48 127
49 ws.broadcast(`/api_cache/monit`, { 128 ws.broadcast(`/api_cache/monit`, {
@@ -55,7 +134,7 @@ class ApiCache { @@ -55,7 +134,7 @@ class ApiCache {
55 } 134 }
56 135
57 restart() { 136 restart() {
58 - this.memcached.stats( (err, stats) => { 137 + this.memcached.stats((err, stats) => {
59 if (err) { 138 if (err) {
60 139
61 } else { 140 } else {
@@ -73,7 +152,7 @@ class ApiCache { @@ -73,7 +152,7 @@ class ApiCache {
73 conn.on('ready', async() => { 152 conn.on('ready', async() => {
74 try { 153 try {
75 conn.exec(`sudo kill ${pid}`, (err, stream) => { 154 conn.exec(`sudo kill ${pid}`, (err, stream) => {
76 - if(err) { 155 + if (err) {
77 console.log(err); 156 console.log(err);
78 } else { 157 } else {
79 stream.on('close', () => { 158 stream.on('close', () => {
@@ -110,8 +189,8 @@ class ApiCache { @@ -110,8 +189,8 @@ class ApiCache {
110 189
111 let count = 0; 190 let count = 0;
112 191
113 - this.memcached.items( ( err, result ) => {  
114 - if( err ) console.error( err ); 192 + this.memcached.items((err, result) => {
  193 + if (err) console.error(err);
115 194
116 if (result.length === 0) { 195 if (result.length === 0) {
117 this._log('empty items') 196 this._log('empty items')
@@ -119,25 +198,25 @@ class ApiCache { @@ -119,25 +198,25 @@ class ApiCache {
119 // for each server... 198 // for each server...
120 result.forEach(itemSet => { 199 result.forEach(itemSet => {
121 200
122 - var keys = Object.keys( itemSet );  
123 - keys.pop(); // we don't need the "server" key, but the other indicate the slab id's  
124 - 201 + var keys = Object.keys(itemSet);
  202 + keys.pop(); // we don't need the "server" key, but the other indicate the slab id's
  203 +
125 var len = keys.length; 204 var len = keys.length;
126 205
127 if (keys.length === 0) { 206 if (keys.length === 0) {
128 this._log('empty item set'); 207 this._log('empty item set');
129 } 208 }
130 - 209 +
131 keys.forEach(stats => { 210 keys.forEach(stats => {
132 - 211 +
133 // get a cachedump for each slabid and slab.number 212 // get a cachedump for each slabid and slab.number
134 - this.memcached.cachedump( itemSet.server, parseInt(stats, 10), itemSet[stats].number, ( err, response ) => { 213 + this.memcached.cachedump(itemSet.server, parseInt(stats, 10), itemSet[stats].number, (err, response) => {
135 // dump the shizzle 214 // dump the shizzle
136 215
137 if (response) { 216 if (response) {
138 if (_.isArray(response)) { 217 if (_.isArray(response)) {
139 _.each(response, keyObj => { 218 _.each(response, keyObj => {
140 - count ++ ; 219 + count++;
141 if (keyObj.key && (keyObj.key.indexOf(key1) >= 0 || keyObj.key.indexOf(key2) >= 0 || keyObj.key.indexOf(key) >= 0)) { 220 if (keyObj.key && (keyObj.key.indexOf(key1) >= 0 || keyObj.key.indexOf(key2) >= 0 || keyObj.key.indexOf(key) >= 0)) {
142 this.delKey(keyObj.key); 221 this.delKey(keyObj.key);
143 } else { 222 } else {
@@ -145,7 +224,7 @@ class ApiCache { @@ -145,7 +224,7 @@ class ApiCache {
145 } 224 }
146 }); 225 });
147 } else { 226 } else {
148 - count ++; 227 + count++;
149 if (response.key && (response.key.indexOf(key1) >= 0 || response.key.indexOf(key2) >= 0 || response.key.indexOf(key) >= 0)) { 228 if (response.key && (response.key.indexOf(key1) >= 0 || response.key.indexOf(key2) >= 0 || response.key.indexOf(key) >= 0)) {
150 this.delKey(response.key); 229 this.delKey(response.key);
151 } else { 230 } else {
@@ -154,7 +233,7 @@ class ApiCache { @@ -154,7 +233,7 @@ class ApiCache {
154 } 233 }
155 } 234 }
156 235
157 - len --; 236 + len--;
158 237
159 if (len === 0) { 238 if (len === 0) {
160 this.memcached.end(); 239 this.memcached.end();
@@ -52,6 +52,26 @@ const defaultDegrades = [ @@ -52,6 +52,26 @@ const defaultDegrades = [
52 path: '/pc/cart/removeMerge', 52 path: '/pc/cart/removeMerge',
53 name: '【移除】购物车>>> 凑单商品' 53 name: '【移除】购物车>>> 凑单商品'
54 }, 54 },
  55 + {
  56 + path: '/pc/clientService/new',
  57 + name: '【开关】开启新客服系统'
  58 + },
  59 + {
  60 + path: '/pc/qcloud_cdn',
  61 + name: '【开关】启动腾讯云备份CDN'
  62 + },
  63 + {
  64 + path: '/pc/user/removeStudentIdentification',
  65 + name: '【移除】学生认证开关'
  66 + },
  67 + {
  68 + path: '/pc/sys/noLimiter',
  69 + name: '【开关】关闭请求限制'
  70 + },
  71 + {
  72 + path: '/pc/pay/oldCart',
  73 + name: '【开关】开启老版购物车'
  74 + },
55 //wap 75 //wap
56 { 76 {
57 path: '/wap/plustar/removeCollect', 77 path: '/wap/plustar/removeCollect',
@@ -76,6 +96,30 @@ const defaultDegrades = [ @@ -76,6 +96,30 @@ const defaultDegrades = [
76 { 96 {
77 path: '/wap/cart/removePrefer', 97 path: '/wap/cart/removePrefer',
78 name: '【移除】购物车>>> 为您优选新品' 98 name: '【移除】购物车>>> 为您优选新品'
  99 + },
  100 + {
  101 + path: '/wap/clientService/new',
  102 + name: '【开关】开启新客服系统'
  103 + },
  104 + {
  105 + path: '/wap/qcloud_cdn',
  106 + name: '【开关】启动腾讯云备份CDN'
  107 + },
  108 + {
  109 + path: '/wap/user/removeStudentIdentification',
  110 + name: '【移除】学生认证开关'
  111 + },
  112 + {
  113 + path: '/wap/sys/noLimiter',
  114 + name: '【开关】关闭请求限制'
  115 + },
  116 + {
  117 + path: '/wap/pay/newCart',
  118 + name: '【开关】开启新版购物车'
  119 + },
  120 + {
  121 + path: '/wap/pay/newPay',
  122 + name: '【开关】开启新版支付'
79 } 123 }
80 ]; 124 ];
81 125
  1 +'use strict';
  2 +
  3 +const Router = require('koa-router');
  4 +const ApiCache = require('../../ci/api_cache');
  5 +const _ = require('lodash');
  6 +
  7 +const {
  8 + MemcachedHost
  9 +} = require('../../models');
  10 +
  11 +let r = new Router();
  12 +
  13 +const defensive = {
  14 + index: async(ctx, next) => {
  15 + const regexp = /pc:limiter:faker:(.*)/;
  16 + const threshold = ctx.request.query.threshold || 100;
  17 + const limit = ctx.request.query.limit || 10;
  18 +
  19 + let hosts = await MemcachedHost.findAll();
  20 +
  21 + const selectedHosts = _.filter(hosts, host => {
  22 + const isCurrent = host.host === ctx.request.query.node;
  23 + if (isCurrent) {
  24 + host.isCurrent = true;
  25 + }
  26 + return isCurrent;
  27 + });
  28 +
  29 + let results = await Promise.all(_.map(selectedHosts, (h) => {
  30 + return (new ApiCache(h.host)).find((key) => {
  31 + return regexp.test(key);
  32 + });
  33 + }));
  34 +
  35 + let list = [];
  36 +
  37 + if (results && results[0]) {
  38 + Object.keys(results[0]).forEach((key) => {
  39 + const index = results[0][key];
  40 +
  41 + if (index > threshold) {
  42 + list.push({
  43 + ip: ((key) => {
  44 + const m = key.match(regexp);
  45 +
  46 + return m && m.length > 0 ? m[1] : 'Unknown';
  47 + })(key),
  48 + index: index
  49 + })
  50 + }
  51 + });
  52 + }
  53 +
  54 + list = _.orderBy(list, (item) => {
  55 + return item.index;
  56 + }, 'desc').slice(0, limit);
  57 +
  58 + await ctx.render('action/abuse_protection', {
  59 + hosts: hosts,
  60 + list: list,
  61 + noData: list.length === 0,
  62 + threshold: threshold,
  63 + limit: limit
  64 + });
  65 + },
  66 + lock: async(ctx, next) => {
  67 + let hosts = await MemcachedHost.findAll();
  68 + await Promise.all(_.map(hosts, (h) => {
  69 + const key = `pc:limiter:${ctx.request.body.remoteIp}`,
  70 + value = 9999,
  71 + ttl = 60 * 60 * 8; // 封停8小时
  72 +
  73 + return (new ApiCache(h.host)).setKey(key, value, ttl);
  74 + }));
  75 +
  76 + return ctx.body = {
  77 + code: 200
  78 + };
  79 + },
  80 + unlock: async(ctx, next) => {
  81 + let hosts = await MemcachedHost.findAll();
  82 + await Promise.all(_.map(hosts, (h) => {
  83 + const key = `pc:limiter:${ctx.request.body.remoteIp}`;
  84 +
  85 + return (new ApiCache(h.host)).delKey(key);
  86 + }));
  87 +
  88 + return ctx.body = {
  89 + code: 200
  90 + };
  91 + }
  92 +};
  93 +
  94 +r.get('/abuse_protection', defensive.index);
  95 +r.post('/lock', defensive.lock);
  96 +r.post('/unlock', defensive.unlock);
  97 +
  98 +module.exports = r;
@@ -26,6 +26,8 @@ module.exports = { @@ -26,6 +26,8 @@ module.exports = {
26 hosts = target; 26 hosts = target;
27 } 27 }
28 28
  29 + hosts = hosts || [];
  30 +
29 hosts.forEach((host) => { 31 hosts.forEach((host) => {
30 if (!servers[host]) { 32 if (!servers[host]) {
31 servers[host] = new Collect(host, p.name, p.cloud); 33 servers[host] = new Collect(host, p.name, p.cloud);
  1 +'use strict';
  2 +
  3 +const Router = require('koa-router');
  4 +
  5 +const {
  6 + MemcachedHost
  7 +} = require('../../models');
  8 +const Operation = require('../../logger/operation');
  9 +const {
  10 + Server
  11 +} = require('../../models');
  12 +
  13 +const {DegradeServer} = require('../../models');
  14 +const zookeeper = require('node-zookeeper-client');
  15 +const tester = require('../../zookeeper/tester');
  16 +const getter = require('../../zookeeper/getter');
  17 +const Model = require('../../models/model');
  18 +const _ = require('lodash');
  19 +const ApiCache = require('../../ci/api_cache');
  20 +
  21 +const envs = {
  22 + p1oduction: '线上环境',
  23 + preview: '灰度环境',
  24 + test: '测试环境'
  25 +};
  26 +
  27 +class Store extends Model {
  28 + constructor() {
  29 + super('abuse_protection');
  30 + }
  31 +}
  32 +const store = new Store();
  33 +const makeServer = ((ipKey, uaKey, listName, black) => {
  34 + const r = new Router;
  35 +
  36 + const servers = {
  37 + ua: async(ctx, next) => {
  38 + let list = await servers.getLists(uaKey);
  39 +
  40 + await ctx.render('action/crawler', {
  41 + listName: listName,
  42 + list: (list ? list.map((item) => {
  43 + return {name: item}
  44 + }) : ''),
  45 + main_name: 'UA'
  46 + });
  47 + },
  48 + ip: async(ctx, next) => {
  49 + let list = await servers.getLists(ipKey);
  50 +
  51 + await ctx.render('action/crawler', {
  52 + listName: listName,
  53 + list: (list ? list.map((item) => {
  54 + return {name: item}
  55 + }) : ''),
  56 + main_name: 'IP'
  57 + });
  58 + },
  59 + change_ua: async(ctx, next) => {
  60 + const doUpdate = async(ua) => {
  61 + console.log('include ua:' + ua);
  62 +
  63 + let hosts = await MemcachedHost.findAll();
  64 + await Promise.all(_.map(hosts, (h) => {
  65 + const key = `pc:limiter:ua:${black ? 'black' : 'white'}`,
  66 + value = JSON.parse(ua || '[]');
  67 +
  68 + return (new ApiCache(h.host)).setKey(key, value, 0);
  69 + }));
  70 + };
  71 +
  72 + let result = await servers.setLists(ctx);
  73 + await doUpdate(ctx.query.val);
  74 +
  75 + if (result) {
  76 + ctx.body = {
  77 + listName: listName,
  78 + code: 200,
  79 + message: 'update success'
  80 + };
  81 + } else {
  82 + ctx.body = {
  83 + listName: listName,
  84 + code: 500,
  85 + message: 'update fail,Please retry'
  86 + }
  87 + }
  88 + },
  89 + change_ip: async(ctx, next) => {
  90 + let oldList = await servers.getLists(ipKey);
  91 +
  92 + const newList = JSON.parse(ctx.query.val || '[]');
  93 +
  94 + const exclude = async(ip) => {
  95 + console.log('exclude:' + ip);
  96 +
  97 + let hosts = await MemcachedHost.findAll();
  98 + await Promise.all(_.map(hosts, (h) => {
  99 + const key = `pc:limiter:${ip}`,
  100 + value = -1,
  101 + ttl = 0;
  102 +
  103 + return (new ApiCache(h.host)).setKey(key, value, ttl);
  104 + }));
  105 + };
  106 +
  107 + const lock = async(ip) => {
  108 + console.log('lock:' + ip);
  109 +
  110 + let hosts = await MemcachedHost.findAll();
  111 + await Promise.all(_.map(hosts, (h) => {
  112 + const key = `pc:limiter:${ip}`,
  113 + value = 9999,
  114 + ttl = 60 * 60 * 8; // 封停8小时
  115 +
  116 + return (new ApiCache(h.host)).setKey(key, value, ttl);
  117 + }));
  118 + };
  119 +
  120 + const unlock = async(ip) => {
  121 + console.log('unlock:' + ip);
  122 +
  123 + let hosts = await MemcachedHost.findAll();
  124 + await Promise.all(_.map(hosts, (h) => {
  125 + const key = `pc:limiter:${ip}`;
  126 +
  127 + return (new ApiCache(h.host)).delKey(key);
  128 + }));
  129 + };
  130 +
  131 + const unlockList = [];
  132 +
  133 + _.each(oldList, (item) => {
  134 + if (_.indexOf(newList, item) < 0) {
  135 + unlockList.push(item);
  136 + }
  137 + });
  138 +
  139 + _.each(newList, (ip) => {
  140 + if (black) {
  141 + lock(ip);
  142 + } else {
  143 + exclude(ip);
  144 + }
  145 + });
  146 +
  147 + _.each(unlockList, (ip) => {
  148 + unlock(ip);
  149 + });
  150 +
  151 + let result = await servers.setLists(ctx);
  152 +
  153 +
  154 + if (result) {
  155 + ctx.body = {
  156 + listName: listName,
  157 + code: 200,
  158 + message: 'update success'
  159 + };
  160 + } else {
  161 + ctx.body = {
  162 + code: 500,
  163 + message: 'update fail,Please retry'
  164 + }
  165 + }
  166 +
  167 + },
  168 + getLists: async(path) => {
  169 + const result = await store.findOne({
  170 + path: path
  171 + });
  172 +
  173 + return result && result.val ? JSON.parse(result.val) : [];
  174 + },
  175 +
  176 + async setLists(ctx, type) {
  177 + let {path, val} = ctx.query;
  178 +
  179 + const rec = await store.findOne({
  180 + path: path
  181 + });
  182 +
  183 + if (rec) {
  184 + store.update({
  185 + path: path
  186 + }, {
  187 + $set: {
  188 + val: val
  189 + }
  190 + })
  191 + } else {
  192 + store.insert({
  193 + path: path,
  194 + val: val
  195 + })
  196 + }
  197 + }
  198 + };
  199 +
  200 + r.get('/ua', servers.ua);
  201 + r.get('/ip', servers.ip);
  202 + r.get('/change_ua', servers.change_ua);
  203 + r.get('/change_ip', servers.change_ip);
  204 + return r;
  205 +});
  206 +
  207 +module.exports = makeServer;
@@ -8,27 +8,29 @@ const servers = require('./actions/servers'); @@ -8,27 +8,29 @@ const servers = require('./actions/servers');
8 const login = require('./actions/login'); 8 const login = require('./actions/login');
9 const monitor = require('./actions/monitor'); 9 const monitor = require('./actions/monitor');
10 const users = require('./actions/users'); 10 const users = require('./actions/users');
11 -const hotfix = require( './actions/hotfix'); 11 +const hotfix = require('./actions/hotfix');
12 const operationLog = require('./actions/operation_log'); 12 const operationLog = require('./actions/operation_log');
13 -const pageCache = require( './actions/page_cache');  
14 -const cdnCache = require( './actions/cdn_cache');  
15 -const productCache = require( './actions/product_cache'); 13 +const pageCache = require('./actions/page_cache');
  14 +const cdnCache = require('./actions/cdn_cache');
  15 +const productCache = require('./actions/product_cache');
16 const apiCache = require('./actions/api_cache'); 16 const apiCache = require('./actions/api_cache');
17 const degrade = require('./actions/degrade'); 17 const degrade = require('./actions/degrade');
18 const deploy = require('./actions/deploy'); 18 const deploy = require('./actions/deploy');
19 const api = require('./actions/api'); 19 const api = require('./actions/api');
  20 +const abuseProtection = require('./actions/abuse_protection');
  21 +const crawler = require('./actions/crawler');
20 22
21 const noAuth = new Router(); 23 const noAuth = new Router();
22 const base = new Router(); 24 const base = new Router();
23 25
24 -module.exports = function (app) { 26 +module.exports = function(app) {
25 27
26 noAuth.use('', login.routes(), login.allowedMethods()); 28 noAuth.use('', login.routes(), login.allowedMethods());
27 noAuth.use('', common.routes(), common.allowedMethods()); 29 noAuth.use('', common.routes(), common.allowedMethods());
28 noAuth.use('/api', api.routes(), api.allowedMethods()); 30 noAuth.use('/api', api.routes(), api.allowedMethods());
29 31
30 app.use(noAuth.routes(), noAuth.allowedMethods()); 32 app.use(noAuth.routes(), noAuth.allowedMethods());
31 - app.use(async (ctx, next) => { 33 + app.use(async(ctx, next) => {
32 34
33 if (ctx.session && ctx.session.user) { 35 if (ctx.session && ctx.session.user) {
34 await next(); 36 await next();
@@ -41,7 +43,7 @@ module.exports = function (app) { @@ -41,7 +43,7 @@ module.exports = function (app) {
41 base.use('/servers', servers.routes(), servers.allowedMethods()); 43 base.use('/servers', servers.routes(), servers.allowedMethods());
42 base.use('/monitor', monitor.routes(), monitor.allowedMethods()); 44 base.use('/monitor', monitor.routes(), monitor.allowedMethods());
43 base.use('/users', users.routes(), users.allowedMethods()); 45 base.use('/users', users.routes(), users.allowedMethods());
44 - base.use('/hotfix', hotfix.routes(), hotfix.allowedMethods()); 46 + // base.use('/hotfix', hotfix.routes(), hotfix.allowedMethods());
45 base.use('/operation', operationLog.routes(), operationLog.allowedMethods()); 47 base.use('/operation', operationLog.routes(), operationLog.allowedMethods());
46 base.use('/page_cache', pageCache.routes(), pageCache.allowedMethods()); 48 base.use('/page_cache', pageCache.routes(), pageCache.allowedMethods());
47 base.use('/cdn_cache', cdnCache.routes(), cdnCache.allowedMethods()); 49 base.use('/cdn_cache', cdnCache.routes(), cdnCache.allowedMethods());
@@ -50,7 +52,13 @@ module.exports = function (app) { @@ -50,7 +52,13 @@ module.exports = function (app) {
50 base.use('/degrade', degrade.routes(), degrade.allowedMethods()); 52 base.use('/degrade', degrade.routes(), degrade.allowedMethods());
51 base.use('/deploys', deploy.routes(), deploy.allowedMethods()); 53 base.use('/deploys', deploy.routes(), deploy.allowedMethods());
52 54
53 - 55 + const white = crawler('/crawler/ip_whitelists', '/crawler/ua_whitelists', '白名单', false);
  56 + const black = crawler('/crawler/ip_blacklists', '/crawler/ua_blacklists', '黑名单', true);
  57 +
  58 + base.use('/crawler_white', white.routes(), white.allowedMethods());
  59 + base.use('/crawler_black', black.routes(), black.allowedMethods());
  60 +
  61 + base.use('/abuse_protection', abuseProtection.routes(), degrade.allowedMethods());
54 62
55 base.use('', index.routes(), index.allowedMethods()); 63 base.use('', index.routes(), index.allowedMethods());
56 64
  1 +<div class="pageheader">
  2 + <div class="media">
  3 + <div class="pageicon pull-left">
  4 + <i class="fa fa-th-list"></i>
  5 + </div>
  6 + <div class="media-body">
  7 + <ul class="breadcrumb">
  8 + <li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
  9 + <li><a href="/abuse_protection">滥用防护</a></li>
  10 + </ul>
  11 + <h4>疑似爬虫监控</h4>
  12 + </div>
  13 + </div>
  14 + <!-- media -->
  15 +</div>
  16 +<!-- pageheader -->
  17 +
  18 +<div class="contentpanel servers-page">
  19 + <form action="/abuse_protection/abuse_protection" class="form-inline" method="get">
  20 + <div class="form-group">
  21 + <select name="node" class="form-control" style="height: 40px">
  22 + <option value="">Memcached节点</option>
  23 + {{#hosts}}
  24 + <option value="{{host}}" {{#if isCurrent}}selected="selected"{{/if}}>{{host}}</option>
  25 + {{/hosts}}
  26 + </select>
  27 + </div>
  28 +
  29 + <div class="form-group input-group">
  30 + <input id="threshold" type="text" class="form-control" size="3" name="threshold" value="{{threshold}}"/>
  31 + <div class="input-group-addon">/天</div>
  32 + </div>
  33 +
  34 + <div class="form-group input-group">
  35 + <div class="input-group-addon">显示</div>
  36 + <input id="threshold" type="text" class="form-control" size="3" name="limit" value="{{limit}}"/>
  37 + <div class="input-group-addon">条</div>
  38 + </div>
  39 + <button class="btn btn-primary">应用</button>
  40 + </form>
  41 +
  42 + <table class="table table-striped table-bordered building-table" style="margin-top: 2rem">
  43 + <tr>
  44 + <th>IP</th>
  45 + <th>爬虫指数</th>
  46 + <th>操作</th>
  47 + </tr>
  48 +
  49 + {{#list}}
  50 + <tr>
  51 + <td>{{ip}}</td>
  52 + <td>{{index}}</td>
  53 + <td>
  54 + <button class="access-deny btn btn-primary btn-sm btn-danger" data-ip="{{ip}}">禁止访问</button>
  55 + <button class="access-allow btn btn-success btn-sm" data-ip="{{ip}}">允许访问</button>
  56 + </td>
  57 + </tr>
  58 + {{/list}}
  59 +
  60 + {{#if noData}}
  61 + <tr>
  62 + <td colspan="4" class="text-center">无数据</td>
  63 + </tr>
  64 + {{/if}}
  65 + </table>
  66 +</div>
  67 +
  68 +<script>
  69 + $(function() {
  70 + $('.access-deny').click(function() {
  71 + $.post('/abuse_protection/lock', {remoteIp: $(this).data('ip')}, function() {
  72 + });
  73 + });
  74 +
  75 + $('.access-allow').click(function() {
  76 + $.post('/abuse_protection/unlock', {remoteIp: $(this).data('ip')}, function() {
  77 + });
  78 + });
  79 + });
  80 +</script>
  1 +<div class="pageheader">
  2 + <div class="media">
  3 + <div class="pageicon pull-left">
  4 + <i class="fa fa-th-list"></i>
  5 + </div>
  6 + <div class="media-body">
  7 + <ul class="breadcrumb">
  8 + <li><a href=""><i class="glyphicon glyphicon-home"></i></a></li>
  9 + <li><a href="">防爬虫设置</a></li>
  10 + <li>{{main_name}}{{listName}}</li>
  11 + </ul>
  12 + <h4>{{main_name}}{{listName}}设置</h4>
  13 + </div>
  14 + </div>
  15 + <!-- media -->
  16 +</div>
  17 +<!-- pageheader -->
  18 +
  19 +<div class="contentpanel page-servers">
  20 + <div class="panel panel-primary-head">
  21 + <!-- panel-heading -->
  22 + <div class="input-group" style="margin: 20px 20px 20px 0">
  23 + <input id="val" type="text" class="form-control" placeholder="请输入..." style="width: 500px;">
  24 + <span class="input-group-btn" style="float: left;">
  25 + <button class="btn btn-default" type="button" id="add_black">添加至{{listName}}</button>
  26 + </span>
  27 + </div>
  28 + <table id="table-servers" class="table table-striped table-bordered responsive" style="border: 1px solid #ddd;">
  29 + <thead class="">
  30 + <tr>
  31 + <th>{{main_name}}</th>
  32 + <th>操作</th>
  33 + </tr>
  34 + </thead>
  35 +
  36 + <tbody>
  37 + {{#each list}}
  38 + <tr>
  39 + <td>{{name}}</td>
  40 + <td>
  41 + <button class="btn btn-danger btn-xs server-del">删除</button>
  42 + </td>
  43 + </tr>
  44 + {{/each}}
  45 + </tbody>
  46 + </table>
  47 + </div>
  48 + <!-- panel -->
  49 +</div>
  50 +
  51 +
  52 +<script>
  53 + var isBlack = '{{listName}}' === '黑名单';
  54 + var path = location.pathname.match(/\/crawler_(white|black)\/(.*)/)[2];
  55 +
  56 + $('#add_black').on('click', function() {
  57 + var val = $('#val').val();
  58 + if (!val)return;
  59 + var dataId = $($("tbody").find("tr")[$("tbody").find("tr").length - 1]).find('[data-id]').attr('data-id');
  60 + $('tbody').append('<tr><td>' + val + '</td><td><button class="btn btn-danger btn-xs server-del">删除</button></td></tr>');
  61 + $('#val').val('');
  62 +
  63 +
  64 + var vallists = [];
  65 + $('tr td:first-child').each(function() {
  66 + vallists.push($(this).html());
  67 + });
  68 + vallists = JSON.stringify(vallists);
  69 +
  70 + if (isBlack) {
  71 + $.get('/crawler_black/change_' + path + '?path=/crawler/' + path + '_blacklists&val=' + vallists);
  72 + } else {
  73 + $.get('/crawler_white/change_' + path + '?path=/crawler/' + path + '_whitelists&val=' + vallists);
  74 +
  75 + }
  76 + });
  77 +
  78 + $('tbody').on('click', '.server-del', function() {
  79 + $(this).parent().parent().remove();
  80 +
  81 + var vallists = [];
  82 + $('tr td:first-child').each(function() {
  83 + vallists.push($(this).html());
  84 + });
  85 + vallists = (vallists.length ? JSON.stringify(vallists) : '');
  86 +
  87 + if (isBlack) {
  88 + $.get('/crawler_black/change_' + path + '?path=/crawler/' + path + '_blacklists&val=' + vallists);
  89 + } else {
  90 + $.get('/crawler_white/change_' + path + '?path=/crawler/' + path + '_whitelists&val=' + vallists);
  91 + }
  92 + });
  93 +
  94 +</script>
@@ -18,12 +18,12 @@ @@ -18,12 +18,12 @@
18 {{/if}} 18 {{/if}}
19 {{#if not_business}} 19 {{#if not_business}}
20 <li><a href="/projects"><i class="glyphicon glyphicon-th"></i> <span>项目</span></a></li> 20 <li><a href="/projects"><i class="glyphicon glyphicon-th"></i> <span>项目</span></a></li>
21 - <li class="parent"><a href=""><i class="glyphicon glyphicon-wrench"></i> <span>APP Hotfix</span></a> 21 + {{!--<li class="parent"><a href=""><i class="glyphicon glyphicon-wrench"></i> <span>APP Hotfix</span></a>
22 <ul class="children"> 22 <ul class="children">
23 <li><a href="/hotfix/Android">Android</a></li> 23 <li><a href="/hotfix/Android">Android</a></li>
24 <li><a href="/hotfix/iOS">iOS</a></li> 24 <li><a href="/hotfix/iOS">iOS</a></li>
25 </ul> 25 </ul>
26 - </li> 26 + </li>--}}
27 <li class="parent"><a href=""><i class="fa fa-eye"></i> <span>监控中心</span></a> 27 <li class="parent"><a href=""><i class="fa fa-eye"></i> <span>监控中心</span></a>
28 <ul class="children"> 28 <ul class="children">
29 <li><a href="/monitor/log">实时日志</a></li> 29 <li><a href="/monitor/log">实时日志</a></li>
@@ -38,15 +38,29 @@ @@ -38,15 +38,29 @@
38 </ul> 38 </ul>
39 </li> 39 </li>
40 {{#if is_master}} 40 {{#if is_master}}
41 - <li class="parent"><a href=""><i class="fa fa-gears"></i> <span>系统管理</span></a> 41 + <li class="parent"><a href=""><i class="fa fa-gears"></i> <span>系统管理</span></a>
  42 + <ul class="children">
  43 + <li><a href="/servers/setting">服务器配置</a></li>
  44 + <li><a href="/users/setting">用户管理</a></li>
  45 + <li><a href="/operation/log">操作记录</a></li>
  46 + </ul>
  47 + </li>
  48 + {{/if}}
  49 + <li><a href="/degrade"><i class="fa fa-hand-o-down"></i> <span>降级配置</span></a></li>
  50 +
  51 + <li class="parent"><a><i class="fa fa-shield"></i> <span>滥用防护</span></a>
42 <ul class="children"> 52 <ul class="children">
43 - <li><a href="/servers/setting">服务器配置</a></li>  
44 - <li><a href="/users/setting">用户管理</a></li>  
45 - <li><a href="/operation/log">操作记录</a></li> 53 + <li><a href="/crawler_black/ip">IP黑名单</a></li>
  54 + <li><a href="/crawler_white/ip">IP白名单</a></li>
  55 + <li><a href="/crawler_black/ua">UA黑名单</a></li>
  56 + <li><a href="/crawler_white/ua">UA白名单</a></li>
  57 + <li>
  58 + <a href="/abuse_protection/abuse_protection">
  59 + <span>滥用防护</span>
  60 + </a>
  61 + </li>
46 </ul> 62 </ul>
47 </li> 63 </li>
48 - {{/if}}  
49 - <li><a href="/degrade"><i class="fa fa-hand-o-down"></i> <span>降级配置</span></a></li>  
50 {{/if}} 64 {{/if}}
51 </ul> 65 </ul>
52 66