滥用防护

@@ -2,197 +2,206 @@
 
 const Router = require('koa-router');
 
+const {
+    MemcachedHost
+} = require('../../models');
 const Operation = require('../../logger/operation');
 const {
     Server
 } = require('../../models');
 
-const r = new Router;
-
 const {DegradeServer} = require('../../models');
 const zookeeper = require('node-zookeeper-client');
 const tester = require('../../zookeeper/tester');
 const getter = require('../../zookeeper/getter');
 const Model = require('../../models/model');
 const _ = require('lodash');
+const ApiCache = require('../../ci/api_cache');
 
 const envs = {
-    production: '线上环境',
+    p1oduction: '线上环境',
     preview: '灰度环境',
     test: '测试环境'
 };
 
-const servers = {
-    ua:async(ctx, next)=>{
-        let ua_lists=await servers.getLists('/crawler/ua_blacklists');console.log(typeof ua_lists,ua_lists.length);
-        try{
-            ua_lists=JSON.parse(ua_lists);
-        }catch(e){
-            ua_lists='';
-        }
-        await ctx.render('action/crawler', {
-            blacklists: (ua_lists?ua_lists.map((item)=>{return {name:item}}):''),
-            main_name:'UA'
-        });
-    },
-    ip:async(ctx, next)=>{
-        let ip_lists=await servers.getLists('/crawler/ip_blacklists');
-        try{
-            ip_lists=JSON.parse(ip_lists);
-        }catch(e){
-            ip_lists='';
-        }
-        await ctx.render('action/crawler', {
-            blacklists: (ip_lists?ip_lists.map((item)=>{return {name:item}}):''),
-            main_name:'IP'
-        });
-    },
-    change_ua:async(ctx, next)=>{
-        let result=await servers.setLists(ctx,'qcloud');
-        result=await servers.setLists(ctx,'aws')||result;
-        if (result) {
-            ctx.body = {
-                code: 200,
-                message: 'update success'
+class Store extends Model {
+    constructor() {
+        super('abuse_protection');
+    }
+}
+const store = new Store();
+const makeServer = ((ipKey, uaKey, listName, black) => {
+    const r = new Router;
+
+    const servers = {
+        ua: async(ctx, next) => {
+            let list = await servers.getLists(uaKey);
+
+            await ctx.render('action/crawler', {
+                listName: listName,
+                list: (list ? list.map((item) => {
+                        return {name: item}
+                    }) : ''),
+                main_name: 'UA'
+            });
+        },
+        ip: async(ctx, next) => {
+            let list = await servers.getLists(ipKey);
+
+            await ctx.render('action/crawler', {
+                listName: listName,
+                list: (list ? list.map((item) => {
+                        return {name: item}
+                    }) : ''),
+                main_name: 'IP'
+            });
+        },
+        change_ua: async(ctx, next) => {
+            const doUpdate = async(ua) => {
+                console.log('include ua:' + ua);
+
+                let hosts = await MemcachedHost.findAll();
+                await Promise.all(_.map(hosts, (h) => {
+                    const key = `pc:limiter:ua:${black ? 'black' : 'white'}`,
+                        value = JSON.parse(ua || '[]');
+
+                    return (new ApiCache(h.host)).setKey(key, value, 0);
+                }));
             };
-        } else {
-            ctx.body = {
-                code: 500,
-                message: 'update fail,Please retry'
+
+            let result = await servers.setLists(ctx);
+            await doUpdate(ctx.query.val);
+
+            if (result) {
+                ctx.body = {
+                    listName: listName,
+                    code: 200,
+                    message: 'update success'
+                };
+            } else {
+                ctx.body = {
+                    listName: listName,
+                    code: 500,
+                    message: 'update fail,Please retry'
+                }
             }
-        }
-    },
-    change_ip:async(ctx, next)=>{
-        let result=await servers.setLists(ctx,'qcloud');
-        result=await servers.setLists(ctx,'aws')||result;
-        if (result) {
-            ctx.body = {
-                code: 200,
-                message: 'update success'
+        },
+        change_ip: async(ctx, next) => {
+            let oldList = await servers.getLists(ipKey);
+
+            const newList = JSON.parse(ctx.query.val || '[]');
+
+            const exclude = async(ip) => {
+                console.log('exclude:' + ip);
+
+                let hosts = await MemcachedHost.findAll();
+                await Promise.all(_.map(hosts, (h) => {
+                    const key = `pc:limiter:${ip}`,
+                        value = -1,
+                        ttl = 0;
+
+                    return (new ApiCache(h.host)).setKey(key, value, ttl);
+                }));
             };
-        } else {
-            ctx.body = {
-                code: 500,
-                message: 'update fail,Please retry'
-            }
-        }
-    },
-    getLists:async(path)=>{
-        let degrades=[];
 
-        let qcloudServer = await DegradeServer.findOne({
-            type: 'qcloud'
-        });
+            const lock = async(ip) => {
+                console.log('lock:' + ip);
 
-        if (qcloudServer) {
-            degrades=degrades.concat(await servers.connect(qcloudServer.ip,qcloudServer.port,path));
-        }
+                let hosts = await MemcachedHost.findAll();
+                await Promise.all(_.map(hosts, (h) => {
+                    const key = `pc:limiter:${ip}`,
+                        value = 9999,
+                        ttl = 60 * 60 * 8; // 封停8小时
 
-        let awsServer = await DegradeServer.findOne({
-            type: 'aws'
-        });
+                    return (new ApiCache(h.host)).setKey(key, value, ttl);
+                }));
+            };
 
-        if (awsServer) {
-            degrades=degrades.concat(await servers.connect(qcloudServer.ip,qcloudServer.port,path));
-        }
+            const unlock = async(ip) => {
+                console.log('unlock:' + ip);
+
+                let hosts = await MemcachedHost.findAll();
+                await Promise.all(_.map(hosts, (h) => {
+                    const key = `pc:limiter:${ip}`;
 
-        return degrades[0];
-    },
-    async connect(ip,port,path) {
-        let server = `${ip}:${port}`;
+                    return (new ApiCache(h.host)).delKey(key);
+                }));
+            };
 
-        let connected = await tester(server);
+            const unlockList = [];
 
-        if (!connected) {
-            console.log('连接失败');
-            return;
-        }
+            _.each(oldList, (item) => {
+                if (_.indexOf(newList, item) < 0) {
+                    unlockList.push(item);
+                }
+            });
 
-        const client = zookeeper.createClient(server);
-        return new Promise((resolve,reject)=>{
-            client.once('connected', function (err) {
-                if(err){
-                    reject(err);
-                }else{
-                    client.getData(
-                        path,
-                        function (event) {
-                            console.log('Got event: %s.', event);
-                        },
-                        function (error, data, stat) {
-                            if (error) {
-                                console.log(error.stack);
-                                reject(error);
-                                return;
-                            }
-                            resolve(data.toString('utf8'));
-                        }
-                    );
+            _.each(newList, (ip) => {
+                if (black) {
+                    lock(ip);
+                } else {
+                    exclude(ip);
                 }
             });
-            client.connect();
-
-        }).then(data=>data).catch(()=>'');
-    },
-    async setLists(ctx,type) {
-        let {path, val} = ctx.query;
-
-        let server = await DegradeServer.findOne({
-            type: type
-        });
-
-        const client = zookeeper.createClient(`${server.ip}:${server.port}`);
-
-        const mkdirp=()=>{
-            return new Promise((resolve,reject)=>{
-                client.mkdirp(path, new Buffer('false'), (err, path) => {
-                    if (err) {
-                        reject(err);
-                    } else {
-                        resolve();
-                    }
-                });
+
+            _.each(unlockList, (ip) => {
+                unlock(ip);
             });
-        };
-        const setData=()=>{
-            return new Promise((resolve,reject)=>{
-                client.setData(path, new Buffer(val.toString()), function(err, data, stat) {
-                    if (err) {
-                        reject(err);
-                    } else {
-                        resolve(true);
-                    }
-                });
+
+            let result = await servers.setLists(ctx);
+
+
+            if (result) {
+                ctx.body = {
+                    listName: listName,
+                    code: 200,
+                    message: 'update success'
+                };
+            } else {
+                ctx.body = {
+                    code: 500,
+                    message: 'update fail,Please retry'
+                }
+            }
+
+        },
+        getLists: async(path) => {
+            const result = await store.findOne({
+                path: path
             });
-        }
 
+            return result && result.val ? JSON.parse(result.val) : [];
+        },
 
-        let result= new Promise((resolve,reject)=>{
-            client.once('connected', function (err) {
-                if(err)reject(err);
-                else resolve();
+        async setLists(ctx, type) {
+            let {path, val} = ctx.query;
+
+            const rec = await store.findOne({
+                path: path
             });
-        })
-        .then(mkdirp)
-        .then(setData)
-        .then((ok)=>{
-            client.close();
-            if(ok)return true;
-        })
-        .catch(function(err){
-            console.log(err);
-            client.close();
-            return false;
-        });
-
-        client.connect();
-        return result;
-    }
-};
 
-r.get('/ua', servers.ua);
-r.get('/ip', servers.ip);
-r.get('/change_ua', servers.change_ua);
-r.get('/change_ip', servers.change_ip);
+            if (rec) {
+                store.update({
+                    path: path
+                }, {
+                    $set: {
+                        val: val
+                    }
+                })
+            } else {
+                store.insert({
+                    path: path,
+                    val: val
+                })
+            }
+        }
+    };
+
+    r.get('/ua', servers.ua);
+    r.get('/ip', servers.ip);
+    r.get('/change_ua', servers.change_ua);
+    r.get('/change_ip', servers.change_ip);
+    return r;
+});
 
-module.exports = r;
\ No newline at end of file
+module.exports = makeServer;
\ No newline at end of file

@@ -8,11 +8,11 @@ const servers = require('./actions/servers');
 const login = require('./actions/login');
 const monitor = require('./actions/monitor');
 const users = require('./actions/users');
-const hotfix = require( './actions/hotfix');
+const hotfix = require('./actions/hotfix');
 const operationLog = require('./actions/operation_log');
-const pageCache = require( './actions/page_cache');
-const cdnCache = require( './actions/cdn_cache');
-const productCache = require( './actions/product_cache');
+const pageCache = require('./actions/page_cache');
+const cdnCache = require('./actions/cdn_cache');
+const productCache = require('./actions/product_cache');
 const apiCache = require('./actions/api_cache');
 const degrade = require('./actions/degrade');
 const deploy = require('./actions/deploy');
@@ -23,14 +23,14 @@ const crawler = require('./actions/crawler');
 const noAuth = new Router();
 const base = new Router();
 
-module.exports = function (app) {
+module.exports = function(app) {
 
     noAuth.use('', login.routes(), login.allowedMethods());
     noAuth.use('', common.routes(), common.allowedMethods());
     noAuth.use('/api', api.routes(), api.allowedMethods());
 
     app.use(noAuth.routes(), noAuth.allowedMethods());
-    app.use(async (ctx, next) => {
+    app.use(async(ctx, next) => {
 
         if (ctx.session && ctx.session.user) {
             await next();
@@ -51,7 +51,13 @@ module.exports = function (app) {
     base.use('/api_cache', apiCache.routes(), apiCache.allowedMethods());
     base.use('/degrade', degrade.routes(), degrade.allowedMethods());
     base.use('/deploys', deploy.routes(), deploy.allowedMethods());
-    base.use('/crawler', crawler.routes(), crawler.allowedMethods());
+
+    const white = crawler('/crawler/ip_whitelists', '/crawler/ua_whitelists', '白名单', false);
+    const black = crawler('/crawler/ip_blacklists', '/crawler/ua_blacklists', '黑名单', true);
+
+    base.use('/crawler_white', white.routes(), white.allowedMethods());
+    base.use('/crawler_black', black.routes(), black.allowedMethods());
+
     base.use('/abuse_protection', abuseProtection.routes(), degrade.allowedMethods());
 
     base.use('', index.routes(), index.allowedMethods());

@@ -7,9 +7,9 @@
             <ul class="breadcrumb">
                 <li><a href=""><i class="glyphicon glyphicon-home"></i></a></li>
                 <li><a href="">防爬虫设置</a></li>
-                <li>{{main_name}}黑名单</li>
+                <li>{{main_name}}{{listName}}</li>
             </ul>
-            <h4>{{main_name}}黑名单设置</h4>
+            <h4>{{main_name}}{{listName}}设置</h4>
         </div>
     </div>
     <!-- media -->
@@ -18,35 +18,30 @@
 
 <div class="contentpanel page-servers">
     <div class="panel panel-primary-head">
-        <div class="panel-heading">
-            <h4 class="panel-title">{{main_name}}黑名单设置</h4>
-            <p>配置黑名单列表</p>
-            
-        </div>
         <!-- panel-heading -->
         <div class="input-group" style="margin: 20px 20px 20px 0">
             <input id="val" type="text" class="form-control" placeholder="请输入..." style="width: 500px;">
             <span class="input-group-btn" style="float: left;">
-                <button class="btn btn-default" type="button" id="add_black">add</button>
+                <button class="btn btn-default" type="button" id="add_black">添加至{{listName}}</button>
             </span>
         </div>
         <table id="table-servers" class="table table-striped table-bordered responsive" style="border: 1px solid #ddd;">
             <thead class="">
-                <tr>
-                    <th>{{main_name}}_name</th>
-                    <th>操作</th>
-                </tr>
+            <tr>
+                <th>{{main_name}}</th>
+                <th>操作</th>
+            </tr>
             </thead>
 
             <tbody>
-                {{#each blacklists}}
+            {{#each list}}
                 <tr>
                     <td>{{name}}</td>
                     <td>
                         <button class="btn btn-danger btn-xs server-del">删除</button>
                     </td>
                 </tr>
-                {{/each}}
+            {{/each}}
             </tbody>
         </table>
     </div>
@@ -55,32 +50,45 @@
 
 
 <script>
-    $('#add_black').on('click',function (){
-        var val=$('#val').val();
-        if(!val)return;
-        var dataId=$($("tbody").find("tr")[$("tbody").find("tr").length-1]).find('[data-id]').attr('data-id');
-        $('tbody').append('<tr><td>'+val+'</td><td><button class="btn btn-danger btn-xs server-del">删除</button></td></tr>');
+    var isBlack = '{{listName}}' === '黑名单';
+    var path = location.pathname.match(/\/crawler_(white|black)\/(.*)/)[2];
+
+    $('#add_black').on('click', function() {
+        var val = $('#val').val();
+        if (!val)return;
+        var dataId = $($("tbody").find("tr")[$("tbody").find("tr").length - 1]).find('[data-id]').attr('data-id');
+        $('tbody').append('<tr><td>' + val + '</td><td><button class="btn btn-danger btn-xs server-del">删除</button></td></tr>');
         $('#val').val('');
-        var path=location.pathname.match(/\/crawler\/(.*)/)[1];
 
-        var vallists=[];
-        $('tr td:first-child').each(function(){
+
+        var vallists = [];
+        $('tr td:first-child').each(function() {
             vallists.push($(this).html());
         });
-        vallists=JSON.stringify(vallists);
-        $.get('/crawler/change_'+path+'?path=/crawler/'+path+'_blacklists&val='+vallists);
+        vallists = JSON.stringify(vallists);
+
+        if (isBlack) {
+            $.get('/crawler_black/change_' + path + '?path=/crawler/' + path + '_blacklists&val=' + vallists);
+        } else {
+            $.get('/crawler_white/change_' + path + '?path=/crawler/' + path + '_whitelists&val=' + vallists);
+
+        }
     });
 
-    $('tbody').on('click','.server-del',function(){
+    $('tbody').on('click', '.server-del', function() {
         $(this).parent().parent().remove();
-        var path=location.pathname.match(/\/crawler\/(.*)/)[1];
 
-        var vallists=[];
-        $('tr td:first-child').each(function(){
+        var vallists = [];
+        $('tr td:first-child').each(function() {
             vallists.push($(this).html());
         });
-        vallists=(vallists.length?JSON.stringify(vallists):'');
-        $.get('/crawler/change_'+path+'?path=/crawler/'+path+'_blacklists&val='+vallists);//
+        vallists = (vallists.length ? JSON.stringify(vallists) : '');
+
+        if (isBlack) {
+            $.get('/crawler_black/change_' + path + '?path=/crawler/' + path + '_blacklists&val=' + vallists);
+        } else {
+            $.get('/crawler_white/change_' + path + '?path=/crawler/' + path + '_whitelists&val=' + vallists);
+        }
     });
 
 </script>
\ No newline at end of file

@@ -34,21 +34,27 @@
             </ul>
         </li>
         {{#if is_master}}
-        <li class="parent"><a href=""><i class="fa fa-gears"></i> <span>系统管理</span></a>
-            <ul class="children">
-                <li><a href="/servers/setting">服务器配置</a></li>
-                <li><a href="/users/setting">用户管理</a></li>
-                <li><a href="/operation/log">操作记录</a></li>
-            </ul>
-        </li>
+            <li class="parent"><a href=""><i class="fa fa-gears"></i> <span>系统管理</span></a>
+                <ul class="children">
+                    <li><a href="/servers/setting">服务器配置</a></li>
+                    <li><a href="/users/setting">用户管理</a></li>
+                    <li><a href="/operation/log">操作记录</a></li>
+                </ul>
+            </li>
         {{/if}}
         <li><a href="/degrade"><i class="fa fa-hand-o-down"></i> <span>降级配置</span></a></li>
-        <li><a href="/abuse_protection/abuse_protection"><i class="fa fa-shield"></i> <span>滥用防护</span></a></li>
 
-        <li class="parent"><a><i class="fa fa-eye"></i> <span>防爬虫设置</span></a>
+        <li class="parent"><a><i class="fa fa-shield"></i> <span>滥用防护</span></a>
             <ul class="children">
-                <li><a href="/crawler/ua">UA黑名单</a></li>
-                <li><a href="/crawler/ip">IP黑名单</a></li>
+                <li><a href="/crawler_black/ip">IP黑名单</a></li>
+                <li><a href="/crawler_white/ip">IP白名单</a></li>
+                <li><a href="/crawler_black/ua">UA黑名单</a></li>
+                <li><a href="/crawler_white/ua">UA白名单</a></li>
+                <li>
+                    <a href="/abuse_protection/abuse_protection">
+                        <span>滥用防护</span>
+                    </a>
+                </li>
             </ul>
         </li>
     </ul>