Merge remote-tracking branch 'origin/feature/avoid-crawler'

# Conflicts:
#	apps/web/routers.js
#	apps/web/views/partials/common/menu.hbs

+'use strict';
+
+const Router = require('koa-router');
+
+const Operation = require('../../logger/operation');
+const {
+    Server
+} = require('../../models');
+
+const r = new Router;
+
+const {DegradeServer} = require('../../models');
+const zookeeper = require('node-zookeeper-client');
+const tester = require('../../zookeeper/tester');
+const getter = require('../../zookeeper/getter');
+const Model = require('../../models/model');
+const _ = require('lodash');
+
+const envs = {
+    production: '线上环境',
+    preview: '灰度环境',
+    test: '测试环境'
+};
+
+const servers = {
+    ua:async(ctx, next)=>{
+        let ua_lists=await servers.getLists('/crawler/ua_blacklists');console.log(typeof ua_lists,ua_lists.length);
+        try{
+            ua_lists=JSON.parse(ua_lists);
+        }catch(e){
+            ua_lists='';
+        }
+        await ctx.render('action/crawler', {
+            blacklists: (ua_lists?ua_lists.map((item)=>{return {name:item}}):''),
+            main_name:'UA'
+        });
+    },
+    ip:async(ctx, next)=>{
+        let ip_lists=await servers.getLists('/crawler/ip_blacklists');
+        try{
+            ip_lists=JSON.parse(ip_lists);
+        }catch(e){
+            ip_lists='';
+        }
+        await ctx.render('action/crawler', {
+            blacklists: (ip_lists?ip_lists.map((item)=>{return {name:item}}):''),
+            main_name:'IP'
+        });
+    },
+    change_ua:async(ctx, next)=>{
+        let result=await servers.setLists(ctx,'qcloud');
+        result=await servers.setLists(ctx,'aws')||result;
+        if (result) {
+            ctx.body = {
+                code: 200,
+                message: 'update success'
+            };
+        } else {
+            ctx.body = {
+                code: 500,
+                message: 'update fail,Please retry'
+            }
+        }
+    },
+    change_ip:async(ctx, next)=>{
+        let result=await servers.setLists(ctx,'qcloud');
+        result=await servers.setLists(ctx,'aws')||result;
+        if (result) {
+            ctx.body = {
+                code: 200,
+                message: 'update success'
+            };
+        } else {
+            ctx.body = {
+                code: 500,
+                message: 'update fail,Please retry'
+            }
+        }
+    },
+    getLists:async(path)=>{
+        let degrades=[];
+
+        let qcloudServer = await DegradeServer.findOne({
+            type: 'qcloud'
+        });
+
+        if (qcloudServer) {
+            degrades=degrades.concat(await servers.connect(qcloudServer.ip,qcloudServer.port,path));
+        }
+
+        let awsServer = await DegradeServer.findOne({
+            type: 'aws'
+        });
+
+        if (awsServer) {
+            degrades=degrades.concat(await servers.connect(qcloudServer.ip,qcloudServer.port,path));
+        }
+
+        return degrades[0];
+    },
+    async connect(ip,port,path) {
+        let server = `${ip}:${port}`;
+
+        let connected = await tester(server);
+
+        if (!connected) {
+            console.log('连接失败');
+            return;
+        }
+
+        const client = zookeeper.createClient(server);
+        return new Promise((resolve,reject)=>{
+            client.once('connected', function (err) {
+                if(err){
+                    reject(err);
+                }else{
+                    client.getData(
+                        path,
+                        function (event) {
+                            console.log('Got event: %s.', event);
+                        },
+                        function (error, data, stat) {
+                            if (error) {
+                                console.log(error.stack);
+                                reject(error);
+                                return;
+                            }
+                            resolve(data.toString('utf8'));
+                        }
+                    );
+                }
+            });
+            client.connect();
+
+        }).then(data=>data).catch(()=>'');
+    },
+    async setLists(ctx,type) {
+        let {path, val} = ctx.query;
+
+        let server = await DegradeServer.findOne({
+            type: type
+        });
+
+        const client = zookeeper.createClient(`${server.ip}:${server.port}`);
+
+        const mkdirp=()=>{
+            return new Promise((resolve,reject)=>{
+                client.mkdirp(path, new Buffer('false'), (err, path) => {
+                    if (err) {
+                        reject(err);
+                    } else {
+                        resolve();
+                    }
+                });
+            });
+        };
+        const setData=()=>{
+            return new Promise((resolve,reject)=>{
+                client.setData(path, new Buffer(val.toString()), function(err, data, stat) {
+                    if (err) {
+                        reject(err);
+                    } else {
+                        resolve(true);
+                    }
+                });
+            });
+        }
+
+
+        let result= new Promise((resolve,reject)=>{
+            client.once('connected', function (err) {
+                if(err)reject(err);
+                else resolve();
+            });
+        })
+        .then(mkdirp)
+        .then(setData)
+        .then((ok)=>{
+            client.close();
+            if(ok)return true;
+        })
+        .catch(function(err){
+            console.log(err);
+            client.close();
+            return false;
+        });
+
+        client.connect();
+        return result;
+    }
+};
+
+r.get('/ua', servers.ua);
+r.get('/ip', servers.ip);
+r.get('/change_ua', servers.change_ua);
+r.get('/change_ip', servers.change_ip);
+
+module.exports = r;

@@ -18,6 +18,7 @@ const degrade = require('./actions/degrade');
 const deploy = require('./actions/deploy');
 const deploy = require('./actions/deploy');
 const api = require('./actions/api');
 const api = require('./actions/api');
 const abuseProtection = require('./actions/abuse_protection');
 const abuseProtection = require('./actions/abuse_protection');
+const crawler = require('./actions/crawler');
 
 
 const noAuth = new Router();
 const noAuth = new Router();
 const base = new Router();
 const base = new Router();
@@ -50,6 +51,7 @@ module.exports = function (app) {
     base.use('/api_cache', apiCache.routes(), apiCache.allowedMethods());
     base.use('/api_cache', apiCache.routes(), apiCache.allowedMethods());
     base.use('/degrade', degrade.routes(), degrade.allowedMethods());
     base.use('/degrade', degrade.routes(), degrade.allowedMethods());
     base.use('/deploys', deploy.routes(), deploy.allowedMethods());
     base.use('/deploys', deploy.routes(), deploy.allowedMethods());
+    base.use('/crawler', crawler.routes(), crawler.allowedMethods());
     base.use('/abuse_protection', abuseProtection.routes(), degrade.allowedMethods());
     base.use('/abuse_protection', abuseProtection.routes(), degrade.allowedMethods());
 
 
     base.use('', index.routes(), index.allowedMethods());
     base.use('', index.routes(), index.allowedMethods());

+<div class="pageheader">
+    <div class="media">
+        <div class="pageicon pull-left">
+            <i class="fa fa-th-list"></i>
+        </div>
+        <div class="media-body">
+            <ul class="breadcrumb">
+                <li><a href=""><i class="glyphicon glyphicon-home"></i></a></li>
+                <li><a href="">防爬虫设置</a></li>
+                <li>{{main_name}}黑名单</li>
+            </ul>
+            <h4>{{main_name}}黑名单设置</h4>
+        </div>
+    </div>
+    <!-- media -->
+</div>
+<!-- pageheader -->
+
+<div class="contentpanel page-servers">
+    <div class="panel panel-primary-head">
+        <div class="panel-heading">
+            <h4 class="panel-title">{{main_name}}黑名单设置</h4>
+            <p>配置黑名单列表</p>
+            
+        </div>
+        <!-- panel-heading -->
+        <div class="input-group" style="margin: 20px 20px 20px 0">
+            <input id="val" type="text" class="form-control" placeholder="请输入..." style="width: 500px;">
+            <span class="input-group-btn" style="float: left;">
+                <button class="btn btn-default" type="button" id="add_black">add</button>
+            </span>
+        </div>
+        <table id="table-servers" class="table table-striped table-bordered responsive" style="border: 1px solid #ddd;">
+            <thead class="">
+                <tr>
+                    <th>{{main_name}}_name</th>
+                    <th>操作</th>
+                </tr>
+            </thead>
+
+            <tbody>
+                {{#each blacklists}}
+                <tr>
+                    <td>{{name}}</td>
+                    <td>
+                        <button class="btn btn-danger btn-xs server-del">删除</button>
+                    </td>
+                </tr>
+                {{/each}}
+            </tbody>
+        </table>
+    </div>
+    <!-- panel -->
+</div>
+
+
+<script>
+    $('#add_black').on('click',function (){
+        var val=$('#val').val();
+        if(!val)return;
+        var dataId=$($("tbody").find("tr")[$("tbody").find("tr").length-1]).find('[data-id]').attr('data-id');
+        $('tbody').append('<tr><td>'+val+'</td><td><button class="btn btn-danger btn-xs server-del">删除</button></td></tr>');
+        $('#val').val('');
+        var path=location.pathname.match(/\/crawler\/(.*)/)[1];
+
+        var vallists=[];
+        $('tr td:first-child').each(function(){
+            vallists.push($(this).html());
+        });
+        vallists=JSON.stringify(vallists);
+        $.get('/crawler/change_'+path+'?path=/crawler/'+path+'_blacklists&val='+vallists);
+    });
+
+    $('tbody').on('click','.server-del',function(){
+        $(this).parent().parent().remove();
+        var path=location.pathname.match(/\/crawler\/(.*)/)[1];
+
+        var vallists=[];
+        $('tr td:first-child').each(function(){
+            vallists.push($(this).html());
+        });
+        vallists=(vallists.length?JSON.stringify(vallists):'');
+        $.get('/crawler/change_'+path+'?path=/crawler/'+path+'_blacklists&val='+vallists);//
+    });
+
+</script>

@@ -43,9 +43,14 @@
         </li>
         </li>
         {{/if}}
         {{/if}}
         <li><a href="/degrade"><i class="fa fa-hand-o-down"></i> <span>降级配置</span></a></li>
         <li><a href="/degrade"><i class="fa fa-hand-o-down"></i> <span>降级配置</span></a></li>
-
         <li><a href="/abuse_protection/abuse_protection"><i class="fa fa-shield"></i> <span>滥用防护</span></a></li>
         <li><a href="/abuse_protection/abuse_protection"><i class="fa fa-shield"></i> <span>滥用防护</span></a></li>
 
 
+        <li class="parent"><a><i class="fa fa-eye"></i> <span>防爬虫设置</span></a>
+            <ul class="children">
+                <li><a href="/crawler/ua">UA黑名单</a></li>
+                <li><a href="/crawler/ip">IP黑名单</a></li>
+            </ul>
+        </li>
     </ul>
     </ul>
 
 
 </div>
 </div>