crawler.js 5.47 KB
'use strict';

const Router = require('koa-router');

const Operation = require('../../logger/operation');
const {
    Server
} = require('../../models');

const r = new Router;

const {DegradeServer} = require('../../models');
const zookeeper = require('node-zookeeper-client');
const tester = require('../../zookeeper/tester');
const getter = require('../../zookeeper/getter');
const Model = require('../../models/model');
const _ = require('lodash');

const envs = {
    production: '线上环境',
    preview: '灰度环境',
    test: '测试环境'
};

const servers = {
    ua:async(ctx, next)=>{
        let ua_lists=await servers.getLists('/crawler/ua_blacklists');console.log(typeof ua_lists,ua_lists.length);
        try{
            ua_lists=JSON.parse(ua_lists);
        }catch(e){
            ua_lists='';
        }
        await ctx.render('action/crawler', {
            blacklists: (ua_lists?ua_lists.map((item)=>{return {name:item}}):''),
            main_name:'UA'
        });
    },
    ip:async(ctx, next)=>{
        let ip_lists=await servers.getLists('/crawler/ip_blacklists');
        try{
            ip_lists=JSON.parse(ip_lists);
        }catch(e){
            ip_lists='';
        }
        await ctx.render('action/crawler', {
            blacklists: (ip_lists?ip_lists.map((item)=>{return {name:item}}):''),
            main_name:'IP'
        });
    },
    change_ua:async(ctx, next)=>{
        let result=await servers.setLists(ctx,'qcloud');
        result=await servers.setLists(ctx,'aws')||result;
        if (result) {
            ctx.body = {
                code: 200,
                message: 'update success'
            };
        } else {
            ctx.body = {
                code: 500,
                message: 'update fail,Please retry'
            }
        }
    },
    change_ip:async(ctx, next)=>{
        let result=await servers.setLists(ctx,'qcloud');
        result=await servers.setLists(ctx,'aws')||result;
        if (result) {
            ctx.body = {
                code: 200,
                message: 'update success'
            };
        } else {
            ctx.body = {
                code: 500,
                message: 'update fail,Please retry'
            }
        }
    },
    getLists:async(path)=>{
        let degrades=[];

        let qcloudServer = await DegradeServer.findOne({
            type: 'qcloud'
        });

        if (qcloudServer) {
            degrades=degrades.concat(await servers.connect(qcloudServer.ip,qcloudServer.port,path));
        }

        let awsServer = await DegradeServer.findOne({
            type: 'aws'
        });

        if (awsServer) {
            degrades=degrades.concat(await servers.connect(qcloudServer.ip,qcloudServer.port,path));
        }

        return degrades[0];
    },
    async connect(ip,port,path) {
        let server = `${ip}:${port}`;

        let connected = await tester(server);

        if (!connected) {
            console.log('连接失败');
            return;
        }

        const client = zookeeper.createClient(server);
        return new Promise((resolve,reject)=>{
            client.once('connected', function (err) {
                if(err){
                    reject(err);
                }else{
                    client.getData(
                        path,
                        function (event) {
                            console.log('Got event: %s.', event);
                        },
                        function (error, data, stat) {
                            if (error) {
                                console.log(error.stack);
                                reject(error);
                                return;
                            }
                            resolve(data.toString('utf8'));
                        }
                    );
                }
            });
            client.connect();

        }).then(data=>data).catch(()=>'');
    },
    async setLists(ctx,type) {
        let {path, val} = ctx.query;

        let server = await DegradeServer.findOne({
            type: type
        });

        const client = zookeeper.createClient(`${server.ip}:${server.port}`);

        const mkdirp=()=>{
            return new Promise((resolve,reject)=>{
                client.mkdirp(path, new Buffer('false'), (err, path) => {
                    if (err) {
                        reject(err);
                    } else {
                        resolve();
                    }
                });
            });
        };
        const setData=()=>{
            return new Promise((resolve,reject)=>{
                client.setData(path, new Buffer(val.toString()), function(err, data, stat) {
                    if (err) {
                        reject(err);
                    } else {
                        resolve(true);
                    }
                });
            });
        }


        let result= new Promise((resolve,reject)=>{
            client.once('connected', function (err) {
                if(err)reject(err);
                else resolve();
            });
        })
        .then(mkdirp)
        .then(setData)
        .then((ok)=>{
            client.close();
            if(ok)return true;
        })
        .catch(function(err){
            console.log(err);
            client.close();
            return false;
        });

        client.connect();
        return result;
    }
};

r.get('/ua', servers.ua);
r.get('/ip', servers.ip);
r.get('/change_ua', servers.change_ua);
r.get('/change_ip', servers.change_ip);

module.exports = r;