crawler.js 6.32 KB
'use strict';

const Router = require('koa-router');

const {
    MemcachedHost
} = require('../../models');
const Operation = require('../../logger/operation');
const {
    Server
} = require('../../models');

const {DegradeServer} = require('../../models');
const zookeeper = require('node-zookeeper-client');
const tester = require('../../zookeeper/tester');
const getter = require('../../zookeeper/getter');
const Model = require('../../models/model');
const _ = require('lodash');
const ApiCache = require('../../ci/api_cache_redis');

const envs = {
    p1oduction: '线上环境',
    preview: '灰度环境',
    test: '测试环境'
};

const UA_LIST_PREFIX = 'pc:limiter:ua:'; // ua 列表
const WHITE_LIST_PREFIX = 'whitelist:ip:'; // 白名单前缀
const BLACK_LIST_PREFIX = 'pc:limiter:'; // 黑名单前缀

class Store extends Model {
    constructor() {
        super('abuse_protection');
    }
}
const store = new Store();
const makeServer = ((ipKey, uaKey, listName, black) => {
    const r = new Router;

    const servers = {
        ua: async(ctx, next) => {
            let list = await servers.getLists(uaKey);

            await ctx.render('action/crawler', {
                listName: listName,
                list: (list ? list.map((item) => {
                        return {name: item}
                    }) : ''),
                main_name: 'UA'
            });
        },
        ip: async(ctx, next) => {
            let list = await servers.getLists(ipKey);

            await ctx.render('action/crawler', {
                listName: listName,
                list: (list ? list.map((item) => {
                        return {name: item}
                    }) : ''),
                main_name: 'IP'
            });
        },
        change_ua: async(ctx, next) => {
            const doUpdate = async (ua) => {
                console.log('include ua:' + ua);

                let uaObj = [];

                try {
                    uaObj = JSON.parse(ua);
                } catch (error) {
                    console.error(error);
                }

                uaObj = _.uniq(uaObj);

                const keyPrefix = `${UA_LIST_PREFIX}${black ? 'black' : 'white'}`;

                await new ApiCache().setKey(keyPrefix, JSON.stringify(uaObj), 0, true);
            };

            await doUpdate(ctx.query.val);

            let result = await servers.setLists(ctx);

            if (result) {
                ctx.body = {
                    listName: listName,
                    code: 200,
                    message: 'update success'
                };
            } else {
                ctx.body = {
                    listName: listName,
                    code: 500,
                    message: 'update fail,Please retry'
                }
            }
        },
        change_ip: async(ctx, next) => {
            let oldList = await servers.getLists(ipKey);
            let newList = JSON.parse(ctx.query.val || '[]');

            // 黑名单封停 IP
            const lock = async (ip) => {
                console.log(`lock ip: ${ip}`);

                const key = `${BLACK_LIST_PREFIX}${ip}`;
                const value = 9999;
                const ttl = 60 * 60 * 24 * 30; // 封停一月

                await new ApiCache().setKey(key, value, ttl);
            };

            // 从黑名单移除 IP,解锁
            const unlock = async (ip) => {
                console.log(`unlock ip: ${ip}`);

                const key = `${BLACK_LIST_PREFIX}${ip}`;

                await new ApiCache().delKey(key);
            };

            // ip 添加到白名单
            const addToWhiteList = async ip => {
                console.log(`addToWhiteList ip: ${ip}`);

                const key = `${WHITE_LIST_PREFIX}${ip}`;

                await new ApiCache().setKey(key, true);
            };

            // ip 从白名单移除
            const removeFromWhiteList = async ip => {
                console.log(`removeFromWhiteList ip: ${ip}`);

                const key = `${WHITE_LIST_PREFIX}${ip}`;

                await new ApiCache().delKey(key);
            }

            const unlockList = [];

            _.each(oldList, (item) => {
                if (_.indexOf(newList, item) < 0) {
                    unlockList.push(item);
                }
            });

            _.each(newList, (ip) => {
                if (black) {
                    lock(ip);
                } else {
                    addToWhiteList(ip);
                }
            });

            _.each(unlockList, (ip) => {
                if (black) {
                    unlock(ip);
                } else {
                    removeFromWhiteList(ip);
                }
            });

            let result = await servers.setLists(ctx);

            if (result) {
                ctx.body = {
                    listName: listName,
                    code: 200,
                    message: 'update success'
                };
            } else {
                ctx.body = {
                    code: 500,
                    message: 'update fail,Please retry'
                }
            }

        },
        getLists: async(path) => {
            const result = await store.findOne({
                path: path
            });

            return result && result.val ? JSON.parse(result.val) : [];
        },

        async setLists(ctx, type) {
            let { path, val } = ctx.query;

            let valArr = [];

            try {
                valArr = JSON.parse(val);
            } catch (error) {
                console.error(error);
            }

            if (valArr.length) {
                valArr = _.uniq(valArr);
                val = JSON.stringify(valArr);
            }

            const rec = await store.findOne({
                path: path
            });

            if (rec) {
                return store.update({
                    path: path
                }, {
                    $set: {
                        val: val
                    }
                })
            } else {
                return store.insert({
                    path: path,
                    val: val
                })
            }
        }
    };

    r.get('/ua', servers.ua);
    r.get('/ip', servers.ip);
    r.get('/change_ua', servers.change_ua);
    r.get('/change_ip', servers.change_ip);
    return r;
});

module.exports = makeServer;