profile.js 11.5 KB
// http://www.daterangepicker.com/#usage

const Router = require('koa-router');
const SqlBuilder = require('../utils/sql-builder');
const request = require('superagent');
const _ = require('lodash');
const fp = require('lodash/fp');
const jp = require('jsonpath');
const config = require('../../../config/config');
const ErrorModel = require('../models/errorModel');
const SlowRouteModel = require('../models/slowRouteModel');
const SortRouteModel = require('../models/sortRouteModel');
const SortApiModel = require('../models/sortApiModel');
const SortClientModel = require('../models/sortClientRouteModel');

const endpoint = (server) => `http://${server.host}:${server.port}`;

const r = new Router;

const TABLE = {
    CLIENT: 'web-client-duration',
    REPORT: 'error-report',
    SERVER: 'web-server-duration'
};

const DB_NAME = 'web-apm';

const APP_NAME = {
    default: 0,
    pc: 'yohobuy-node',
    h5: 'yohobuywap-node'
};

const APP_NAME2 = {
    pc: 'www.yohobuy.com',
    h5: 'm.yohobuy.com'
};

const TYPE = {
    DOMContentLoaded: 0,
    load: 0,
    firstscreen: 0
};

const profile_sql = {
    client() {
        return SqlBuilder.of(TABLE.CLIENT);
    },
    error() {
        return SqlBuilder.of(TABLE.REPORT);
    },
    server() {
        return SqlBuilder.of(TABLE.SERVER);
    }
};

async function exec(server, sql) {
    console.log('influx query from ', `[${server}] [${DB_NAME}]`, 'sql =>', sql);
    return request.get(`${server}/query`)
        .query({
            q: sql,
            db: DB_NAME
        }).then(({body: result}) => {
            return result;
        }).catch((err) => {
            console.log(err);
            return {};
        });
}

const APP = {
    default: {field: '', op: '', value: ''},
    pc: {field: 'app', op: '=', value: APP_NAME.pc},
    h5: {field: 'app', op: '=', value: APP_NAME.h5}
};

const APP2 = {
    default: {field: '', op: '', value: ''},
    pc: {field: 'app', op: '=', value: APP_NAME2.pc},
    h5: {field: 'app', op: '=', value: APP_NAME2.h5}
};

const SERVER = {
    aws: endpoint(config.apm.aws),
    qcloud: endpoint(config.apm.qcloud)
};


function handleZip(items) {
    const col = _.get(items, 'columns', []);
    const values = _.get(items, "values", []);

    return values.map((v) => {
        return _.zipObject(col, v)
    });
}

function handleZip2(item) {
    const col = _.get(item, 'columns', []);
    const values = _.get(item, "values[0]", []);
    const tags = _.get(item, "tags", []);

    return Object.assign({}, _.zipObject(col, values), tags)
}

function handleRows(rows) {
    let stats = rows.reduce((acc, cur) => {
        _.updateWith(acc, [cur.route, cur.type, 'duration'], function(n) {
            return _.isNil(n) ? cur.duration : n + cur.duration;
        });

        _.updateWith(acc, [cur.route, cur.type, 'count'], function(n) {
            return _.isNil(n) ? 1 : n + 1
        });

        _.updateWith(acc, [cur.route, 'detail'], function(n) {
            if (_.isNil(n)) {
                return [cur];
            } else {
                n.push(cur);
            }

            return n;
        });

        return acc;
    }, {});

    let result = _.map(stats, (v,k) => {
        let result = {
            route: '',
            count: 0,
            mean: 0,
            DOMContentLoaded: 0,
            load: 0,
            firstscreen: 0,
            detail: []

        };

        result.route = k;
        result.count = _.sum(jp.query(v, '$..count'));
        result.DOMContentLoaded = +(_.get(v, 'DOMContentLoaded.duration', 0) / _.get(v, 'DOMContentLoaded.count', 1)).toFixed(0);
        result.load = +(_.get(v, 'load.duration', 0) / _.get(v, 'load.count', 1)).toFixed(0);
        result.firstscreen = +(_.get(v, 'firstscreen.duration', 0) / _.get(v, 'firstscreen.count', 1)).toFixed(0);
        result.mean = result.load + result.DOMContentLoaded + result.firstscreen;
        result.detail = v.detail;

        return result;
    });

    return result;
}

function handleApi(totalRows, apiRows) {
    let totalObj = totalRows.map(handleZip2);
    let apiObj = apiRows.map(handleZip2);

    totalObj.forEach(i => {
        i.detail = jp.query(apiObj, `$..[?(@.route=="${i.route}")]`);
        i.length = i.detail.length;
    });

    return totalObj;
}

const profile_service = {
    async client_time(server, start, end, app, lastTime) {
        const model = profile_sql.client()
            .select('*')
            .where(APP[app]);

        if (lastTime) {
            model.where('time', '>=', SqlBuilder.raw(`now() - ${lastTime}`))
        }

        if (start && end) {
            model.where('time', '>=', start)
                .where('time', '<', end);
        }

        let rows = await exec(SERVER[server], model.toSql())
            .then(result => _.get(result, 'results[0].series[0]', []))
            .then(handleZip);

        rows = handleRows(rows);

        return {code: 200, data: rows}
    },
    async server_time(ctx, app, start, length) {
        const slowRouterModel = new SlowRouteModel(ctx);
        const result = await slowRouterModel.getList(app, start, length);

        return result;
    },
    async error(ctx, app, host, type, time, code, api, route, start, length) {
        const errorModel = new ErrorModel(ctx);
        const result = await errorModel.getList(app, host, type, time, code, api, route, start, length);

        return result;
    },
    async sort_route_time(ctx, app, time) {
        const sortRouteModel = new SortRouteModel(ctx);
        const result = await sortRouteModel.getTimeList(app, time);

        return result;
    },
    async sort_route_count(ctx, app, time) {
        const sortRouteModel = new SortRouteModel(ctx);
        const result = await sortRouteModel.getCountList(app, time);

        return result;
    },
    async sort_api_time(ctx, app, time) {
        const sortRouteModel = new SortApiModel(ctx);
        const result = await sortRouteModel.getTimeList(app, time);

        return result;
    },
    async sort_api_count(ctx, app, time) {
        const sortRouteModel = new SortApiModel(ctx);
        const result = await sortRouteModel.getCountList(app, time);

        return result;
    },
    async getApp(ctx) {
        let model = new ErrorModel(ctx);
        let app = await model.getApp().then(r => r.map(fp.prop('app')));

        return app;
    },
    async sort_server_route(ctx, app, time) {
        let routeTime = await this.sort_route_time(ctx, app, time);
        let routeCount = await this.sort_route_count(ctx, app, time);

        let sortRouteTime = _.orderBy(routeTime, ['avg'], ['desc']);
        let sortRouteCount = _.orderBy(routeCount, ['count'], ['desc']);

        return {time: sortRouteTime, count: sortRouteCount};
    },
    async sort_server_api(ctx, app, time) {
        let routeTime = await this.sort_api_time(ctx, app, time);
        let routeCount = await this.sort_api_count(ctx, app, time);

        let sortRouteTime = _.orderBy(routeTime, ['avg'], ['desc']);
        let sortRouteCount = _.orderBy(routeCount, ['count'], ['desc']);

        return {time: sortRouteTime, count: sortRouteCount};
    },
    async sort_client_time(ctx, app, time) {
        const newClientTime = new SortClientModel(ctx);
        const result = await newClientTime.getFirstScreenTimeList(app, time);

        return result;
    },
    async sort_client_count(ctx, app, time) {
        const newClientCount = new SortClientModel(ctx);
        const result = await newClientCount.getCountList(app, time);

        return result;
    },
    async sort_client_route(ctx, app, time) {
        let routeTime = await this.sort_client_time(ctx, app, time);
        let routeCount = await this.sort_client_count(ctx, app, time);

        let sortRouteTime = _.orderBy(routeTime, ['avg'], ['desc']);
        let sortRouteCount = _.orderBy(routeCount, ['count'], ['desc']);

        return {time: sortRouteTime, count: sortRouteCount};
    }
};

const profile_controller = {
    async server_mean_report_index(ctx) {
        await ctx.render('action/profile_server');
    },
    async server_mean_report_json(ctx) {
        const app = APP_NAME[ctx.query.app] || APP_NAME.default;
        const start = parseInt(ctx.query.start) || 0;
        const length = parseInt(ctx.query.length) || 10;
        const draw = ctx.query.draw;

        const result = await profile_service.server_time(ctx, app, start, length);
        ctx.body = Object.assign({}, result, {draw});
    },
    async error_report_index(ctx) {
        let model = new ErrorModel(ctx);

        let hostName = await model.getHost().then(r => r.map(fp.prop('hostname')));
        let app = await model.getApp().then(r => r.map(fp.prop('app')));
        let type = await model.getType().then(r => r.map(fp.prop('type')));
        let code = await model.getCode().then(r => r.map(fp.prop('code')));
        let route = await model.getRoute().then(r => r.map(fp.prop('route')));
        let api = await model.getApi().then(r => r.dmap(fp.prop('api')));

        await ctx.render('action/profile_error', {hostName, app, type, code, route, api});
    },
    async error_report_json(ctx) {
        const start = parseInt(ctx.query.start) || 0;
        const length = parseInt(ctx.query.length) || 10;
        const draw = ctx.query.draw;

        const app = ctx.query.app;
        const host = ctx.query.hostname;
        const type = ctx.query.type;
        const time = ctx.query.time;
        const code = ctx.query.code;
        const route = ctx.query.route;
        const api = ctx.query.api;

        const result = await profile_service.error(ctx, app, host, type, time, code, api, route, start, length);
        ctx.body = Object.assign({}, result, {draw});
    },
    async sort_route(ctx) {
        const result = await profile_service.getApp(ctx);

        await ctx.render('action/sort_route', {app: result})
    },
    async sort_route_list(ctx) {
        const app = ctx.query.app;
        const time = ctx.query.time;
        const result = await profile_service.sort_server_route(ctx, app, time);

        await ctx.render('action/sort_route_list', Object.assign({}, {
            layout: false
        }, result));
    },
    async sort_api(ctx) {
        const result = await profile_service.getApp(ctx);

        await ctx.render('action/sort_api', {app: result})
    },
    async sort_api_list(ctx) {
        const app = ctx.query.app;
        const time = ctx.query.time;
        const result = await profile_service.sort_server_api(ctx, app, time);

        await ctx.render('action/sort_api_list', Object.assign({}, {
            layout: false
        }, result));
    },
    async sort_client(ctx) {
        const result = await profile_service.getApp(ctx);

        await ctx.render('action/sort_client', {app: result})
    },
    async sort_client_list(ctx) {
        const app = ctx.query.app;
        const time = ctx.query.time;
        const result = await profile_service.sort_client_route(ctx, app, time);

        await ctx.render('action/sort_client_list', Object.assign({}, {
            layout: false
        }, result));
    }
};

r.get('/server', profile_controller.server_mean_report_index);
r.get('/server.json', profile_controller.server_mean_report_json);

r.get('/error', profile_controller.error_report_index);
r.get('/error.json', profile_controller.error_report_json);

r.get('/sort_route', profile_controller.sort_route);
r.post('/sort_route_list', profile_controller.sort_route_list);

r.get('/sort_api', profile_controller.sort_api);
r.post('/sort_api_list', profile_controller.sort_api_list);

r.get('/sort_client', profile_controller.sort_client);
r.post('/sort_client_list', profile_controller.sort_client_list);

module.exports = r;