sitemap-task.js 3.34 KB
const ROOT_PATH = global.ROOT_PATH;
const os = require('os');
const schedule = require('node-schedule');
const rp = require('request-promise');
const TASK_KEY_LOG = 'global:yoho:task:log:sitemap';
const Promise = require('bluebird');
const _ = require('lodash');
const redis = require(`${ROOT_PATH}/libs/redis`);
const fs = Promise.promisifyAll(require('fs'));
const logger = global.yoho.logger;
const config = global.yoho.config;
const path = require('path');

const request = require('request-promise');

class SitemapTask {
    constructor(name, type) {
        this.type = type;
        this.name = name;
        this.isRunning = false;
        this.logs = [];

        // 每周六 1:00:00 执行任务
        schedule.scheduleJob('0 0 1 * * 6', () => {
            this.exec()
        });
    }

    async init() {
        this.isRunning = false;
        this.exec();
    }

    async exec() {
        await Promise.delay(10000);

        this.logs = [];

        logger.info(`${this.type} sitemap task start`);

        if (this.isRunning) {
            return;
        }

        const dir = config.sitemap[this.type].dir;
        const host = config.sitemap[this.type].host;

        const INDEX = `http://localhost:${config.port}/${this.type}/sitemap/sitemapindex.json`;
        const URLREPLACE = [host + '/sitemap_' + this.type, `http://localhost:${config.port}/${this.type}/sitemap`];

        if (!fs.existsSync(dir)) {
            fs.mkdirSync(dir)
        }

        this.isRunning = true;

        let fileUrls = await request.get(INDEX, {
            timeout: 5 * 60 * 1000
        });

        fileUrls = JSON.parse(fileUrls);

        // let fileUrls = [];

        fileUrls.push(`${host}/sitemap_${this.type}/sitemapindex.xml`);
        fileUrls.push(`${host}/sitemap_${this.type}/productindex.xml`);

        // fileUrls.push(`${host}/sitemap_${this.type}/hot_1.xml`);

        logger.info('start task [%s]', JSON.stringify(fileUrls));

        await Promise.map(fileUrls, async (url) => {
            try {
                const newUrl = url.replace(URLREPLACE[0], URLREPLACE[1]);
                const fileName = _.last(newUrl.split('/'));
                const content = await request.get(newUrl, {
                    timeout: 5 * 60 * 1000
                });
                const localFile = path.join(dir, fileName);

                await fs.writeFileSync(localFile, content, 'utf8');

                logger.info(`process type:[${this.type}] status:[200] ${url}`);
                this.logs.push({
                    time: Date.now(),
                    url,
                    status: 'ok'
                });
            } catch (e) {
                this.logs.push({
                    time: Date.now(),
                    url,
                    status: 'fail'
                });

                logger.info(`process type:[${this.type}] status:[500] ${url} ${e}`);
            }
        }, {
            concurrency: 1
        });

        this.isRunning = false;
        logger.info(`${this.type} sitemap task end`);

        await this.addLog(TASK_KEY_LOG, this.logs)
    }

    async addLog(key, logs) {
        try {
            logs = logs.map(i => JSON.stringify(i));
            await redis.ltrimAsync(key, 0, 1000);
            return redis.lpushAsync(key, ...logs);
        } catch (e) {
            logger.error(e);

        }
    }
}

module.exports = SitemapTask;