sitemap-task.js
3.07 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
const ROOT_PATH = global.ROOT_PATH;
const os = require('os');
const schedule = require('node-schedule');
const rp = require('request-promise');
const TASK_KEY_LOG = 'global:yoho:task:log:sitemap';
const Promise = require('bluebird');
const _ = require('lodash');
const redis = require(`${ROOT_PATH}/libs/redis`);
const fs = Promise.promisifyAll(require('fs'));
const logger = global.yoho.logger;
const config = global.yoho.config;
const path = require('path');
const request = require('request-promise');
class SitemapTask {
constructor(name, type) {
this.type = type;
this.name = name;
this.isRunning = false;
this.logs = [];
schedule.scheduleJob('* * * * * 6', () => {
this.exec()
});
}
async init() {
this.isRunning = false;
this.exec();
}
async exec() {
await Promise.delay(10000);
this.logs = [];
logger.info(`${this.type} sitemap task start`);
if (this.isRunning) {
return;
}
const dir = config.sitemap[this.type].dir;
const host = config.sitemap[this.type].host;
const INDEX = `http://localhost:${config.port}/${this.type}/sitemap/sitemapindex.json`;
const URLREPLACE = [host + '/sitemap_' + this.type, `http://localhost:${config.port}/${this.type}/sitemap`];
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir)
}
this.isRunning = true;
let fileUrls = await request.get(INDEX, {
timeout: 5 * 60 * 1000
});
fileUrls = JSON.parse(fileUrls);
// let fileUrls = [];
// fileUrls.push(`${host}/sitemap_${this.type}/sitemapindex.xml`);
// fileUrls.push(`${host}/sitemap_${this.type}/productindex.xml`);
await Promise.map(fileUrls, async (url) => {
try {
logger.info(`processing ${url}`);
const newUrl = url.replace(URLREPLACE[0], URLREPLACE[1]);
const fileName = _.last(newUrl.split('/'));
const content = await request.get(newUrl, {
timeout: 5 * 60 * 1000
});
const localFile = path.join(dir, fileName);
await fs.writeFileSync(localFile, content, 'utf8');
logger.info(`end ${url}`);
this.logs.push({
time: Date.now(),
url,
status: 'ok'
});
} catch (e) {
this.logs.push({
time: Date.now(),
url,
status: 'fail'
});
logger.error(e);
}
}, {
concurrency: 1
});
this.isRunning = false;
logger.info('sitemap task end');
await this.addLog(TASK_KEY_LOG, this.logs)
}
async addLog(key, logs) {
try {
await redis.ltrimAsync(key, 0, 1000);
return redis.lpushAsync(key, ...logs);
} catch (e) {
logger.error(e);
}
}
}
module.exports = SitemapTask;