guang-service.js
2.26 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
const _ = require('lodash');
const GuangModel = require('./guang-api');
const {MAX_URL, PAGE_SIZE} = require('./vars');
const logger = global.yoho.logger;
const tpmUrl = id => `/guang/${id}.html`;
const indexUrl = (type, page) => `/sitemap_${type}/guang_${page}.xml`;
module.exports = class extends global.yoho.BaseModel {
constructor(ctx) {
super(ctx);
this.guangModel = new GuangModel(ctx);
}
async getIndex(type) {
const {data, error} = await this._getList(1);
if (error) {
return [];
}
const {total} = data;
const page = Math.ceil(total / MAX_URL);
const urls = _.range(1, page + 1).map((i) => indexUrl(type, i));
return urls;
}
// 从第 1 页开始
async getUrls(page) {
let startPage = ((page - 1) * MAX_URL / PAGE_SIZE) + 1;
let endPage = (page * MAX_URL / PAGE_SIZE) + 1;
const firstPage = await this._getList(1);
if (firstPage.error) {
return [];
}
if (startPage > firstPage.data.totalPage) {
startPage = firstPage.data.totalPage;
}
if (endPage > firstPage.data.totalPage) {
endPage = firstPage.data.totalPage;
}
let urls = [];
await Promise.map(_.range(startPage, endPage), async (i) => {
const result = await this._getList(i);
if (result.error) {
return;
}
urls = _.concat(urls, result.data.urls);
}, {concurrency: 1});
return urls;
}
async _getList(page) {
const result = await this.guangModel.getList(page);
if (result.code !== 200) {
logger.error('get guang page %d error %d', page, result.code);
return {error: Error()};
}
const artList = _.get(result, 'data.list.artList', []);
const total = _.get(result, 'data.total', 0);
const totalPage = _.get(result, 'data.totalPage', 0);
logger.info('get guang count %d total page %d', total, totalPage);
const urls = artList.map(({id}) => {
return tpmUrl(id);
});
return {
data: {
urls,
total,
totalPage
}
};
}
};