price-task.js
4.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
const dayjs = require('dayjs');
const nodeLockup = require('node-lockup');
const spider = require('../libs/spider');
const _ = require('lodash');
const {sendMessage} = require('../libs/influx-report');
const {ShenjianModel, DuDataModel} = require('../models');
const {logger} = require('../libs/logger');
const fs = require('fs');
const sjApi = require('../libs/sj-api');
// const REG_HOUR = /^[1,2]小时/;
// const REG_MINUES = /^\d+分钟/;
// const fetchBuys = async(productId, lsId = 0) => {
// let lastList = [];
// const result = await spider.spiderFetch(productId, 'https://du.hupu.com/mapi/product/lastSoldList', {
// lastId: lsId
// });
// if (result.status === 200) {
// const {list, lastId} = result.data;
// lastList = list.filter(info => {
// return REG_HOUR.test(info.formatTime) || REG_MINUES.test(info.formatTime) || info.formatTime.indexOf('刚刚') >= 0;
// });
// if (lastList.length) {
// const nextList = await fetchBuys(productId, lastId);
// lastList = lastList.concat(nextList);
// }
// }
// return lastList;
// };
// const processBuys = async(data) => {
// const allList = await fetchBuys(data.productId);
// data.sizes.forEach(size => {
// const buys = allList.filter(info => {
// return info.item && info.item.size === size.size;
// });
// size.soldNum += buys.length;
// });
// return data;
// };
const fetchAllSj = async(__id) => {
let products = [];
const fields = '__id,product_id,name,params,shop_id';
const res = await sjApi(`source(__id:{lt: ${__id}}, limit: 2000){data{${fields}},page_info{end_cursor,has_next_page}}`);
if (res.code === 0) {
products = res.result.data;
if (res.result.has_next_page) {
const nextData = await fetchAllSj(res.result.end_cursor);
products = products.concat(nextData);
}
}
return products;
};
const refreshShenjian = async() => {
try {
const products = await fetchAllSj(999999);
const filterData = products.map(product => {
const modelParams = product.params.find(p => p.label === '款号') ||
product.params.find(p => p.label === '货号');
if (modelParams) {
return {
product_id: product.product_id,
id: product.__id,
model: modelParams.value
};
}
return void 0;
}).filter(p => p);
if (filterData.length) {
await ShenjianModel.remove({});
await ShenjianModel.insert(...filterData);
}
sendMessage(filterData.length, new Date().getTime() * 1000000, 'price-task-shenjian');
logger.info(`[price-task] fetch shenjian ${filterData.length}`);
} catch (error) {
logger.error(error);
}
};
const task = async(id) => {
const result = await spider.spiderFetch(id);
if (result.status === 200) {
const {detail, item} = result.data;
const productId = detail.productId;
const soldNum = detail.soldNum;
const articleNumber = detail.articleNumber;
const price = (item.price || 0) / 100;
const title = detail.title;
return {
productId,
soldNum,
articleNumber,
price,
title,
brandId: detail.brandId
};
}
};
module.exports = async() => {
const now = dayjs().format('YYYY-MM-DD');
const fw = fs.createWriteStream(`/Data/logs/node/prices/${now}.log`, {
flags: 'a'
});
const lockTask = nodeLockup(task, 1);
const allData = await DuDataModel.findAll();
let maxProduct = _.get(_.maxBy(allData, 'productId'), 'productId', 0);
if (maxProduct < 24000) {
maxProduct = 24000;
}
const ids = Array.from(new Array(maxProduct + 500)).map((v, i) => i + 1);
logger.info(`[price-task] maxproductId: ${maxProduct + 500}`);
refreshShenjian();
try {
let update = 0, added = 0;
ids.forEach((id, inx) => {
lockTask(id).then(async result => {
if (result) {
logger.info(`[price-task]time:${now}; productId:${id}; soldNum:${result.soldNum};`);
const find = await DuDataModel.findOne({productId: id});
if (find) {
DuDataModel.update({productId: id}, result);
update++;
} else {
DuDataModel.insert(result);
added++;
}
fw.write(`${JSON.stringify(result)}\n`);
}
if (inx >= ids.length - 1) {
fw.end('');
sendMessage(update, new Date().getTime() * 1000000, 'price-task-update');
sendMessage(added, new Date().getTime() * 1000000, 'price-task-added');
}
});
});
} catch (error) {
fw.end(error.toString());
}
};