Showing
2 changed files
with
15 additions
and
1 deletions
@@ -43,9 +43,10 @@ const task = async(options, tick = 1) => { | @@ -43,9 +43,10 @@ const task = async(options, tick = 1) => { | ||
43 | return task(options, tick + 1); | 43 | return task(options, tick + 1); |
44 | } | 44 | } |
45 | }; | 45 | }; |
46 | -const lockTask = nodeLockup(task, config.delay); | ||
47 | 46 | ||
48 | module.exports = (ids, url = 'http://du.hupu.com/mapi/product/detail', params) => { | 47 | module.exports = (ids, url = 'http://du.hupu.com/mapi/product/detail', params) => { |
48 | + const lockTask = nodeLockup(task, config.delay); | ||
49 | + | ||
49 | return ids.map(id => lockTask(Object.assign({ | 50 | return ids.map(id => lockTask(Object.assign({ |
50 | productId: id, | 51 | productId: id, |
51 | source: 'boutiqueRecommend', | 52 | source: 'boutiqueRecommend', |
other/spider.js
0 → 100644
1 | +const spider = require('../libs/spider'); | ||
2 | + | ||
3 | +const spiderTask = async ids => { | ||
4 | + spider(ids).forEach(promise => promise.then(result => { | ||
5 | + if (result.status === 200) { | ||
6 | + console.log(`${result.data.detail.productId},${result.data.detail.articleNumber},${result.data.detail.title},${result.data.item.price / 100},${result.data.detail.soldNum},http://du.hupu.com/mdu/product/detail.html?id=${result.data.detail.productId}`); | ||
7 | + } | ||
8 | + })) | ||
9 | +} | ||
10 | + | ||
11 | +const allIds = Array.from(new Array(21500)).map((v, i) => i + 1) | ||
12 | + | ||
13 | +spiderTask(allIds) |
-
Please register or login to post a comment