spider.js
1.69 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
const md5 = require('yoho-md5');
const rp = require('request-promise');
const nodeLockup = require('node-lockup');
const chalk = require('chalk');
const {logger} = require('./logger');
const config = require('../config/index');
const sign = (obj) => {
let constr = '';
Object.keys(obj).sort().forEach(k => {
constr += k + obj[k].toString();
});
return md5(constr + '048a9c4943398714b356a696503d2d36');
};
const task = async(options, tick = 1) => {
const params = Object.assign({}, options);
delete params.url;
params.sign = sign(params);
try {
const result = await rp.get({
url: options.url,
qs: params,
json: true,
headers: {
Accept: 'application/json, text/plain, */*',
appVersion: '3.5.0',
Referer: 'http://du.hupu.com/mdu/product/detail.html?id=1&source=boutiqueRecommend',
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1',
'X-Requested-With': 'XMLHttpRequest'
}
});
return result;
} catch (error) {
logger.error(chalk.red(`error:${options}:tick:${tick} ====> ${error}`));
if (tick >= 3) {
return Promise.resolve({});
}
return task(options, tick + 1);
}
};
module.exports = (ids, url = 'http://du.hupu.com/mapi/product/detail', params, delay) => {
const lockTask = nodeLockup(task, delay || config.delay);
return ids.map(id => lockTask(Object.assign({
productId: id,
source: 'boutiqueRecommend',
url
}, params)));
};
module.exports.spiderFetch = (id, url = 'http://du.hupu.com/mapi/product/detail', params) => {
return task(Object.assign({
productId: id,
source: 'boutiqueRecommend',
url
}, params));
};