spider-buyers.js
5.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
const dayjs = require('dayjs');
const _ = require('lodash');
require('../libs/logger');
const config = require('../config/index');
const MysqlAdapter = require('../libs/mysql');
const spider = require('../libs/spider');
const mysql = new MysqlAdapter(config.database.connect, config.database.database);
const RegDay = /(\d+)天前/;
const RegDate = /(\d+)月(\d+)日/;
let inx = 2617;
const parseTime = (relativeTime) => {
const match = relativeTime.match(RegDay);
if (match) {
return dayjs().add(0 - match[1], 'day');
}
const matchDate = relativeTime.match(RegDate);
if (matchDate) {
return dayjs(`2018/${matchDate[1]}/${matchDate[2]}`);
}
return dayjs();
};
let time = Date.now();
let end = 0;
const getLeftTime = () => {
let offset = (end || Date.now()) - time;
let intervalTime = (offset / inx) / 1000;
let leftTime = (19635 - inx) * intervalTime;
const hour = parseInt(leftTime / 36000);
leftTime = leftTime % 3600;
const mins = parseInt(leftTime / 60);
return `${hour}小时${mins}分钟${parseInt(leftTime % 60)}秒`;
};
const spiderBuyers = async(productId, lsId = 0, page = 0) => {
const promises = spider([productId], 'https://du.hupu.com/mapi/product/lastSoldList', {
lastId: lsId
});
promises.forEach(promise => promise.then(result => {
if (result.status === 200) {
const {list, lastId} = result.data;
list.forEach(info => {
mysql.insert('INSERT INTO `buyers` (`productId`, `nickName`, `icon`, `time`, `size`, `soldNum`, `price`) VALUES (:productId, :nickName, :icon, :time, :size, :soldNum, :price)', {
productId: productId,
nickName: info.buyer.userName,
icon: info.buyer.icon,
size: info.item.size,
time: parseTime(info.formatTime).format('YYYY-MM-DD'),
soldNum: products[productId].count,
price: products[productId].price,
});
});
console.clear();
console.log(`page: ${page}, count: ${products[productId].count}, lastId: ${lastId}, 进度: ${inx}/${19635}(${parseInt((page * 20 + list.length) / products[productId].count * 100)}%), 剩余: ${getLeftTime()}, productId: ${productId}`);
if (lastId) {
spiderBuyers(productId, lastId, page + 1);
} else {
end = Date.now();
start();
}
}
}));
};
const ids = [3, 8, 10, 16, 17, 20, 21, 33, 39, 113, 126, 134, 161, 203, 206, 278, 685, 775, 913, 920, 930, 954, 1154, 1181, 1188, 1226, 1541, 2139, 2188, 2189, 2200, 2201, 2284, 2290, 2291, 2292, 2294, 2306, 2307, 2347, 2348, 2353, 2354, 2355, 2356, 2357, 2358, 2363, 2390, 2405, 2453, 2701, 3428, 3430, 3494, 3638, 3655, 3688, 3736, 3751, 3869, 3881, 3941, 4233, 4371, 5889, 5936, 6079, 7705, 8699, 8707, 8708, 8722, 8727, 8741, 8814, 8815, 8822, 8846, 8847, 8865, 8866, 8868, 8873, 8874, 8880, 8881, 8906, 8920, 8921, 8924, 8928, 8929, 8949, 8960, 8972, 8981, 8990, 9002, 9007, 9008, 9018, 9027, 9033, 9043, 9057, 9074, 9084, 9104, 9119, 9179, 9212, 9224, 9225, 9235, 9236, 9240, 9242, 9249, 9296, 9309, 9316, 9333, 9345, 9356, 9357, 9391, 9395, 9405, 9416, 9419, 9426, 9429, 9431, 9451, 9511, 9534, 9536, 9565, 9604, 9670, 9675, 9680, 9684, 9754, 9773, 9819, 9887, 9888, 9933, 9937, 9940, 9946, 9947, 9948, 9949, 9951, 9952, 9957, 9969, 9970, 9971, 9982, 10063, 10097, 10129, 10130, 10145, 10207, 10237, 10238, 10239, 10240, 10241, 10274, 10293, 10307, 10337, 10338, 10351, 10407, 10412, 10417, 10424, 10437, 10451, 10484, 10485, 10486, 10515, 10567, 10574, 10579, 10636, 10637, 10710, 10762, 10779, 10787, 10797, 10798, 10801, 10881, 10904, 10941, 10942, 11058, 11061, 11063, 11119, 11132, 11172, 11213, 11329, 11330, 11331, 11373, 11408, 11415, 11443, 11446, 11448, 11520, 11521, 11538, 11558, 11568, 11618, 11621, 11641, 11653, 11687, 11723, 11724, 11746, 11818, 11826, 11839, 11842, 11945, 11989, 12058, 12225, 12226, 12238, 12241, 12366, 12508, 12548, 12578, 12622, 12652, 12766, 12769, 12859, 12893, 12951, 12954, 13014, 13166, 13270, 13309, 13325, 13387, 13612, 13644, 13704, 13936, 14058, 14450, 14522, 14545, 14546, 14670, 14683, 14775, 14886, 14887, 14888, 14900, 14917, 14960, 15081, 15191, 15192, 15204, 15205, 15316, 15357, 15363, 15538, 15581, 15676, 15685, 16161, 16231, 16256, 16420, 16423, 17364];
const products = {};
const start = () => {
const productId = inx;
inx++;
if (inx >= 19635) {
console.log('over!!!!!!');
return;
}
if (ids.some(id => id === productId)) {
spider([productId]).forEach(promise => promise.then(result => {
mysql.update('UPDATE `buyers` SET `soldNum` = :soldNum, `price` = :price WHERE `productId` = :productId', {
productId: productId,
soldNum: result.data.detail.soldNum,
price: result.data.item.price
});
}));
start();
} else {
if (products[productId]) {
spiderBuyers(productId);
} else {
spider([productId]).forEach(promise => promise.then(result => {
products[productId] = {
count: _.get(result, 'data.detail.soldNum', 0),
price: _.get(result, 'data.item.price', 0)
};
spiderBuyers(productId);
}));
}
}
};
start();
// 1164952