spider-buyers.js
4.71 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
const dayjs = require('dayjs')
const _ = require('lodash')
require('../libs/logger')
const config = require('../config/index')
const MysqlAdapter = require('../libs/mysql')
const spider = require('../libs/spider')
const mysql = new MysqlAdapter(config.database.connect, config.database.database)
const RegDay = /(\d+)天前/
const RegDate = /(\d+)月(\d+)日/
let count = 0
const parseTime = (relativeTime) => {
const match = relativeTime.match(RegDay)
if (match) {
return dayjs().add(0 - match[1], 'day')
}
const matchDate = relativeTime.match(RegDate)
if (matchDate) {
return dayjs(`2018/${matchDate[1]}/${matchDate[2]}`)
}
return dayjs()
}
let time = Date.now()
let end = 0
const getLeftTime = () => {
let offset = (end || Date.now()) - time
let intervalTime = (offset / inx) / 1000
let leftTime = (19635 - inx) * intervalTime
const hour = parseInt(leftTime / 36000)
leftTime = leftTime % 3600
const mins = parseInt(leftTime / 60)
return `${hour}小时${mins}分钟${parseInt(leftTime % 60)}秒`
}
const spiderBuyers = async (productId, lsId = 0, page = 0) => {
const promises = spider([productId], 'https://du.hupu.com/mapi/product/lastSoldList', {
lastId: lsId
})
promises.forEach(promise => promise.then(result => {
if (result.status === 200) {
const {list, lastId} = result.data
list.forEach(info => {
mysql.insert('INSERT INTO `buyers` (`productId`, `nickName`, `icon`, `time`, `size`, `soldNum`, `price`) VALUES (:productId, :nickName, :icon, :time, :size, :soldNum, :price)', {
productId: productId,
nickName: info.buyer.userName,
icon: info.buyer.icon,
size: info.item.size,
time: parseTime(info.formatTime).format('YYYY-MM-DD'),
soldNum: products[productId].count,
price: products[productId].price,
})
})
console.clear()
console.log(`page: ${page}, count: ${products[productId].count}, lastId: ${lastId}, 进度: ${inx}/${19635}(${parseInt((page * 20 + list.length) / products[productId].count * 100)}%), 剩余: ${getLeftTime()}, productId: ${productId}`)
if (lastId) {
spiderBuyers(productId, lastId, page + 1)
} else {
end = Date.now()
start()
}
}
}))
}
const ids = [3,8,10,16,17,20,21,33,39,113,126,134,161,203,206,278,685,775,913,920,930,954,1154,1181,1188,1226,1541,2139,2188,2189,2200,2201,2284,2290,2291,2292,2294,2306,2307,2347,2348,2353,2354,2355,2356,2357,2358,2363,2390,2405,2453,2701,3428,3430,3494,3638,3655,3688,3736,3751,3869,3881,3941,4233,4371,5889,5936,6079,7705,8699,8707,8708,8722,8727,8741,8814,8815,8822,8846,8847,8865,8866,8868,8873,8874,8880,8881,8906,8920,8921,8924,8928,8929,8949,8960,8972,8981,8990,9002,9007,9008,9018,9027,9033,9043,9057,9074,9084,9104,9119,9179,9212,9224,9225,9235,9236,9240,9242,9249,9296,9309,9316,9333,9345,9356,9357,9391,9395,9405,9416,9419,9426,9429,9431,9451,9511,9534,9536,9565,9604,9670,9675,9680,9684,9754,9773,9819,9887,9888,9933,9937,9940,9946,9947,9948,9949,9951,9952,9957,9969,9970,9971,9982,10063,10097,10129,10130,10145,10207,10237,10238,10239,10240,10241,10274,10293,10307,10337,10338,10351,10407,10412,10417,10424,10437,10451,10484,10485,10486,10515,10567,10574,10579,10636,10637,10710,10762,10779,10787,10797,10798,10801,10881,10904,10941,10942,11058,11061,11063,11119,11132,11172,11213,11329,11330,11331,11373,11408,11415,11443,11446,11448,11520,11521,11538,11558,11568,11618,11621,11641,11653,11687,11723,11724,11746,11818,11826,11839,11842,11945,11989,12058,12225,12226,12238,12241,12366,12508,12548,12578,12622,12652,12766,12769,12859,12893,12951,12954,13014,13166,13270,13309,13325,13387,13612,13644,13704,13936,14058,14450,14522,14545,14546,14670,14683,14775,14886,14887,14888,14900,14917,14960,15081,15191,15192,15204,15205,15316,15357,15363,15538,15581,15676,15685,16161,16231,16256,16420,16423,17364,]
let inx = 2617
const products = {}
const start = () => {
const productId = inx
inx++
if (inx >= 19635) {
console.log('over!!!!!!')
return
}
if (ids.some(id => id === productId)) {
spider([productId]).forEach(promise => promise.then(result => {
mysql.update('UPDATE `buyers` SET `soldNum` = :soldNum, `price` = :price WHERE `productId` = :productId', {
productId: productId,
soldNum: result.data.detail.soldNum,
price: result.data.item.price
})
}));
start()
} else {
if (products[productId]) {
spiderBuyers(productId)
} else {
spider([productId]).forEach(promise => promise.then(result => {
products[productId] = {
count: _.get(result, 'data.detail.soldNum', 0),
price: _.get(result, 'data.item.price', 0)
}
spiderBuyers(productId)
}))
}
}
}
start()
//1164952