spider-buyers.js 4.71 KB
const dayjs = require('dayjs')
const _ = require('lodash')

require('../libs/logger')
const config = require('../config/index')
const MysqlAdapter = require('../libs/mysql')
const spider = require('../libs/spider')

const mysql = new MysqlAdapter(config.database.connect, config.database.database)

const RegDay = /(\d+)天前/
const RegDate = /(\d+)(\d+)日/

let count = 0

const parseTime = (relativeTime) => {
  const match = relativeTime.match(RegDay)

  if (match) {
    return dayjs().add(0 - match[1], 'day')
  }
  const matchDate = relativeTime.match(RegDate)

  if (matchDate) {
    return dayjs(`2018/${matchDate[1]}/${matchDate[2]}`)
  }
  return dayjs()
}

let time = Date.now()
let end = 0

const getLeftTime = () => {
  let offset = (end || Date.now()) - time
  let intervalTime = (offset / inx) / 1000
  let leftTime = (19635 - inx) * intervalTime
  const hour = parseInt(leftTime / 36000)
  leftTime = leftTime % 3600
  const mins = parseInt(leftTime / 60)
  return `${hour}小时${mins}分钟${parseInt(leftTime % 60)}秒`
}

const spiderBuyers = async (productId, lsId = 0, page = 0) => {
  const promises = spider([productId], 'https://du.hupu.com/mapi/product/lastSoldList', {
    lastId: lsId
  })

  promises.forEach(promise => promise.then(result => {
    if (result.status === 200) {
      const {list, lastId} = result.data


      list.forEach(info => {
        mysql.insert('INSERT INTO `buyers` (`productId`, `nickName`, `icon`, `time`, `size`, `soldNum`, `price`) VALUES (:productId, :nickName, :icon, :time, :size, :soldNum, :price)', {
          productId: productId,
          nickName: info.buyer.userName,
          icon: info.buyer.icon,
          size: info.item.size,
          time: parseTime(info.formatTime).format('YYYY-MM-DD'),
          soldNum: products[productId].count,
          price: products[productId].price,
        })
      })

      console.clear()
      console.log(`page: ${page}, count: ${products[productId].count}, lastId: ${lastId}, 进度: ${inx}/${19635}(${parseInt((page * 20 + list.length) / products[productId].count * 100)}%), 剩余: ${getLeftTime()}, productId: ${productId}`)
      if (lastId) {
        spiderBuyers(productId, lastId, page + 1)
      } else {
        end = Date.now()
        start()
      }
    }
  }))
}
const ids = [3,8,10,16,17,20,21,33,39,113,126,134,161,203,206,278,685,775,913,920,930,954,1154,1181,1188,1226,1541,2139,2188,2189,2200,2201,2284,2290,2291,2292,2294,2306,2307,2347,2348,2353,2354,2355,2356,2357,2358,2363,2390,2405,2453,2701,3428,3430,3494,3638,3655,3688,3736,3751,3869,3881,3941,4233,4371,5889,5936,6079,7705,8699,8707,8708,8722,8727,8741,8814,8815,8822,8846,8847,8865,8866,8868,8873,8874,8880,8881,8906,8920,8921,8924,8928,8929,8949,8960,8972,8981,8990,9002,9007,9008,9018,9027,9033,9043,9057,9074,9084,9104,9119,9179,9212,9224,9225,9235,9236,9240,9242,9249,9296,9309,9316,9333,9345,9356,9357,9391,9395,9405,9416,9419,9426,9429,9431,9451,9511,9534,9536,9565,9604,9670,9675,9680,9684,9754,9773,9819,9887,9888,9933,9937,9940,9946,9947,9948,9949,9951,9952,9957,9969,9970,9971,9982,10063,10097,10129,10130,10145,10207,10237,10238,10239,10240,10241,10274,10293,10307,10337,10338,10351,10407,10412,10417,10424,10437,10451,10484,10485,10486,10515,10567,10574,10579,10636,10637,10710,10762,10779,10787,10797,10798,10801,10881,10904,10941,10942,11058,11061,11063,11119,11132,11172,11213,11329,11330,11331,11373,11408,11415,11443,11446,11448,11520,11521,11538,11558,11568,11618,11621,11641,11653,11687,11723,11724,11746,11818,11826,11839,11842,11945,11989,12058,12225,12226,12238,12241,12366,12508,12548,12578,12622,12652,12766,12769,12859,12893,12951,12954,13014,13166,13270,13309,13325,13387,13612,13644,13704,13936,14058,14450,14522,14545,14546,14670,14683,14775,14886,14887,14888,14900,14917,14960,15081,15191,15192,15204,15205,15316,15357,15363,15538,15581,15676,15685,16161,16231,16256,16420,16423,17364,]
let inx = 2617

const products = {}

const start = () => {
  const productId = inx

  inx++
  if (inx >= 19635) {
    console.log('over!!!!!!')
    return
  }
  if (ids.some(id => id === productId)) {

    spider([productId]).forEach(promise => promise.then(result => {
      mysql.update('UPDATE `buyers` SET `soldNum` = :soldNum, `price` = :price WHERE `productId` = :productId', {
        productId: productId,
        soldNum: result.data.detail.soldNum,
        price: result.data.item.price
      })
    }));
    start()
  } else {
    if (products[productId]) {
      spiderBuyers(productId)
    } else {
      spider([productId]).forEach(promise => promise.then(result => {
        products[productId] = {
          count: _.get(result, 'data.detail.soldNum', 0),
          price: _.get(result, 'data.item.price', 0)
        }
        spiderBuyers(productId)
      }))
    }
  }
}
start()

//1164952