spider-buyers.js 5.09 KB
const dayjs = require('dayjs');
const _ = require('lodash');

require('../libs/logger');
const config = require('../config/index');
const MysqlAdapter = require('../libs/mysql');
const spider = require('../libs/spider');

const mysql = new MysqlAdapter(config.database.connect, config.database.database);

const RegDay = /(\d+)天前/;
const RegDate = /(\d+)(\d+)日/;


const parseTime = (relativeTime) => {
  const match = relativeTime.match(RegDay);

  if (match) {
    return dayjs().add(0 - match[1], 'day');
  }
  const matchDate = relativeTime.match(RegDate);

  if (matchDate) {
    return dayjs(`2018/${matchDate[1]}/${matchDate[2]}`);
  }
  return dayjs();
};

let time = Date.now();
let end = 0;

const getLeftTime = () => {
  let offset = (end || Date.now()) - time;
  let intervalTime = (offset / inx) / 1000;
  let leftTime = (19635 - inx) * intervalTime;
  const hour = parseInt(leftTime / 36000);

  leftTime = leftTime % 3600;
  const mins = parseInt(leftTime / 60);

  return `${hour}小时${mins}分钟${parseInt(leftTime % 60)}秒`;
};

const spiderBuyers = async(productId, lsId = 0, page = 0) => {
  const promises = spider([productId], 'https://du.hupu.com/mapi/product/lastSoldList', {
    lastId: lsId
  });

  promises.forEach(promise => promise.then(result => {
    if (result.status === 200) {
      const {list, lastId} = result.data;


      list.forEach(info => {
        mysql.insert('INSERT INTO `buyers` (`productId`, `nickName`, `icon`, `time`, `size`, `soldNum`, `price`) VALUES (:productId, :nickName, :icon, :time, :size, :soldNum, :price)', {
          productId: productId,
          nickName: info.buyer.userName,
          icon: info.buyer.icon,
          size: info.item.size,
          time: parseTime(info.formatTime).format('YYYY-MM-DD'),
          soldNum: products[productId].count,
          price: products[productId].price,
        });
      });

      console.clear();
      console.log(`page: ${page}, count: ${products[productId].count}, lastId: ${lastId}, 进度: ${inx}/${max}(${parseInt((page * 20 + list.length) / products[productId].count * 100)}%), 剩余: ${getLeftTime()}, productId: ${productId}`);
      if (lastId) {
        spiderBuyers(productId, lastId, page + 1);
      } else {
        end = Date.now();
        start();
      }
    }
  }));
};

const ids = [3, 8, 10, 16, 17, 20, 21, 33, 39, 113, 126, 134, 161, 203, 206, 278, 685, 775, 913, 920, 930, 954, 1154, 1181, 1188, 1226, 1541, 2139, 2188, 2189, 2200, 2201, 2284, 2290, 2291, 2292, 2294, 2306, 2307, 2347, 2348, 2353, 2354, 2355, 2356, 2357, 2358, 2363, 2390, 2405, 2453, 2701, 3428, 3430, 3494, 3638, 3655, 3688, 3736, 3751, 3869, 3881, 3941, 4233, 4371, 5889, 5936, 6079, 7705, 8699, 8707, 8708, 8722, 8727, 8741, 8814, 8815, 8822, 8846, 8847, 8865, 8866, 8868, 8873, 8874, 8880, 8881, 8906, 8920, 8921, 8924, 8928, 8929, 8949, 8960, 8972, 8981, 8990, 9002, 9007, 9008, 9018, 9027, 9033, 9043, 9057, 9074, 9084, 9104, 9119, 9179, 9212, 9224, 9225, 9235, 9236, 9240, 9242, 9249, 9296, 9309, 9316, 9333, 9345, 9356, 9357, 9391, 9395, 9405, 9416, 9419, 9426, 9429, 9431, 9451, 9511, 9534, 9536, 9565, 9604, 9670, 9675, 9680, 9684, 9754, 9773, 9819, 9887, 9888, 9933, 9937, 9940, 9946, 9947, 9948, 9949, 9951, 9952, 9957, 9969, 9970, 9971, 9982, 10063, 10097, 10129, 10130, 10145, 10207, 10237, 10238, 10239, 10240, 10241, 10274, 10293, 10307, 10337, 10338, 10351, 10407, 10412, 10417, 10424, 10437, 10451, 10484, 10485, 10486, 10515, 10567, 10574, 10579, 10636, 10637, 10710, 10762, 10779, 10787, 10797, 10798, 10801, 10881, 10904, 10941, 10942, 11058, 11061, 11063, 11119, 11132, 11172, 11213, 11329, 11330, 11331, 11373, 11408, 11415, 11443, 11446, 11448, 11520, 11521, 11538, 11558, 11568, 11618, 11621, 11641, 11653, 11687, 11723, 11724, 11746, 11818, 11826, 11839, 11842, 11945, 11989, 12058, 12225, 12226, 12238, 12241, 12366, 12508, 12548, 12578, 12622, 12652, 12766, 12769, 12859, 12893, 12951, 12954, 13014, 13166, 13270, 13309, 13325, 13387, 13612, 13644, 13704, 13936, 14058, 14450, 14522, 14545, 14546, 14670, 14683, 14775, 14886, 14887, 14888, 14900, 14917, 14960, 15081, 15191, 15192, 15204, 15205, 15316, 15357, 15363, 15538, 15581, 15676, 15685, 16161, 16231, 16256, 16420, 16423, 17364];


const products = {};

let max = 23000;
let inx = 21046;
const start = () => {
  const productId = inx;

  inx++;
  if (inx >= max) {
    console.log('over!!!!!!');
    return;
  }
  // if (ids.some(id => id === productId)) {

  //   spider([productId]).forEach(promise => promise.then(result => {
  //     mysql.update('UPDATE `buyers` SET `soldNum` = :soldNum, `price` = :price WHERE `productId` = :productId', {
  //       productId: productId,
  //       soldNum: result.data.detail.soldNum,
  //       price: result.data.item.price
  //     });
  //   }));
  //   start();
  // } else {
    if (products[productId]) {
      spiderBuyers(productId);
    } else {
      spider([productId]).forEach(promise => promise.then(result => {
        products[productId] = {
          count: _.get(result, 'data.detail.soldNum', 0),
          price: _.get(result, 'data.item.price', 0)
        };
        spiderBuyers(productId);
      }));
    }
  // }
};

start();

// 1164952