spider-buyers.js 4.44 KB
const dayjs = require('dayjs');
const lockup = require('node-lockup');
const isBetween = require('dayjs/plugin/isBetween');
const _ = require('lodash');

require('../libs/logger');
const config = require('../config/index');
const MysqlAdapter = require('../libs/mysql');
const spider = require('../libs/spider');

dayjs.extend(isBetween)

const mysql = new MysqlAdapter(config.database.connect, config.database.database);

const RegNow = /刚刚/;
const RegMinute = /(\d+)分钟前/;
const RegHour = /(\d+)小时前/;
const RegDay = /(\d+)天前/;
const RegDate = /(\d+)(\d+)日/;
const RegDateFull = /\d+\.\d+\.\d+/;


const parseTime = (relativeTime) => {
  const matchMinute = relativeTime.match(RegMinute);

  if (matchMinute) {
    return dayjs().add(0 - matchMinute[1], 'minute');
  }
  const matchHour = relativeTime.match(RegHour);

  if (matchHour) {
    return dayjs().add(0 - matchHour[1], 'hour');
  }
  const matchDay = relativeTime.match(RegDay);

  if (matchDay) {
    return dayjs().add(0 - matchDay[1], 'day');
  }
  const matchDate = relativeTime.match(RegDate);

  if (matchDate) {
    return dayjs(`2019/${matchDate[1]}/${matchDate[2]}`);
  }
  if (RegDateFull.test(relativeTime)) {
    return dayjs(relativeTime);
  }
  if (RegNow.test(relativeTime)) {
    return dayjs();
  }
  return dayjs('1990-01-01');
};

let time = Date.now();
let end = 0;

const getLeftTime = () => {
  let offset = (end || Date.now()) - time;
  let intervalTime = (offset / inx) / 1000;
  let leftTime = (19635 - inx) * intervalTime;
  const hour = parseInt(leftTime / 36000);

  leftTime = leftTime % 3600;
  const mins = parseInt(leftTime / 60);

  return `${hour}小时${mins}分钟${parseInt(leftTime % 60)}秒`;
};

const spiderBuyers = async(productId, lsId = 0, page = 0, trys = 0) => {
  const result = await spider.spiderFetch(productId, 'https://du.hupu.com/mapi/product/lastSoldList', {
    lastId: lsId
  });

  let lastId;
  let tryLogic = false;
  let list = [];
  if (result.status === 200) {
    let skip = false;
    lastId = result.data.lastId;
    list = result.data.list;


    list.forEach(info => {
      const time = parseTime(info.formatTime);
      const startTime = dayjs().subtract(1, 'day').add(1, 'second');
      
      if (!time.isBefore(startTime)) {
        mysql.insert('INSERT INTO `buyers_copy` (`productId`, `nickName`, `icon`, `time`, `size`, `soldNum`, `price`) VALUES (:productId, :nickName, :icon, :time, :size, :soldNum, :price)', {
          productId: productId,
          nickName: info.buyer.userName,
          icon: info.buyer.icon,
          size: info.item.size,
          time: time.format('YYYY-MM-DD HH:mm:ss'),
          soldNum: products[productId].count,
          price: products[productId].price,
        });
      } else {
        if (!skip) {
          console.log(time.format('YYYY-MM-DD HH:mm:ss'), startTime.format('YYYY-MM-DD HH:mm:ss'))
        }
        skip = true;
      }
    });

    if (skip) {
      return {productId};
    }
  } else {
    if (trys < 3) {
      tryLogic = true;
    }
  }

  if (tryLogic) {
    console.log(`tryLogic ${trys}`)
    return spiderBuyers(productId, lsId, page, trys + 1);
  } else {
    if (lastId) {
      console.log(`spiderBuyers ${lastId}, ${page}`)
      return spiderBuyers(productId, lastId, page + 1);
    }
    return {productId};
  }
};

//2721219
const products = {};

let max = 34200;
let failNum = 0;
let inx = 1;
let over = false;

const start = async() => {
  if (over) {
    return;
  }
  const productId = inx;
  console.log(`start 1 , ${productId}`)

  inx++;
  if (inx >= max) {
    over = true;
    console.log('over!!!!!!');
    return;
  }
  
  const result = await spider.spiderFetch(productId);
  console.log(`start 2 , ${productId}`)

  if (result.status === 200) {
    failNum = 0;
    products[productId] = {
      count: _.get(result, 'data.detail.soldNum', 0),
      price: _.get(result, 'data.item.price', 0)
    };
    try {
      const bl = await spiderBuyers(productId);
      console.log(`start 3 , ${productId}`)

      if (bl) {
        console.log(JSON.stringify(bl));
      }
    } catch (error) {
      console.log('error', error);
    }
    

  } else {
    console.log(productId, result.status)
    if (result.status) {
      failNum++;
    }
  }
  if (failNum >= 20) {
    over = true;
    return console.log('over!!!!!!', JSON.stringify(result));
  }
  return true;
};

const locktask = lockup(start);


module.exports = () => {
  Array.from({length: max}).map((v, i) => {
    locktask();
  })
}