price-task.js 4.5 KB
const dayjs = require('dayjs');
const nodeLockup = require('node-lockup');
const spider = require('../libs/spider');
const _ = require('lodash');
const {sendMessage} = require('../libs/influx-report');
const {ShenjianModel, DuDataModel} = require('../models');
const {logger} = require('../libs/logger');
const fs = require('fs');
const sjApi = require('../libs/sj-api');


// const REG_HOUR = /^[1,2]小时/;
// const REG_MINUES = /^\d+分钟/;

// const fetchBuys = async(productId, lsId = 0) => {
//   let lastList = [];
//   const result = await spider.spiderFetch(productId, 'https://du.hupu.com/mapi/product/lastSoldList', {
//     lastId: lsId
//   });

//   if (result.status === 200) {
//     const {list, lastId} = result.data;

//     lastList = list.filter(info => {
//       return REG_HOUR.test(info.formatTime) || REG_MINUES.test(info.formatTime) || info.formatTime.indexOf('刚刚') >= 0;
//     });

//     if (lastList.length) {
//       const nextList = await fetchBuys(productId, lastId);

//       lastList = lastList.concat(nextList);
//     }
//   }

//   return lastList;
// };

// const processBuys = async(data) => {
//   const allList = await fetchBuys(data.productId);

//   data.sizes.forEach(size => {
//     const buys = allList.filter(info => {
//       return info.item && info.item.size === size.size;
//     });

//     size.soldNum += buys.length;
//   });

//   return data;
// };

const fetchAllSj = async(__id) => {
  let products = [];
  const fields = '__id,product_id,name,params,shop_id';
  const res = await sjApi(`source(__id:{lt: ${__id}}, limit: 2000){data{${fields}},page_info{end_cursor,has_next_page}}`);

  if (res.code === 0) {
    products = res.result.data;

    if (res.result.has_next_page) {
      const nextData = await fetchAllSj(res.result.end_cursor);

      products = products.concat(nextData);
    }
  }
  return products;
};

const refreshShenjian = async() => {
  try {
    const products = await fetchAllSj(999999);
    const filterData = products.map(product => {
      const modelParams = product.params.find(p => p.label === '款号') ||
        product.params.find(p => p.label === '货号');

      if (modelParams) {
        return {
          product_id: product.product_id,
          id: product.__id,
          model: modelParams.value
        };
      }
      return void 0;
    }).filter(p => p);

    if (filterData.length) {
      await ShenjianModel.remove({});
      await ShenjianModel.insert(...filterData);
    }

    sendMessage(filterData.length, new Date().getTime() * 1000000, 'price-task-shenjian');
    logger.info(`[price-task] fetch shenjian ${filterData.length}`);
  } catch (error) {
    logger.error(error);
  }
};
const task = async(id) => {
  const result = await spider.spiderFetch(id);

  if (result.status === 200) {
    const {detail, item} = result.data;

    const productId = detail.productId;
    const soldNum = detail.soldNum;
    const articleNumber = detail.articleNumber;
    const price = (item.price || 0) / 100;
    const title = detail.title;

    return {
      productId,
      soldNum,
      articleNumber,
      price,
      title,
      brandId: detail.brandId
    };
  }
};

module.exports = async() => {
  const now = dayjs().format('YYYY-MM-DD');
  const fw = fs.createWriteStream(`/Data/logs/node/prices/${now}.log`, {
    flags: 'a'
  });
  const lockTask = nodeLockup(task, 1);

  const allData = await DuDataModel.findAll();
  let maxProduct = _.get(_.maxBy(allData, 'productId'), 'productId', 0);

  if (maxProduct < 24000) {
    maxProduct = 24000;
  }

  const ids = Array.from(new Array(maxProduct + 500)).map((v, i) => i + 1);

  logger.info(`[price-task] maxproductId: ${maxProduct + 500}`);

  refreshShenjian();
  try {
    let update = 0, added = 0;

    ids.forEach((id, inx) => {
      lockTask(id).then(async result => {
        if (result) {

          logger.info(`[price-task]time:${now}; productId:${id}; soldNum:${result.soldNum};`);

          const find = await DuDataModel.findOne({productId: id});

          if (find) {
            DuDataModel.update({productId: id}, result);
            update++;
          } else {
            DuDataModel.insert(result);
            added++;
          }

          fw.write(`${JSON.stringify(result)}\n`);
        }
        if (inx >= ids.length - 1) {
          fw.end('');
          sendMessage(update, new Date().getTime() * 1000000, 'price-task-update');
          sendMessage(added, new Date().getTime() * 1000000, 'price-task-added');
        }
      });
    });
  } catch (error) {
    fw.end(error.toString());
  }
};