spider-task.js 1.96 KB
const _ = require('lodash');

const spider = require('../libs/spider');
const report = require('../libs/report');
const sknsData = require('../data/skns.json');
const {mysqlPool} = require('../libs/mysql');
const {logger} = require('../libs/logger');
const {sendMessage} = require('../libs/influx-report');

module.exports = async(time) => {
  const sizeData = await mysqlPool.query('select `size_name`,min(`id`) as id from `size` where `size_name` <> \'-\' group by `size_name`');
  const sizeRelation = {};

  sizeData.forEach(size => {
    sizeRelation[_.trim(size.size_name)] = {
      sizeId: size.id
    };
  });
  spider(sknsData.map(p => p.productId), 'http://du.hupu.com/mapi/product/detail', {}, 1)
    .forEach(promise => promise.then(result => {
      if (result.status !== 200 && !_.has(result, 'data.detail')) {
        logger.warn(`[spider-task]爬取失败记录: ${JSON.stringify(result)}`);
        return;
      }

      const {data: {detail, sizeList}} = result;
      const yhProduct = sknsData.find(p => p.productId === detail.productId);
      const hrtime = process.hrtime();
      const timestamp = parseInt(Date.now() / 1000, 10) * 1000000000;

      sizeList.forEach(s => {
        if (_.isPlainObject(s.item) && s.item.price > 0) {
          const find = sizeRelation[s.size];

          if (!find) {
            return logger.error(`[spider-task]爬取尺码对应关系未找到: ${JSON.stringify(s)}`);
          }

          report({
            productId: yhProduct.yhId,
            sizeId: find.sizeId,
            price: s.item.price / 100,
            time
          }).then(() => {
            const reportTime = timestamp + process.hrtime(hrtime)[1];

            logger.info(`[spider-task]更新sku价格成功记录, duId:${detail.productId}, yhId: ${yhProduct.yhId}, sizeId: ${find.sizeId}, time: ${time}`);
            sendMessage(1, reportTime);
          }).catch(error => {
            logger.error(error.message);
          });
        }
      });
    }));
};