spider.js 4.32 KB
const md5 = require('yoho-md5');
const rp = require('request-promise');
const nodeLockup = require('node-lockup');
const chalk = require('chalk');
const {logger} = require('./logger');

const ddAlert = require('../libs/dingding-alert');
const config = require('../config/index');

const moment = require('moment');
const detailUrl = 'https://app.poizon.com/api/v1/h5/index/fire/flow/product/detail';
const buyNowInfoUrl = 'https://app.poizon.com/api/v1/h5/inventory/price/h5/queryBuyNowInfo';

const { ipList } = require('../libs/ip-list');

const sign = (obj) => {
  let constr = '';

  Object.keys(obj).sort().forEach(k => {
    constr += k + obj[k].toString();
  });
  return md5(constr + '048a9c4943398714b356a696503d2d36');
};

const requestDu = (params, type) => {
  let baseInfo = {};
  let url = params.url;

  delete params.url;
  params.sign = sign(params);

  if (type === 'post') {
    baseInfo = {
      method: 'POST',
      body: params
    };
  } else {
    baseInfo = {
      qs: params
    };
  }

  let ip = params.ip || '';
  return rp({
    ...baseInfo,
    uri: url,
    json: true,
    headers: {
    Accept: 'application/json, text/plain, */*',
      appVersion: '4.4.0',
      Referer: `https://m.poizon.com/router/product/ProductDetail?spuId=${params.spuId || 1}&sourceName=shareDetail`,
      'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1',
      'X-Requested-With': 'XMLHttpRequest',
      'X-Forwarded-For': ip
    },
    timeout: 2000
  });
}

const taskDu = (params) => {
  let apis = [];
  let isDetailUrl = false;

  if (params.url === detailUrl) {
    isDetailUrl = true;

    if (params.productId) {
      params.spuId = params.productId;
      delete params.productId;
    }

    apis.push(requestDu(params, 'post'), requestDu({spuId: params.spuId, url: buyNowInfoUrl}, 'post'));
  } else {
    apis.push(requestDu(params));
  }

  return Promise.all(apis).then(result => {
    let detailInfo = result[0].data;

    if (isDetailUrl && detailInfo) {
      let {saleProperties: {list = []}, skus = [], relationList = []} = detailInfo;
      let {data: {skuInfoList = []}} = result[1];
      let sizeData = {};
      let priceData = {};
      let sizeList = [];

      list.forEach(size => {
        sizeData[size.propertyValueId] = size;
      });

      skuInfoList.forEach(price => {
        priceData[price.skuId] = price;
      });

      skus.forEach(val => {
        if (val.properties && val.properties.length) {
          val.properties.forEach(p => {
            if (+p.level === 1 && sizeData[p.propertyValueId]) {
              let { tradeChannelInfoList = [] } = priceData[val.skuId] || {};
              tradeChannelInfoList.forEach(item=>{
                if(+item.tradeType === 0) {
                  sizeData[p.propertyValueId].price = item.price;
                }
              })
            }
          });
        }
      });

      detailInfo.detail.productId = detailInfo.detail.spuId;
      detailInfo.sizeList = Object.keys(sizeData).map(size => {
        let { value, price } = sizeData[size];
        return {
          size: value,
          item: {
            price: price || 0
          }
        };
      });
    }

    return result[0];
  });

};

const task = async(options, tick = 1) => {
  const params = Object.assign({}, options);
  const ip = getRandomIp();
  params['ip'] = ip;
  try {
    return await taskDu(params);
  } catch (error) {
    logger.error(chalk.red(`error:${options}:tick:${tick} ====> ${error}`));
    if (tick >= 3) {
      var currentime =  moment(Date.now()).format('YYYY-MM-DD HH:mm:ss')
      ddAlert(`监控报警 : ${error}  ${currentime} ( IP : ${ip} ) @15101660386`, 'bjSpider');
      return Promise.resolve({});
    }
    return task(options, tick + 1);
  }
};
/**
 * 获取随机ip
 */
const getRandomIp = () => {
  let index = Math.floor((Math.random()*ipList.length));
  let ip = ipList[index];
  return ip;
}
module.exports = (ids, url, params, delay) => {
  const lockTask = nodeLockup(task, delay || config.delay);

  return ids.map(id => lockTask(Object.assign({
    productId: id,
    productSourceName: 'shareDetail',
    url: url || detailUrl
  }, params)));
};

module.exports.spiderFetch = (id, url, params) => {
  return task(Object.assign({
    productId: id,
    productSourceName: 'shareDetail',
    url: url || detailUrl
  }, params));
};