spider.js 4.97 KB
const md5 = require('yoho-md5');
const rp = require('request-promise');
const nodeLockup = require('node-lockup');
const chalk = require('chalk');
const {logger} = require('./logger');

const ddAlert = require('../libs/dingding-alert');
const config = require('../config/index');

const moment = require('moment');
const detailUrl = 'https://app.poizon.com/api/v1/h5/index/fire/flow/product/detail';
const buyNowInfoUrl = 'https://app.poizon.com/api/v1/h5/inventory/price/h5/queryBuyNowInfo';

const sign = (obj) => {
  let constr = '';

  Object.keys(obj).sort().forEach(k => {
    constr += k + obj[k].toString();
  });
  return md5(constr + '048a9c4943398714b356a696503d2d36');
};

const requestDu = (params, type) => {
  let baseInfo = {};
  let url = params.url;

  delete params.url;
  params.sign = sign(params);

  if (type === 'post') {
    baseInfo = {
      method: 'POST',
      body: params
    };
  } else {
    baseInfo = {
      qs: params
    };
  }
  // 爬虫 123.206.21.19 百度 220.181.38.148  搜狗 221.122.82.30  搜搜 106.39.246.42 360搜索 36.110.236.68 有道 220.181.76.83 雅虎 124.108.103.103

  // 必应 202.89.233.100 慧聪 180.97.232.53 淘宝 140.205.220.96 天猫 124.236.61.227 京东 120.52.148.118 国搜 1.180.19.71 新浪搜索 183.60.95.142

  // 新浪 123.126.157.222 今日头条 140.249.240.234 人民搜索网 61.164.153.222 凤凰搜索网 49.233.102.63 乐搜 220.181.90.8 央视搜索 115.182.34.79

  let ipList = [
    '221.122.82.30',
    '106.39.246.42',
    '36.110.236.68',
    '202.89.233.100',
    '180.97.232.53',
    '140.205.220.96',
    '124.236.61.227',
    '120.52.148.118',
    '1.180.19.71',
    '183.60.95.142',
    '123.126.157.222',
    '140.249.240.234',
    '61.164.153.222',
    '49.233.102.63',
    '220.181.90.8',
    '115.182.34.79',
    '124.64.19.86',
    '124.64.30.120'
  ];
  let index = Math.floor((Math.random()*ipList.length));
  let ip = ipList[index];
  return rp({
    ...baseInfo,
    uri: url,
    json: true,
    headers: {
    Accept: 'application/json, text/plain, */*',
      appVersion: '4.4.0',
      Referer: `https://m.poizon.com/router/product/ProductDetail?spuId=${params.spuId || 1}&sourceName=shareDetail`,
      'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1',
      'X-Requested-With': 'XMLHttpRequest',
      'X-Forwarded-For': ip
    },
    timeout: 2000
  });
}

const taskDu = (params) => {
  let apis = [];
  let isDetailUrl = false;

  if (params.url === detailUrl) {
    isDetailUrl = true;

    if (params.productId) {
      params.spuId = params.productId;
      delete params.productId;
    }

    apis.push(requestDu(params, 'post'), requestDu({spuId: params.spuId, url: buyNowInfoUrl}, 'post'));
  } else {
    apis.push(requestDu(params));
  }

  return Promise.all(apis).then(result => {
    let detailInfo = result[0].data;

    if (isDetailUrl && detailInfo) {
      let {saleProperties: {list = []}, skus = [], relationList = []} = detailInfo;
      let {data: {skuInfoList = []}} = result[1];
      let sizeData = {};
      let priceData = {};
      let sizeList = [];

      list.forEach(size => {
        sizeData[size.propertyValueId] = size;
      });

      skuInfoList.forEach(price => {
        priceData[price.skuId] = price;
      });

      skus.forEach(val => {
        if (val.properties && val.properties.length) {
          val.properties.forEach(p => {
            if (+p.level === 1 && sizeData[p.propertyValueId]) {
              let { tradeChannelInfoList = [] } = priceData[val.skuId] || {};
              tradeChannelInfoList.forEach(item=>{
                if(+item.tradeType === 0) {
                  sizeData[p.propertyValueId].price = item.price;
                }
              })
            }
          });
        }
      });

      detailInfo.detail.productId = detailInfo.detail.spuId;
      detailInfo.sizeList = Object.keys(sizeData).map(size => {
        let { value, price } = sizeData[size];
        return {
          size: value,
          item: {
            price: price || 0
          }
        };
      });
    }

    return result[0];
  });

};

const task = async(options, tick = 1) => {
  const params = Object.assign({}, options);

  try {
    return await taskDu(params);
  } catch (error) {
    logger.error(chalk.red(`error:${options}:tick:${tick} ====> ${error}`));
    if (tick >= 3) {
      var currentime =  moment(Date.now()).format('YYYY-MM-DD HH:mm:ss')
      ddAlert(`监控报警 : ${error}  ${currentime} `, 'bjSpider');
      return Promise.resolve({});
    }
    return task(options, tick + 1);
  }
};

module.exports = (ids, url, params, delay) => {
  const lockTask = nodeLockup(task, delay || config.delay);

  return ids.map(id => lockTask(Object.assign({
    productId: id,
    productSourceName: 'shareDetail',
    url: url || detailUrl
  }, params)));
};

module.exports.spiderFetch = (id, url, params) => {
  return task(Object.assign({
    productId: id,
    productSourceName: 'shareDetail',
    url: url || detailUrl
  }, params));
};