spider.js 1.89 KB
const md5 = require('yoho-md5');
const rp = require('request-promise');
const nodeLockup = require('node-lockup');
const chalk = require('chalk');
const {logger} = require('./logger');

const detailUrl = 'https://app.poizon.com/api/v1/h5/index/fire/flow/product/detail';

const sign = (obj) => {
  let constr = '';

  Object.keys(obj).sort().forEach(k => {
    constr += k + obj[k].toString();
  });
  return md5(constr + '048a9c4943398714b356a696503d2d36');
};

const task = async(options, tick = 1) => {
  const params = Object.assign({}, options);

  delete params.url;
  params.sign = sign(params);
  try {
    const result = await rp.get({
      url: options.url,
      qs: params,
      json: true,
      headers: {
        Accept: 'application/json, text/plain, */*',
        appid: 'h5',
        appVersion: '4.4.0',
        Referer: 'http://du.hupu.com/mdu/product/detail.html?id=1&source=shareDetail',
        'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1',
        'X-Requested-With': 'XMLHttpRequest',
        cookie: 'duToken=d41d8cd9%7C16075871%7C1558328526%7C37c4ab8a8cd16e3f; UM_distinctid=16a341fe1c86a-099fe9f71aa88b8-5743047f-4a640-16a341fe1c94c5'
      },
      timeout: 2000
    });

    return result;
  } catch (error) {
    logger.error(chalk.red(`error:${JSON.stringify(options)}:tick:${tick} ====> ${error}`));
    if (tick >= 3) {
      return Promise.resolve({});
    }
    return task(options, tick + 1);
  }
};

module.exports = (ids, url, params, delay) => {
  const lockTask = nodeLockup(task, delay || 1);

  return ids.map(id => lockTask(Object.assign({
    productId: id,
    productSourceName: 'shareDetail',
    url: url || detailUrl
  }, params)));
};

module.exports.spiderFetch = (id, url, params) => {
  return task(Object.assign({
    productId: id,
    productSourceName: 'shareDetail',
    url: url || detailUrl
  }, params));
};