spider.js 1.48 KB
const md5 = require('yoho-md5')
const _ = require('lodash')
const rp = require('request-promise')
const nodeLockup = require('node-lockup')
const chalk = require('chalk')

const config = require('../config/index')
const logger = global.logger

const sign = (obj) => {
  let constr = ''

  Object.keys(obj).sort().forEach(k => {
    constr += k + obj[k].toString();
  })
  return md5(constr + '048a9c4943398714b356a696503d2d36');
}

const task = async (options, tick = 1) => {
  const params = Object.assign({}, options)

  delete params.url
  params.sign = sign(params)
  try {
    const result = await rp.get({
      url: options.url,
      qs: params,
      json: true,
      headers: {
        'Accept': 'application/json, text/plain, */*',
        'appVersion': '3.4.2',
        'Referer': 'http://du.hupu.com/mdu/product/detail.html?id=1&source=boutiqueRecommend',
        'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1',
        'X-Requested-With': 'XMLHttpRequest'
      }
    })

    return result
  } catch (error) {
    logger.error(chalk.red(`error:${options}:tick:${tick} ====> ${error}`))
    if (tick >= 3) {
      return Promise.resolve({})
    }
    return task(options, tick + 1)
  }
}
const lockTask = nodeLockup(task, config.delay)

module.exports = (ids, url = 'http://du.hupu.com/mapi/product/detail', params) => {
  return ids.map(id => lockTask(Object.assign({
    productId: id,
    source: 'boutiqueRecommend',
    url
  }, params)))
}