fe / spider-ufo · Commits

Go to a project

Authored by 杨延青
Commit 602c1b2b487bec1119ddf7331fbb5b4cb9a229ac 2 parents e1d2e9a4 94833392

Merge branch 'feature/detailApi' into 'master' 

change du api



See merge request !17
Inline Side-by-side
Showing 6 changed files with 25 additions and 20 deletions
@@ -4,6 +4,9 @@ const nodeLockup = require('node-lockup'); @@ -4,6 +4,9 @@ const nodeLockup = require('node-lockup');
4 const chalk = require('chalk'); 4 const chalk = require('chalk');
5 const {logger} = require('./logger'); 5 const {logger} = require('./logger');
6 6
  7 +
  8 +const detailUrl = 'https://app.poizon.com/api/v1/h5/index/fire/flow/product/detail';
  9 +
7 const sign = (obj) => { 10 const sign = (obj) => {
8 let constr = ''; 11 let constr = '';
9 12
@@ -25,8 +28,8 @@ const task = async(options, tick = 1) => { @@ -25,8 +28,8 @@ const task = async(options, tick = 1) => {
25 json: true, 28 json: true,
26 headers: { 29 headers: {
27 Accept: 'application/json, text/plain, */*', 30 Accept: 'application/json, text/plain, */*',
28 - appVersion: '3.5.0',  
29 - Referer: 'http://du.hupu.com/mdu/product/detail.html?id=1&source=boutiqueRecommend', 31 + appVersion: '4.4.0',
  32 + Referer: 'https://m.poizon.com/mdu/product/detail.html?id=1&source=shareDetail',
30 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1', 33 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1',
31 'X-Requested-With': 'XMLHttpRequest' 34 'X-Requested-With': 'XMLHttpRequest'
32 }, 35 },
@@ -43,20 +46,20 @@ const task = async(options, tick = 1) => { @@ -43,20 +46,20 @@ const task = async(options, tick = 1) => {
43 } 46 }
44 }; 47 };
45 48
46 -module.exports = (ids, url = 'http://du.hupu.com/mapi/product/detail', params, delay) => { 49 +module.exports = (ids, url, params, delay) => {
47 const lockTask = nodeLockup(task, delay || 1); 50 const lockTask = nodeLockup(task, delay || 1);
48 51
49 return ids.map(id => lockTask(Object.assign({ 52 return ids.map(id => lockTask(Object.assign({
50 productId: id, 53 productId: id,
51 - source: 'boutiqueRecommend',  
52 - url 54 + productSourceName: 'shareDetail',
  55 + url: url || detailUrl
53 }, params))); 56 }, params)));
54 }; 57 };
55 58
56 -module.exports.spiderFetch = (id, url = 'http://du.hupu.com/mapi/product/detail', params) => { 59 +module.exports.spiderFetch = (id, url, params) => {
57 return task(Object.assign({ 60 return task(Object.assign({
58 productId: id, 61 productId: id,
59 - source: 'boutiqueRecommend',  
60 - url 62 + productSourceName: 'shareDetail',
  63 + url: url || detailUrl
61 }, params)); 64 }, params));
62 }; 65 };
@@ -6,6 +6,8 @@ const {logger} = require('./logger'); @@ -6,6 +6,8 @@ const {logger} = require('./logger');
6 6
7 const config = require('../config/index'); 7 const config = require('../config/index');
8 8
  9 +const detailUrl = 'https://app.poizon.com/api/v1/h5/index/fire/flow/product/detail';
  10 +
9 const sign = (obj) => { 11 const sign = (obj) => {
10 let constr = ''; 12 let constr = '';
11 13
@@ -27,8 +29,8 @@ const task = async(options, tick = 1) => { @@ -27,8 +29,8 @@ const task = async(options, tick = 1) => {
27 json: true, 29 json: true,
28 headers: { 30 headers: {
29 Accept: 'application/json, text/plain, */*', 31 Accept: 'application/json, text/plain, */*',
30 - appVersion: '3.5.0',  
31 - Referer: 'http://du.hupu.com/mdu/product/detail.html?id=1&source=boutiqueRecommend', 32 + appVersion: '4.4.0',
  33 + Referer: 'https://m.poizon.com/mdu/product/detail.html?id=1&source=shareDetail',
32 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1', 34 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1',
33 'X-Requested-With': 'XMLHttpRequest' 35 'X-Requested-With': 'XMLHttpRequest'
34 }, 36 },
@@ -45,20 +47,20 @@ const task = async(options, tick = 1) => { @@ -45,20 +47,20 @@ const task = async(options, tick = 1) => {
45 } 47 }
46 }; 48 };
47 49
48 -module.exports = (ids, url = 'http://du.hupu.com/mapi/product/detail', params, delay) => { 50 +module.exports = (ids, url, params, delay) => {
49 const lockTask = nodeLockup(task, delay || config.delay); 51 const lockTask = nodeLockup(task, delay || config.delay);
50 52
51 return ids.map(id => lockTask(Object.assign({ 53 return ids.map(id => lockTask(Object.assign({
52 productId: id, 54 productId: id,
53 - source: 'boutiqueRecommend',  
54 - url 55 + productSourceName: 'shareDetail',
  56 + url: url || detailUrl
55 }, params))); 57 }, params)));
56 }; 58 };
57 59
58 -module.exports.spiderFetch = (id, url = 'http://du.hupu.com/mapi/product/detail', params) => { 60 +module.exports.spiderFetch = (id, url, params) => {
59 return task(Object.assign({ 61 return task(Object.assign({
60 productId: id, 62 productId: id,
61 - source: 'boutiqueRecommend',  
62 - url 63 + productSourceName: 'shareDetail',
  64 + url: url || detailUrl
63 }, params)); 65 }, params));
64 }; 66 };
@@ -2,7 +2,7 @@ const spider = require('../libs/spider'); @@ -2,7 +2,7 @@ const spider = require('../libs/spider');
2 const _ = require('lodash'); 2 const _ = require('lodash');
3 3
4 const spiderTask = async ids => { 4 const spiderTask = async ids => {
5 - spider(ids, 'http://du.hupu.com/mapi/product/detail', {}, 1).forEach(promise => promise.then(result => { 5 + spider(ids, '', {}, 1).forEach(promise => promise.then(result => {
6 if (result.status === 200) { 6 if (result.status === 200) {
7 const sizes = result.data.sizeList; 7 const sizes = result.data.sizeList;
8 8
@@ -51,7 +51,7 @@ const lockReportTask = nodeLockup(report); @@ -51,7 +51,7 @@ const lockReportTask = nodeLockup(report);
51 function spiderData(products) { 51 function spiderData(products) {
52 const ids = products.map(p => p.duId); 52 const ids = products.map(p => p.duId);
53 53
54 - return spider(ids, 'http://du.hupu.com/mapi/product/detail', {}, 1).forEach(promise => promise.then(result => { 54 + return spider(ids, '', {}, 1).forEach(promise => promise.then(result => {
55 if (result && result.status === 200) { 55 if (result && result.status === 200) {
56 const {detail} = result.data; 56 const {detail} = result.data;
57 const find = _.find(products, p => p.duId === detail.productId); 57 const find = _.find(products, p => p.duId === detail.productId);
@@ -24,7 +24,7 @@ module.exports = async(time) => { @@ -24,7 +24,7 @@ module.exports = async(time) => {
24 const allProducts = await ProductRelationModel.findAll(); 24 const allProducts = await ProductRelationModel.findAll();
25 const duProducts = allProducts.filter(p => _.has(p, 'third.du.productId')); 25 const duProducts = allProducts.filter(p => _.has(p, 'third.du.productId'));
26 26
27 - spider(duProducts.map(p => _.get(p, 'third.du.productId')), 'http://du.hupu.com/mapi/product/detail', {}, 1) 27 + spider(duProducts.map(p => _.get(p, 'third.du.productId')), '', {}, 1)
28 .forEach(promise => promise.then(result => { 28 .forEach(promise => promise.then(result => {
29 if (result.status !== 200 && !_.has(result, 'data.detail')) { 29 if (result.status !== 200 && !_.has(result, 'data.detail')) {
30 logger.warn(`[spider-task]爬取失败记录: ${JSON.stringify(result)}`); 30 logger.warn(`[spider-task]爬取失败记录: ${JSON.stringify(result)}`);
@@ -46,7 +46,7 @@ module.exports = async(time) => { @@ -46,7 +46,7 @@ module.exports = async(time) => {
46 // }); 46 // });
47 // } 47 // }
48 48
49 - spider(sknsData.map(p => p.productId), 'http://du.hupu.com/mapi/product/detail', {}, 1) 49 + spider(sknsData.map(p => p.productId), '', {}, 1)
50 .forEach(promise => promise.then(result => { 50 .forEach(promise => promise.then(result => {
51 if (result.status !== 200 && !_.has(result, 'data.detail')) { 51 if (result.status !== 200 && !_.has(result, 'data.detail')) {
52 logger.warn(`[spider-task]爬取失败记录: ${JSON.stringify(result)}`); 52 logger.warn(`[spider-task]爬取失败记录: ${JSON.stringify(result)}`);