commit

陈峰
Commit b411554f2531764dec18ba6e4604d96fb0ba4f49 1 parent 6ffeafab
Showing 3 changed files with 89 additions and 30 deletions
app-buyers.js
libs/spider.js
other/spider-buyers.js
--- a/app-buyers.js 0 → 100644
View file @b411554
+++ b/app-buyers.js 0 → 100644
View file @b411554
+const schedule = require('node-schedule');
+const spiderBuyers = require('./other/spider-buyers');
+
+schedule.scheduleJob('0 0 2 * * *', () => {
+  spiderBuyers();
+});
--- a/libs/spider.js
View file @b411554
+++ b/libs/spider.js
View file @b411554
@@ -31,7 +31,8 @@ const task = async(options, tick = 1) => {
         Referer: 'http://du.hupu.com/mdu/product/detail.html?id=1&source=boutiqueRecommend',
         'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1',
         'X-Requested-With': 'XMLHttpRequest'
-      }
+      },
+      timeout: 2000
     });
     return result;
--- a/other/spider-buyers.js
View file @b411554
+++ b/other/spider-buyers.js
View file @b411554
 const dayjs = require('dayjs');
+const lockup = require('node-lockup');
+const isBetween = require('dayjs/plugin/isBetween');
 const _ = require('lodash');
 require('../libs/logger');
@@ -6,6 +8,8 @@ const config = require('../config/index');
 const MysqlAdapter = require('../libs/mysql');
 const spider = require('../libs/spider');
+dayjs.extend(isBetween)
+
 const mysql = new MysqlAdapter(config.database.connect, config.database.database);
 const RegNow = /刚刚/;
@@ -61,77 +65,125 @@ const getLeftTime = () => {
   return `${hour}小时${mins}分钟${parseInt(leftTime % 60)}秒`;
 };
-const minTime = dayjs('2018-12-22');
-
 const spiderBuyers = async(productId, lsId = 0, page = 0, trys = 0) => {
   const result = await spider.spiderFetch(productId, 'https://du.hupu.com/mapi/product/lastSoldList', {
     lastId: lsId
   });
-  let skipTime, lastId;
+  let lastId;
+  let tryLogic = false;
+  let list = [];
   if (result.status === 200) {
-    const {list} = result.data;
-    
+    let skip = false;
     lastId = result.data.lastId;
+    list = result.data.list;
     list.forEach(info => {
       const time = parseTime(info.formatTime);
-
-      if (time.isBefore(minTime)) {
-        skipTime = true;
-      } else {
+      const startTime = dayjs().subtract(1, 'day').add(1, 'second');
+      
+      if (!time.isBefore(startTime)) {
         mysql.insert('INSERT INTO `buyers_copy` (`productId`, `nickName`, `icon`, `time`, `size`, `soldNum`, `price`) VALUES (:productId, :nickName, :icon, :time, :size, :soldNum, :price)', {
           productId: productId,
           nickName: info.buyer.userName,
           icon: info.buyer.icon,
           size: info.item.size,
-          time: parseTime(info.formatTime).format('YYYY-MM-DD'),
+          time: time.format('YYYY-MM-DD HH:mm:ss'),
           soldNum: products[productId].count,
           price: products[productId].price,
         });
+      } else {
+        if (!skip) {
+          console.log(time.format('YYYY-MM-DD HH:mm:ss'), startTime.format('YYYY-MM-DD HH:mm:ss'))
+        }
+        skip = true;
       }
     });
+
+    if (skip) {
+      return {productId};
+    }
   } else {
     if (trys < 3) {
-      return spiderBuyers(productId, lsId, page, trys + 1);
+      tryLogic = true;
     }
   }
-  console.clear();
-  console.log(`page: ${page}, count: ${products[productId].count}, lastId: ${lastId}, 进度: ${inx}/${max}(${parseInt((page * 20 + list.length) / products[productId].count * 100)}%), 剩余: ${getLeftTime()}, productId: ${productId}`);
-  if (lastId && !skipTime) {
-    spiderBuyers(productId, lastId, page + 1);
+  if (tryLogic) {
+    console.log(`tryLogic ${trys}`)
+    return spiderBuyers(productId, lsId, page, trys + 1);
   } else {
-    end = Date.now();
-    start();
+    if (lastId) {
+      console.log(`spiderBuyers ${lastId}, ${page}`)
+      return spiderBuyers(productId, lastId, page + 1);
+    }
+    return {productId};
   }
 };
 //2721219
 const products = {};
-let max = 24200;
-let inx = 8707;
-const start = () => {
+let max = 34200;
+let failNum = 0;
+let inx = 1;
+let over = false;
+
+const start = async() => {
+  if (over) {
+    return;
+  }
   const productId = inx;
+  console.log(`start 1 , ${productId}`)
   inx++;
   if (inx >= max) {
+    over = true;
     console.log('over!!!!!!');
     return;
   }
-  if (products[productId]) {
-    spiderBuyers(productId);
+  const result = await spider.spiderFetch(productId);
+  console.log(`start 2 , ${productId}`)
+
+  if (result.status === 200) {
+    failNum = 0;
+    products[productId] = {
+      count: _.get(result, 'data.detail.soldNum', 0),
+      price: _.get(result, 'data.item.price', 0)
+    };
+    try {
+      const bl = await spiderBuyers(productId);
+      console.log(`start 3 , ${productId}`)
+
+      if (bl) {
+        console.log(JSON.stringify(bl));
+      }
+    } catch (error) {
+      console.log('error', error);
+    }
+    
+
   } else {
-    spider([productId]).forEach(promise => promise.then(result => {
-      products[productId] = {
-        count: _.get(result, 'data.detail.soldNum', 0),
-        price: _.get(result, 'data.item.price', 0)
-      };
-      spiderBuyers(productId);
-    }));
+    console.log(productId, result.status)
+    if (result.status) {
+      failNum++;
+    }
+  }
+  if (failNum >= 20) {
+    over = true;
+    return console.log('over!!!!!!', JSON.stringify(result));
   }
+  return true;
 };
+const locktask = lockup(start);
+
+
+module.exports = () => {
+  Array.from({length: max}).map((v, i) => {
+    locktask();
+  })
+}
+