fe / spider-ufo · Commits

Go to a project

Authored by 陈峰
Commit 4e368160adb5f74c400593fcee7f75d472d91086 1 parent 4a8ae757

commit

Inline Side-by-side
Showing 3 changed files with 268 additions and 92 deletions
select
A.productId,
sum(`2019-03-01`) AS '2019-03-01',
sum(`2019-03-02`) AS '2019-03-02',
sum(`2019-03-03`) AS '2019-03-03',
sum(`2019-03-04`) AS '2019-03-04',
sum(`2019-03-05`) AS '2019-03-05',
sum(`2019-03-06`) AS '2019-03-06',
sum(`2019-03-07`) AS '2019-03-07',
sum(`2019-03-08`) AS '2019-03-08',
sum(`2019-03-09`) AS '2019-03-09',
sum(`2019-03-10`) AS '2019-03-10',
sum(`2019-03-11`) AS '2019-03-11',
sum(`2019-03-12`) AS '2019-03-12',
sum(`2019-03-13`) AS '2019-03-13',
sum(`2019-03-14`) AS '2019-03-14',
sum(`2019-03-15`) AS '2019-03-15',
sum(`2019-03-16`) AS '2019-03-16',
sum(`2019-03-17`) AS '2019-03-17',
sum(`2019-03-18`) AS '2019-03-18',
sum(`2019-03-19`) AS '2019-03-19',
sum(`2019-03-20`) AS '2019-03-20',
sum(`2019-03-21`) AS '2019-03-21',
sum(`2019-03-22`) AS '2019-03-22',
sum(`2019-03-23`) AS '2019-03-23',
sum(`2019-03-24`) AS '2019-03-24',
sum(`2019-03-25`) AS '2019-03-25',
sum(`2019-03-26`) AS '2019-03-26',
sum(`2019-03-27`) AS '2019-03-27',
sum(`2019-03-28`) AS '2019-03-28',
sum(`2019-03-29`) AS '2019-03-29',
sum(`2019-03-30`) AS '2019-03-30',
sum(`2019-03-31`) AS '2019-03-31'
b.id,
b.model,
b.price,
IFNULL(sum(`2019-04-01`), 0) AS '2019-04-01',
IFNULL(sum(`2019-04-02`), 0) AS '2019-04-02',
IFNULL(sum(`2019-04-03`), 0) AS '2019-04-03',
IFNULL(sum(`2019-04-04`), 0) AS '2019-04-04',
IFNULL(sum(`2019-04-05`), 0) AS '2019-04-05',
IFNULL(sum(`2019-04-06`), 0) AS '2019-04-06',
IFNULL(sum(`2019-04-07`), 0) AS '2019-04-07',
IFNULL(sum(`2019-04-08`), 0) AS '2019-04-08',
IFNULL(sum(`2019-04-09`), 0) AS '2019-04-09',
IFNULL(sum(`2019-04-10`), 0) AS '2019-04-10',
IFNULL(sum(`2019-04-11`), 0) AS '2019-04-11',
IFNULL(sum(`2019-04-12`), 0) AS '2019-04-12',
IFNULL(sum(`2019-04-13`), 0) AS '2019-04-13',
IFNULL(sum(`2019-04-14`), 0) AS '2019-04-14',
IFNULL(sum(`2019-04-15`), 0) AS '2019-04-15',
IFNULL(sum(`2019-04-16`), 0) AS '2019-04-16',
IFNULL(sum(`2019-04-17`), 0) AS '2019-04-17',
IFNULL(sum(`2019-04-18`), 0) AS '2019-04-18',
IFNULL(sum(`2019-04-19`), 0) AS '2019-04-19',
IFNULL(sum(`2019-04-20`), 0) AS '2019-04-20',
IFNULL(sum(`2019-04-21`), 0) AS '2019-04-21',
IFNULL(sum(`2019-04-22`), 0) AS '2019-04-22',
IFNULL(sum(`2019-04-23`), 0) AS '2019-04-23',
IFNULL(sum(`2019-04-24`), 0) AS '2019-04-24',
IFNULL(sum(`2019-04-25`), 0) AS '2019-04-25',
IFNULL(sum(`2019-04-26`), 0) AS '2019-04-26',
IFNULL(sum(`2019-04-27`), 0) AS '2019-04-27',
IFNULL(sum(`2019-04-28`), 0) AS '2019-04-28',
IFNULL(sum(`2019-04-29`), 0) AS '2019-04-29',
IFNULL(sum(`2019-04-30`), 0) AS '2019-04-30'
from
sknprice b
left join
(
select
productId,
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-01' then 1 else 0 end as '2019-03-01',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-02' then 1 else 0 end as '2019-03-02',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-03' then 1 else 0 end as '2019-03-03',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-04' then 1 else 0 end as '2019-03-04',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-05' then 1 else 0 end as '2019-03-05',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-06' then 1 else 0 end as '2019-03-06',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-07' then 1 else 0 end as '2019-03-07',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-08' then 1 else 0 end as '2019-03-08',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-09' then 1 else 0 end as '2019-03-09',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-10' then 1 else 0 end as '2019-03-10',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-11' then 1 else 0 end as '2019-03-11',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-12' then 1 else 0 end as '2019-03-12',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-13' then 1 else 0 end as '2019-03-13',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-14' then 1 else 0 end as '2019-03-14',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-15' then 1 else 0 end as '2019-03-15',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-16' then 1 else 0 end as '2019-03-16',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-17' then 1 else 0 end as '2019-03-17',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-18' then 1 else 0 end as '2019-03-18',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-19' then 1 else 0 end as '2019-03-19',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-20' then 1 else 0 end as '2019-03-20',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-21' then 1 else 0 end as '2019-03-21',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-22' then 1 else 0 end as '2019-03-22',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-23' then 1 else 0 end as '2019-03-23',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-24' then 1 else 0 end as '2019-03-24',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-25' then 1 else 0 end as '2019-03-25',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-26' then 1 else 0 end as '2019-03-26',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-27' then 1 else 0 end as '2019-03-27',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-28' then 1 else 0 end as '2019-03-28',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-29' then 1 else 0 end as '2019-03-29',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-30' then 1 else 0 end as '2019-03-30',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-03-31' then 1 else 0 end as '2019-03-31'
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-01' then b.price else 0 end as '2019-04-01',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-02' then b.price else 0 end as '2019-04-02',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-03' then b.price else 0 end as '2019-04-03',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-04' then b.price else 0 end as '2019-04-04',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-05' then b.price else 0 end as '2019-04-05',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-06' then b.price else 0 end as '2019-04-06',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-07' then b.price else 0 end as '2019-04-07',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-08' then b.price else 0 end as '2019-04-08',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-09' then b.price else 0 end as '2019-04-09',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-10' then b.price else 0 end as '2019-04-10',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-11' then b.price else 0 end as '2019-04-11',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-12' then b.price else 0 end as '2019-04-12',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-13' then b.price else 0 end as '2019-04-13',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-14' then b.price else 0 end as '2019-04-14',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-15' then b.price else 0 end as '2019-04-15',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-16' then b.price else 0 end as '2019-04-16',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-17' then b.price else 0 end as '2019-04-17',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-18' then b.price else 0 end as '2019-04-18',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-19' then b.price else 0 end as '2019-04-19',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-20' then b.price else 0 end as '2019-04-20',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-21' then b.price else 0 end as '2019-04-21',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-22' then b.price else 0 end as '2019-04-22',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-23' then b.price else 0 end as '2019-04-23',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-24' then b.price else 0 end as '2019-04-24',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-25' then b.price else 0 end as '2019-04-25',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-26' then b.price else 0 end as '2019-04-26',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-27' then b.price else 0 end as '2019-04-27',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-28' then b.price else 0 end as '2019-04-28',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-29' then b.price else 0 end as '2019-04-29',
case DATE_FORMAT(time,'%Y-%m-%d') when '2019-04-30' then b.price else 0 end as '2019-04-30'
from
buyers
buyers_2019_04 a left join sknprice b on a.productId = b.id
where
time >= '2019-03-01' and time < '2019-04-01'
) A
group by A.productId
\ No newline at end of file
time >= '2019-04-01' and time < '2019-05-01'
) A on A.productId = b.id
group by b.id,b.model,b.price
\ No newline at end of file
... ...
const dayjs = require('dayjs');
const lockup = require('node-lockup');
const isBetween = require('dayjs/plugin/isBetween');
const _ = require('lodash');
const spider = require('../libs/spider');
const config = require('./spider-buyers.json');
const {logger} = require('../libs/logger');
const Model = require('../libs/model');
dayjs.extend(isBetween)
class DuSaleRecord extends Model {
constructor() {
super('du-sale');
}
}
const duSaleRecord = new DuSaleRecord();
const RegNow = /刚刚/;
const RegMinute = /(\d+)分钟前/;
const RegHour = /(\d+)小时前/;
const RegDay = /(\d+)天前/;
const RegDate = /(\d+)月(\d+)日/;
const RegDateFull = /\d+\.\d+\.\d+/;
const parseTime = (relativeTime) => {
const matchMinute = relativeTime.match(RegMinute);
if (matchMinute) {
return dayjs().add(0 - matchMinute[1], 'minute');
}
const matchHour = relativeTime.match(RegHour);
if (matchHour) {
return dayjs().add(0 - matchHour[1], 'hour');
}
const matchDay = relativeTime.match(RegDay);
if (matchDay) {
return dayjs().add(0 - matchDay[1], 'day');
}
const matchDate = relativeTime.match(RegDate);
if (matchDate) {
return dayjs(`2019/${matchDate[1]}/${matchDate[2]}`);
}
if (RegDateFull.test(relativeTime)) {
return dayjs(relativeTime);
}
if (RegNow.test(relativeTime)) {
return dayjs();
}
return dayjs('1990-01-01');
};
let time = Date.now();
let end = 0;
const spiderBuyers = async(productId, lsId = 0, page = 0, trys = 0, tid) => {
const result = await spider.spiderFetch(productId, 'https://du.hupu.com/mapi/product/lastSoldList', {
lastId: lsId
});
let lastId;
let tryLogic = false;
let list = [];
if (result.status === 200) {
let skip = false;
lastId = result.data.lastId;
list = result.data.list;
list.forEach(info => {
const time = parseTime(info.formatTime);
const startTime = dayjs().add(-5, 'day');
if (!time.isBefore(startTime)) {
duSaleRecord.insert({
productId: productId,
nickName: info.buyer.userName,
icon: info.buyer.icon,
size: info.item.size,
time: time.format('YYYY-MM-DD HH:mm:ss'),
formatTime: info.formatTime,
insertTime: dayjs().format('YYYY-MM-DD HH:mm:ss')
})
} else {
if (!skip) {
logger.info(`thread: ${tid}, productid: ${productId}, skip time:${time.format('YYYY-MM-DD HH:mm:ss')}`)
}
skip = true;
}
});
if (skip) {
return {productId};
}
} else {
if (trys < 3) {
tryLogic = true;
}
}
if (tryLogic) {
logger.info(`tryLogic ${trys}`)
return spiderBuyers(productId, lsId, page, trys + 1, tid);
} else {
if (lastId) {
logger.info(`thread: ${tid}, productId: ${productId}, nextPage: ${page}, lastId: ${lastId}`)
return spiderBuyers(productId, lastId, page + 1, 0, tid);
}
return {productId};
}
};
//8202200
const products = {};
let max = 32000;
let failNum = 0;
let over = false;
const start = async({inx, tid}) => {
if (over) {
return;
}
const productId = inx;
inx++;
if (inx >= max) {
over = true;
logger.info('over!!!!!!');
return;
}
const result = await spider.spiderFetch(productId);
if (result.status === 200) {
failNum = 0;
products[productId] = {
count: _.get(result, 'data.detail.soldNum', 0),
price: _.get(result, 'data.item.price', 0)
};
try {
await spiderBuyers(productId, 0, 0, 0, tid);
} catch (error) {
logger.info('error', error);
}
} else {
logger.info(productId, result.status)
if (result.status) {
failNum++;
}
}
if (failNum >= 20) {
over = true;
return logger.info('over!!!!!!', JSON.stringify(result));
}
return true;
};
//4211292
module.exports = () => {
const locktask = lockup(start);
locktask({inx: 26715, tid: 1});
}
module.exports()
\ No newline at end of file
... ...
... ... @@ -52,19 +52,8 @@ const parseTime = (relativeTime) => {
let time = Date.now();
let end = 0;
const getLeftTime = () => {
let offset = (end || Date.now()) - time;
let intervalTime = (offset / inx) / 1000;
let leftTime = (19635 - inx) * intervalTime;
const hour = parseInt(leftTime / 36000);
leftTime = leftTime % 3600;
const mins = parseInt(leftTime / 60);
return `${hour}小时${mins}分钟${parseInt(leftTime % 60)}秒`;
};
const spiderBuyers = async(productId, lsId = 0, page = 0, trys = 0) => {
const spiderBuyers = async(productId, lsId = 0, page = 0, trys = 0, tid) => {
const result = await spider.spiderFetch(productId, 'https://du.hupu.com/mapi/product/lastSoldList', {
lastId: lsId
});
... ... @@ -80,7 +69,7 @@ const spiderBuyers = async(productId, lsId = 0, page = 0, trys = 0) => {
list.forEach(info => {
const time = parseTime(info.formatTime);
const startTime = dayjs().subtract(1, 'day').add(1, 'second');
const startTime = dayjs('2018-03-31').add(1, 'second');
if (!time.isBefore(startTime)) {
mysql.insert('INSERT INTO `' + config.table + '` (`productId`, `nickName`, `icon`, `time`, `size`, `soldNum`, `price`) VALUES (:productId, :nickName, :icon, :time, :size, :soldNum, :price)', {
... ... @@ -94,7 +83,7 @@ const spiderBuyers = async(productId, lsId = 0, page = 0, trys = 0) => {
});
} else {
if (!skip) {
logger.info(`productid: ${productId}, skip time:${time.format('YYYY-MM-DD HH:mm:ss')}`)
logger.info(`thread: ${tid}, productid: ${productId}, skip time:${time.format('YYYY-MM-DD HH:mm:ss')}`)
}
skip = true;
}
... ... @@ -111,25 +100,24 @@ const spiderBuyers = async(productId, lsId = 0, page = 0, trys = 0) => {
if (tryLogic) {
logger.info(`tryLogic ${trys}`)
return spiderBuyers(productId, lsId, page, trys + 1);
return spiderBuyers(productId, lsId, page, trys + 1, tid);
} else {
if (lastId) {
logger.info(`productId: ${productId}, nextPage: ${page}, lastId: ${lastId}`)
return spiderBuyers(productId, lastId, page + 1);
logger.info(`thread: ${tid}, productId: ${productId}, nextPage: ${page}, lastId: ${lastId}`)
return spiderBuyers(productId, lastId, page + 1, 0, tid);
}
return {productId};
}
};
//2721219
//8202200
const products = {};
let max = 50000;
let max = 32000;
let failNum = 0;
let inx = 1;
let over = false;
const start = async() => {
const start = async({inx, tid}) => {
if (over) {
return;
}
... ... @@ -151,7 +139,7 @@ const start = async() => {
price: _.get(result, 'data.item.price', 0)
};
try {
await spiderBuyers(productId);
await spiderBuyers(productId, 0, 0, 0, tid);
} catch (error) {
logger.info('error', error);
}
... ... @@ -170,13 +158,25 @@ const start = async() => {
return true;
};
const locktask = lockup(start);
//4211292
module.exports = () => {
Array.from({length: max}).map((v, i) => {
locktask();
const thread = 4;
const ids = Array.from({length: max}).map((v, i) => i);
const threads = [];
const page = ids.length / thread;
for (let index = 0; index < thread; index++) {
threads.push(ids.splice(0, page));
}
threads.forEach((tids, tid) => {
const locktask = lockup(start);
tids.map((inx) => {
locktask({inx, tid});
})
})
}
module.exports()
\ No newline at end of file
... ...