Toggle navigation
Toggle navigation
This project
Loading...
Sign in
fe
/
spider-ufo
·
Commits
Go to a project
GitLab
Go to group
Project
Activity
Files
Commits
Pipelines
0
Builds
0
Graphs
Milestones
Issues
0
Merge Requests
0
Members
Labels
Wiki
Forks
Network
Create a new issue
Download as
Plain Diff
Browse Files
Authored by
刘志远
5 years ago
Commit
f5779e4a44179692e81754607304128846d4b01e
2 parents
5d69d0a3
98cc18ad
Merge branch 'feature/random_ip' into 'master'
ip 按顺序取 ip 按顺序取 See merge request
!46
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
9 additions
and
7 deletions
libs/spider.js
libs/spider.js
View file @
f5779e4
...
...
@@ -51,7 +51,8 @@ const requestDu = (params, type) => {
Referer
:
`
https
:
//m.poizon.com/router/product/ProductDetail?spuId=${params.spuId || 1}&sourceName=shareDetail`,
'User-Agent'
:
'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 Safari/604.1'
,
'X-Requested-With'
:
'XMLHttpRequest'
,
'X-Forwarded-For'
:
ip
'X-Forwarded-For'
:
ip
,
'X-Real-IP'
:
ip
},
timeout
:
2000
});
...
...
@@ -126,7 +127,7 @@ const taskDu = (params) => {
const
task
=
async
(
options
,
tick
=
1
)
=>
{
const
params
=
Object
.
assign
({},
options
);
const
ip
=
getRandomIp
();
const
ip
=
getRandomIp
(
params
.
i
);
params
[
'ip'
]
=
ip
;
try
{
return
await
taskDu
(
params
);
...
...
@@ -143,18 +144,19 @@ const task = async(options, tick = 1) => {
/**
* 获取随机ip
*/
const
getRandomIp
=
()
=>
{
let
index
=
Math
.
floor
((
Math
.
random
()
*
ipList
.
length
));
let
ip
=
ipList
[
index
];
const
getRandomIp
=
(
index
)
=>
{
// let index = Math.floor((Math.random()*ipList.length));
let
ip
=
index
>
1500
?
ipList
[
1000
]
:
ipList
[
index
];
return
ip
;
}
module
.
exports
=
(
ids
,
url
,
params
,
delay
)
=>
{
const
lockTask
=
nodeLockup
(
task
,
delay
||
config
.
delay
);
return
ids
.
map
(
id
=>
lockTask
(
Object
.
assign
({
return
ids
.
map
(
(
id
,
index
)
=>
lockTask
(
Object
.
assign
({
productId
:
id
,
productSourceName
:
'shareDetail'
,
url
:
url
||
detailUrl
url
:
url
||
detailUrl
,
i
:
index
},
params
)));
};
...
...
Please
register
or
login
to post a comment