index.js
2.68 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
const req = require('request');
const async = require('async');
const md5 = require('md5');
const queryStr = 'http://svip.kuaidaili.com/api/getproxy/?orderid=989456293109158&num=100&b_pcchrome=1&b_pcie=1&b_pcff=1&protocol=1&method=2&sp1=1&format=json&sep=1';
const fastIps = {};
const btime = new Date().getTime();
/**
* check proxy valid
*
* @param {*} task
* @param {*} callback
*/
let checkIps = (task,callback) => {
req({
url:'http://www.163.com//',
timeout:1000,
'proxy':'http://'+task.url
},(error, response, body) => {
const key = md5(task.url);
if (!error && response.statusCode == 200) {
fastIps[key] = task.url;
console.log(`add ${task.type} ${task.url}`);
callback(false,task);
} else {
if(fastIps[key]) {
delete fastIps[key];
console.log(`delete ${task.type} ${task.url}`);
}
callback(true,null);
}
}).setMaxListeners(0);
};
/**
* create two queues
*/
const addQ = async.queue(checkIps,5);
const delQ = async.queue(checkIps,5);
/**
* call api get proxy server ip and port
*/
let callApi = ()=>{
req({
url:queryStr
},(error, response, body)=>{
if(!error&&response.statusCode===200) {
console.log(body);
let ret = JSON.parse(body).data.proxy_list;
if(ret) {
ret.forEach(val => {
addQ.push({url:val,type:'addProxy'},()=>{});
});
}
}
});
};
/**
* self check valid
*/
let selfCheck = ()=> {
for(let key in fastIps) {
delQ.push({url:fastIps[key],type:'checkProxy'},()=>{});
}
}
callApi();
/**
* add queue execute done
*/
addQ.drain = () => {
console.log('addQ have been processed');
console.log('cost:'+ (new Date().getTime()-btime));
console.log(fastIps);
callApi();
selfCheck();
};
/**
* del queue execute done
*/
delQ.drain = () => {
console.log('delQ have been processed');
console.log('cost:'+ (new Date().getTime()-btime));
};
const http = require('http');
const hostname = '127.0.0.1';
const port = 3000;
/**
* create server
*/
const server = http.createServer((req, res) => {
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
let list = [];
for(let key in fastIps) {
list.push(fastIps[key]);
}
const ret = {
code:0,
msg:'',
data:{
count:list.length,
proxy_list:list
}
};
const body = JSON.stringify(ret);
res.end(body);
});
/**
* start proxy api
*/
server.listen(port, hostname, () => {
console.log(`Server running at http://${hostname}:${port}/`);
});