我应该登录并通过nodejs调用http一些api点，
var mins = 150;
var url = require('url');
const domainim= 'example.com';
const request = require('requestretry').defaults({
jar : true,
timeout : 120000
});
const req = require('request');
var fs = require('fs');
var linkler = [];
var timeoutsure = 120 * 1000;
var separateReqPool = {
maxSockets : 20
};
let success = 0, fail = 0, toplam = 0, plusplus = 0;
let adet = 0;
let jardata = {};
let baslama = Math.round(new Date().getTime() / 1000);
function texts(l) {
var t = "";
var p = "abcdefghijklmnopqrstuvwxyz0123456789";
for (var i = 0; i < l; i++) {
t += p.charAt(Math.floor(Math.random() * p.length));
}
return t;
}
fs.readFile("data.txt", function (err, data) {
if (err)
throw err;
data = data.toString().replace("\r", "");
data = data.split(/\r\n|\r|\n/g);
console.log("toplam data : " + data.length);
toplam = data.length;
data.forEach(
function (line) {
isle(line);
});
});
function onExit() {
console.log('\n%d sec,\n%d total,\n%d suc,\n%d fai,\n%d plusplus,\n---------------------\n',
(Math.round(new Date().getTime() / 1000)) - baslama, toplam, success, fail, plusplus);
}
process.on('exit', onExit);
function dataok(id, pass, bakiye) {
console.log(id + " " + pass + " " + bakiye);
var fd = fs.openSync("hardblock.txt", 'a+');
fs.writeSync(fd, id + " " + pass + " " + bakiye + "\r\n");
fs.closeSync(fd);
}
function extradata(id, pass, bakiye) {
console.log(id + " " + pass + " " + bakiye);
var fd = fs.openSync("harddata.txt", 'a+');
fs.writeSync(fd, id + " " + pass + " " + bakiye + "\r\n");
fs.closeSync(fd);
}
function isle(line) {
jardata[line.split(" ")[0]] = req.jar();
request({
url : 'http://'+domainim+'/',
jar : jardata[line.split(" ")[0]],
headers : {
'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
},
maxAttempts : 15,
retryDelay : 2000,
followRedirect : true,
// proxy :"http://127.0.0.1:38936",
timeout : timeoutsure,
pool : separateReqPool
}, function optionalCallback(err, httpResponse, body) {
if (httpResponse && httpResponse.attempts) {
adet += httpResponse.attempts;
}
if (err) {
return console.error('failed: 49', err);
} else {
data = {
username : line.split(" ")[0],
password : line.split(" ")[1],
submit : "Giriş"
};
request.post({
url : 'http://'+domainim+'/dologin/',
jar : jardata[line.split(" ")[0]],
header : httpResponse.headers,
headers : {
'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
},
maxAttempts : 15,
retryDelay : 2000,
followRedirect : false,
followAllRedirects : false,
// proxy :"http://127.0.0.1:38936",
timeout : timeoutsure,
pool : separateReqPool,
form : data
}, function optionalCallback(err, httpResponse, body) {
if (httpResponse && httpResponse.attempts) {
adet += httpResponse.attempts;
}
if (err) {
return console.error('failed: 49', err);
} else {
request({
url : 'http://'+domainim+'/',
jar : jardata[line.split(" ")[0]],
headers : {
'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
},
maxAttempts : 15,
retryDelay : 2000,
followRedirect : true,
// proxy :"http://127.0.0.1:38936",
timeout : timeoutsure,
pool : separateReqPool
}, function optionalCallback(err, httpResponse, body) {
if (httpResponse && httpResponse.attempts) {
adet += httpResponse.attempts;
}
if (err) {}
else {
if (body.indexOf('class="success"') > -1) {
dataok(line.split(" ")[0], line.split(" ")[1], body.split('class="succes" title="')[1].split('"')[0]);
success++;
if (parseFloat(body.split('class="succes" title="')[1].split('"')[0]) >= mins) {
plusplus++;
extradata(line.split(" ")[0], line.split(" ")[1], body.split('class="succes" title="')[1].split('"')[0]);
} else {
request({
url : 'http://'+domainim+'/refresh/',
jar : jardata[line.split(" ")[0]],
headers : {
'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
},
maxAttempts : 15,
retryDelay : 2000,
followRedirect : true,
// proxy :"http://127.0.0.1:38936",
timeout : timeoutsure,
pool : separateReqPool
}, function optionalCallback(err, httpResponse, body) {
if (httpResponse && httpResponse.attempts) {
adet += httpResponse.attempts;
}
if (err) {}
else {
if (parseFloat(body)>= mins){
plusplus++;
extradata(line.split(" ")[0], line.split(" ")[1], parseFloat(body));
}
}
});
}
} else {
fail++;
}
}
});
}
});
}
});
}
当我在data.txt中的数据类似于5,10,1000代码时没有问题。
但是当数据变得像15,000或50,000或更大时
它只能以％10成功率进行http调用。和其他人失败并给我这样的错误：
failed: 49 Error: connect EADDRINUSE 103.253.186.202:80
at Object._errnoException (util.js:1031:13)
at _exceptionWithHostPort (util.js:1052:20)
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1195:14)
errno: 'EADDRINUSE',
code: 'EADDRINUSE',
syscall: 'connect',
address: '1.2.3.4',
port: 80
我可能做错了什么？ nodejs不能做到这一点吗？
并且jar无法按预期工作，它有时会使用其他会话错误
答案 0 :(得分：1)
您不应该尝试一次拨打15,000个http电话。对于您的服务器来说，这对于同时使用的资源来说太多了，如果请求都是在同一个主机上，那么这可能比单个主机可以同时处理的更多。我猜想EADDRINUSE正是你得到的，因为当你尝试过多的同时请求时，某些资源已经耗尽。
相反，您应该一次运行少得多的请求（您可以尝试介于10到50之间的最佳性能）。还有许多其他答案显示了如何做到这一点。有些使用Bluebird或Async等外部库，有些则手工编写代码。以下是几个例子：
Run 1000 requests so that only 10 runs at a time
How to make millions of parallel http requests from nodejs app?
Error: connect ETIMEDOUT when scraping
Make several requests to an API that can only handle 20 request a minute