我使用nodejs向crawler网站写了一个简单的代码所有url。
当我运行代码时,内存使用率不断增长,即使我没有将任何数据放入块和队列。
如何防止内存使用量增加。
var http = require("http");
var options = {
host: "...",
path: "...",
method: "GET",
chunk: "",
queue: []
}
function http_request(options) {
setTimeout(function() {
var req = http.request(options, function(res) {
if(res.statusCode === 200){
res.on('data', function(chunk){
options.chunk += chunk;
});
res.on('end', function(){
http_request_chunk_handler(options);
});
}
else {
res.destroy();
http_request_set(options)
}
});
req.end();
}, 1000);
}
function http_request_chunk_handler(options){
//do somthing here, put some url into options.queue
//write data to local file
http_request_set(options);
}
function http_request_set(options){
//do somthing here, shift queue item to option.path and reset option.chunk
http_request(options);
}
http_request(options);