Question

我写了一段代码：

use LWP::UserAgent;
use HTTP::Cookies;
use threads;
use threads::shared;
$| = 1;

$threads = 50; 
my @urls : shared = loadf('url.txt');   

my @thread_list = ();
$thread_list[$_] = threads->create(\&thread) for 0 .. $threads - 1;
$_->join for @thread_list;
thread();

sub thread
{
    my ($web, $ck) = browser();
    while(1)
    {
        my $url = shift @urls;
        if(!$url)
        {
            last;
        }
        $code = $web->get($url)->code;
        print "[+] $url - code: $code\n";
        if($code == 200)
        {
            open F, ">>200.txt";
            print F $url."\n";
            close F;
        }
        elsif($code == 301)
        {
            open F, ">>301.txt";
            print F $url."\n";
            close F;
        }
        else
        {
            open F, ">>else.txt";
            print F "$url code - $code\n";
            close F; 
        }
    }
}


sub loadf {
    open (F, "<".$_[0]) or erroropen($_[0]); 
    chomp(my @data = <F>);
    close F;
    return @data;
}

sub browser 
{
 my $web = new LWP::UserAgent;
 my $ck = new HTTP::Cookies;
    $web->cookie_jar($ck);
    $web->agent('Opera/9.80 (Windows 7; U; en) Presto/2.9.168 Version/11.50');
    $web->timeout(5);
    return $web, $ck;
}

在工作一段时间后，物理存储空间已满。你可以帮我用AnyEvent重新编写它吗？我试过，但我的代码不起作用。我读到它会帮助我保护一些记忆。非常感谢任何帮助者。

Answer 1

线程是邪恶的;考虑使用异步AnyEvent::Net::Curl::Queued：

#!/usr/bin/env perl
use common::sense;

use File::Slurp;
use YADA;

my @url = read_file('url.txt');
YADA->new(50)->append(
    \@url => {
        retry   => 0,
        timeout => 5,
        opts    => {
            useragent   => 'Opera/9.80 (Windows 7; U; en) Presto/2.9.168 Version/11.50',
            #verbose     => 1,
        },
    } => sub {
        my $self = shift;

        my $code = $self->getinfo('response_code');
        my $url = $self->final_url;

        say "[+] $url - code: $code";
        given ($code) {
            when (200) {
                open my $fh, '>>', '200.txt';
                say {$fh} $url;
                close $fh;
            } when (301) {
                open my $fh, '>>', '301.txt';
                say {$fh} $url;
                close $fh;
            } default {
                open my $fh, '>>', 'else.txt';
                say {$fh} "$url code - $code";
                close $fh;
            }
        }
    },
)->wait;

将代码从Threads重写为AnyEvent

1 个答案: