Question

有一些我需要将所有URL和内容输入到Elasticsearch中的现有站点，因此使用一些示例，我构建了一种方法来获取站点上的所有内部链接并将它们放入数组中。问题是我看到的每个例子都不是完全递归的，有深度设置。我一直在尝试过去6个小时（严重）的不同方式让它完全递归。这就是我现在正在做的事情，但我认为我无限循环和崩溃，因为经过一分钟的运行后我没有错误，只是一个“没有数据接收”页面。我对任何有关更好方法的建议持开放态度。

    <?php
    set_time_limit (1209600);
    ini_set('memory_limit', '-1');

    $seen = array();
    $urls = crawl_page("http://example.com", $seen);

    foreach($urls as $url){
        echo $url.'<br />';
    }

    function crawl_page($url, $seen){

        //CURL TO GRAB PAGE
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_TIMEOUT, 60);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
        $result = curl_exec ($ch);
        curl_close ($ch);


        //URL NO HTTP/S OR WWW
        $urlStripped = str_replace('www', '', $url);
        $urlStripped = str_replace('http://', '', $urlStripped);
        $urlStripped = str_replace('https://', '', $urlStripped);

        //ADD THIS URL TO THE ARRAY
        if(!in_array($url, $seen)){
            $seen[] = $url;
        }


        //GET ALL LINKS IN PAGE
        $stripped_file = strip_tags($result, "<a>");
        preg_match_all("/<a[\s]+[^>]*?href[\s]?=[\s\"\']+"."(.*?)[\"\']+.*?>"."([^<]+|.*?)?<\/a>/", $stripped_file, $matches, PREG_SET_ORDER ); 
        foreach($matches as $match){
            $href = $match[1];
            //MAKE SURE LINK ISNT DUPLICATE AND MAKE SURE THE LINK IS INTERNAL
            if(!in_array($href, $seen) && is_in_string($urlStripped, $href)){
                $seen[] = $href;
            }
        }

        //HERE'S WHERE THE PROBLEM LIES, ATTEMPTING TO MAKE THIS RECURSIVE. 
        //I'VE DONE THIS MANY DIFFERENT WAYS WITH NO LUCK.
        //I DON'T REALLY HAVE A REASON FOR IT'S CURRENT STATE.
        //I ENDED UP TAKING SHOTS IN THE DARK AND THATS WHAT LED ME TO ASK ON STACKOVERFLOW
        $seenTemp1 = $seen;
        foreach($seenTemp1 as $aUrl){
            $seenTemp2 = crawl_page($aUrl, $seenTemp1);
            $seen = array_merge($seen, $seenTemp2);
        }

        //RETRUN ARRAY
        return $seen;
    }

    function is_in_string($needle, $string){
        $before = strlen($string);
        $after = strlen(str_replace($needle, '', $string));

        if($before != $after){
            return true;
        }
        else{
            return false;
        }
    }

?>

递归网站爬虫PHP

0 个答案: