Question

我有这个代码尝试使用php获取分页链接，但结果并不安静。任何人都可以帮助我。

我得到的只是第一个链接的重复实例。

<?php
include_once('simple_html_dom.php');
function dlPage($href) {
    $curl = curl_init();
    curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
    curl_setopt($curl, CURLOPT_HEADER, false);
    curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($curl, CURLOPT_URL, $href);
    curl_setopt($curl, CURLOPT_REFERER, $href);
    curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
    curl_setopt($curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4");
    $str = curl_exec($curl);
    curl_close($curl);

    // Create a DOM object
    $dom = new simple_html_dom();
    // Load HTML from a string
    $dom->load($str);


    $Next_Link = array();
    foreach($dom->find('a[title=Next]') as $element){
        $Next_Link[] = $element->href; 
    }

    print_r($Next_Link);

    $next_page_url = $Next_Link[0];
    if($next_page_url !='') {
        echo '<br>' . $next_page_url;
        $dom->clear();
        unset($dom);

        //load the next page from the pagination to collect the next link
        dlPage($next_page_url);
    }

}

$url = 'https://www.jumia.com.gh/phones/';
$data = dlPage($url);
//print_r($data)
?>

我想得到的是什么 mySiteUrl/?facet_is_mpg_child=0&viewType=gridView&page=2
mySiteUrl//?facet_is_mpg_child=0&viewType=gridView&page=3

。。。到分页的最后一个环节。请帮忙

Answer 1

在这里。看看我htmlspecialchars_decode链接。导致卷曲中的href不应该是＆amp;喜欢在xml中。应该将dlPage的返回值作为分页中的最后一个链接。我明白了。

<?php
include_once('simple_html_dom.php');

function dlPage($href, $already_loaded = array()) {
    $curl = curl_init();
    curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
    curl_setopt($curl, CURLOPT_HEADER, false);
    curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($curl, CURLOPT_URL, $href);
    curl_setopt($curl, CURLOPT_REFERER, $href);
    curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
    curl_setopt($curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4");
    $htmlPage = curl_exec($curl);
    curl_close($curl);

    echo "Loading From URL:" . $href . "<br/>\n";
    $already_loaded[$href] = true;

    // Create a DOM object
    $dom = file_get_html($href);
    // Load HTML from a string
    $dom->load($htmlPage);

    $next_page_url = null;
    $items = $dom->find('ul[class="osh-pagination"] li[class="item"] a[title="Next"]');

    foreach ($items as $item) {
        $link = htmlspecialchars_decode($item->href);
        if (!isset($already_loaded[$link])) {
            $next_page_url = $link;
            break;
        }
    }

    if ($next_page_url !== null) {
        $dom->clear();
        unset($dom);

        //load the next page from the pagination to collect the next link
        return dlPage($next_page_url, $already_loaded);
    }

    return $href;
}

$url = 'https://www.jumia.com.gh/phones/';
$data = dlPage($url);
echo "DATA:" . $data . "\n";

输出是：

Loading From URL:https://www.jumia.com.gh/phones/<br/>
Loading From URL:https://www.jumia.com.gh/phones/?facet_is_mpg_child=0&viewType=gridView&page=2<br/>
Loading From URL:https://www.jumia.com.gh/phones/?facet_is_mpg_child=0&viewType=gridView&page=3<br/>
Loading From URL:https://www.jumia.com.gh/phones/?facet_is_mpg_child=0&viewType=gridView&page=4<br/>
Loading From URL:https://www.jumia.com.gh/phones/?facet_is_mpg_child=0&viewType=gridView&page=5<br/>
DATA:https://www.jumia.com.gh/phones/?facet_is_mpg_child=0&viewType=gridView&page=5

使用php和简单的html dom获取Paginated链接

1 个答案: