使用BeautifulSoup在分页的情况下捕获产品网址数据？

Question

我从网站上获取了html数据并尝试获取产品网址：

def get_soup(url):
soup = None
response = requests.get(url)
if response.status_code == 200:
    html = response.content
    soup = BeautifulSoup(html, "html.parser")
return soup

def get_category_urls(url):
    soup = get_soup(url)
    cat_urls = []
    categories = soup.find('div', attrs={'id': 'menu_oc'})
    if categories is not None:
        for c in categories.findAll('a'):
        if c['href'] is not None:
            cat_urls.append(c['href'])
        return cat_urls

def get_product_urls(url):
     soup = get_soup(url)
     prod_urls = []
     if soup.find('div', attrs={'class': 'pagination'}):
         for link in soup.select('div.links a'):
             if link.string.isdecimal():  # dump next and last links
                prod_urls.append(link['href'])
     print("Found following product urls::", prod_urls)
     return prod_urls

if __name__ == '__main__':
category_urls = get_category_urls(URL)
product_urls = get_product_urls(URL)

如何有效识别上述loc中的分页条件？

带有分页的实际网站的屏幕截图：

没有分页：

• site link
• pagination category

Answer 1

应该没问题

from bs4 import BeautifulSoup
import requests


def get_soup(url):
    soup = None
    response = requests.get(url)
    if response.status_code == 200:
        html = response.content
        soup = BeautifulSoup(html, "html.parser")
    return soup


def get_category_urls(url):
    soup = get_soup(url)
    cat_urls = []
    categories = soup.find('div', attrs={'id': 'menu_oc'})
    if categories is not None:
        for c in categories.findAll('a'):
            if c['href'] is not None:
                cat_urls.append(c['href'])
        return cat_urls


def get_all_products(url):

    prod_urls = []

    soup = get_soup(url)
    prod_urls.append(get_product_urls(soup))

    links = get_pagination(soup)
    print("Found those pages:", links)
    if not links:
        return prod_urls

    for link in links:
        soup = get_soup(link)
        prod_urls.append(get_product_urls(soup))

    print("Found following product urls:", prod_urls)
    return prod_urls


def get_product_urls(soup):
    links = soup.select('div.product-list .span .name a')
    return [link['href'] for link in links]


def get_pagination(soup):
    pages = soup.select('div.pagination div.links a')
    return [link['href'] for link in pages if link.string.isdecimal()]


if __name__ == '__main__':
    URL = 'http://www.example.com/shop/index.php?route=product/category&path=63_64'
    category_urls = get_category_urls(URL)
    product_urls = get_all_products(URL)