Question

当尝试抓取该网站的多个页面时，没有任何回报。我通常会检查以确保我创建的所有lists的长度都相等，但是所有返回的都是len = 0。

我已经使用类似的代码来抓取其他网站，所以为什么此代码无法正常工作？

我尝试过一些解决方案，但并未达到目的：requests.Session()如this answer所建议，.json如suggested here中所建议。

for page in range(100, 350):

    page = requests.get("https://www.ghanaweb.com/GhanaHomePage/election2012/parliament.constituency.php?ID=" + str(page) + "&res=pm")

    page.encoding = page.apparent_encoding

    if not page:
        pass

    else:

        soup = BeautifulSoup(page.text, 'html.parser')

        ghana_tbody = soup.find_all('tbody')

        sleep(randint(2,10))

         for container in ghana_tbody:

            #### CANDIDATES ####
            candidate = container.find_all('div', class_='can par')
            for data in candidate:
                cand = data.find('h4')
                for info in cand:
                    if cand is not None:
                        can2 = info.get_text()
                        can.append(can2)

            #### PARTY NAMES ####
            partyn = container.find_all('h5')
            for data in partyn:
                if partyn is not None:
                    partyn2 = data.get_text()
                    pty_n.append(partyn2)

            #### CANDIDATE VOTES ####
            votec = container.find_all('td', class_='votes')
            for data in votec:
                if votec is not None:
                    votec2 = data.get_text()
                    cv1.append(votec2)

            #### CANDIDATE VOTE SHARE ####
            cansh = container.find_all('td', class_='percent')
            for data in cansh:
                if cansh is not None:
                    cansh2 = data.get_text()
                    cvs1.append(cansh2)

        #### TOTAL  VOTES ####`
        tfoot = soup.find_all('tr', class_='total')
        for footer in tfoot:
            fvote = footer.find_all('td', class_='votes')
            for data in fvote:
                if fvote is not None:
                    fvote2 = data.get_text()
                    fvoteindiv = [fvote2]
                    fvotelist = fvoteindiv * (len(pty_n) - len(vot1))
                    vot1.extend(fvotelist)

在此先感谢您的帮助！

Answer 1

我做了一些简化更改。需要更改的主要更改是：

--output-path
ghana_tbody = soup.find_all('table', class_='canResults')

我仅针对第112页对此进行了测试；生命太短暂了。

can2 = info # not info.get_text()

打印：

import requests
from bs4 import BeautifulSoup
from random import randint
from time import sleep

can = []
pty_n = []
cv1 = []
cvs1 = []
vot1 = []

START_PAGE = 112
END_PAGE = 112

for page in range(START_PAGE, END_PAGE + 1):
    page = requests.get("https://www.ghanaweb.com/GhanaHomePage/election2012/parliament.constituency.php?ID=112&res=pm")
    page.encoding = page.apparent_encoding
    if not page:
        pass
    else:
        soup = BeautifulSoup(page.text, 'html.parser')
        ghana_tbody = soup.find_all('table', class_='canResults')
        sleep(randint(2,10))
        for container in ghana_tbody:

            #### CANDIDATES ####
            candidate = container.find_all('div', class_='can par')
            for data in candidate:
                cand = data.find('h4')
                for info in cand:
                    can2 = info # not info.get_text()
                    can.append(can2)

            #### PARTY NAMES ####
            partyn = container.find_all('h5')
            for data in partyn:
                partyn2 = data.get_text()
                pty_n.append(partyn2)


            #### CANDIDATE VOTES ####
            votec = container.find_all('td', class_='votes')
            for data in votec:
                votec2 = data.get_text()
                cv1.append(votec2)

            #### CANDIDATE VOTE SHARE ####
            cansh = container.find_all('td', class_='percent')
            for data in cansh:
                cansh2 = data.get_text()
                cvs1.append(cansh2)

        #### TOTAL  VOTES ####`
        tfoot = soup.find_all('tr', class_='total')
        for footer in tfoot:
            fvote = footer.find_all('td', class_='votes')
            for data in fvote:
                fvote2 = data.get_text()
                fvoteindiv = [fvote2]
                fvotelist = fvoteindiv * (len(pty_n) - len(vot1))
                vot1.extend(fvotelist)

print('can = ', can)
print('pty_n = ', pty_n)
print('cv1 = ', cv1)
print('cvs1 = ', cvs1)
print('vot1 = ', vot1)

请确保首先将START_PAGE和END_PAGE分别更改为100和350。

Python request.get（）循环不返回任何内容

1 个答案: