Question

from bs4 import BeautifulSoup
import urllib
import json
import os

jaren = [str("2012"), str("2010"), str("2006"), str("2003"),str("2002"), str("1998"), str("1994"), str("1989"), str("1986"), str("1982"), str("1981"), str("1977"), str("1972"), str("1971"), str("1967"), str("1963"), str("1959"), str("1956")]
DESIRED_COLUMNS = {1, 2, 5} #scrapes only afk, aantal & zetels
verkiezingsData = []

filename = raw_input('Enter a filename: ') or 'data.json'

#open file and open json array
with open(filename, "w") as file:
   file.write("[{")
for Jaargetal in jaren:

     #url source 
     r = urllib.urlopen("http://www.nlverkiezingen.com/TK" + Jaargetal +".html").read()
     soup = BeautifulSoup(r, "html.parser")
     tables = soup.find_all("table")
     for table in tables:
         header = soup.find_all("h1")[0].getText()

         #print header
         with open(filename, "a+") as file:
              file.write("\"%s\": [" % header) #header as beginning json
         trs = table.find_all("tr")[0].getText()
         del verkiezingsData[:] #clear list before adding new data


         #add the 3 columns to a list
         for tr in table.find_all("tr")[1:22]: #22 columns top till bottom
             for index, val in enumerate(tr.find_all('td')):
                  if index in DESIRED_COLUMNS: #linkt naar desired columns bovenin
                      verkiezingsData.append(val.getText().strip())


         #json array van de 3 vallues
         for a, b, c in zip(verkiezingsData[::3],        verkiezingsData[1::3], verkiezingsData[2::3]): #link naar desired columns 1,2,5
             data2 = {'afk':a,"aantal":b, "zetels":c}

             #file writing
             with open(filename, 'a') as outfile:
                 json.dump(data2, outfile)
                    outfile.write(",")


        #open file, delete last comma and close array
        with open(filename, 'ab+') as file:
                file.seek(-1, os.SEEK_END)
                file.truncate()
                file.write("],")

#open file, delete last comma, and close array
with open(filename, 'r+b') as file:
    file.seek(-1, os.SEEK_END)
    file.truncate()
    file.write("}]")

#open file and pretty print json data
with open(filename, 'r') as file:
    prettydata = json.load(file)
with open(filename, 'w') as file:
    json.dump(prettydata, file, sort_keys=True, indent=4, separators=(',', ': '))

我从nlverkiezingen.com上刮了一个刮刀

它刮掉了Aantal / Afk / Zetels

现在我有：

for tr in table.find_all("tr")[1:22]: #22 columns top till bottom

唯一的问题是，例如：

http://www.nlverkiezingen.com/TK1959.html

1959年＆gt; Zetels直到第8行

1963＆gt; Zetels直到第10行

如何在我的代码中进行调整？代码知道当表行没有“zetels”时停止。

Answer 1

只需检查列中没有文本并退出循环：

# add the 3 columns to a list
for tr in table.find_all("tr")[1:22]: #22 columns top till bottom
    cells = tr.find_all('td')

    # no Zetels value - break
    if not cells[5].get_text():
        break

    for index, val in enumerate(cells):
        if index in DESIRED_COLUMNS: #linkt naar desired columns bovenin
            verkiezingsData.append(val.getText().strip())

行需要停止/ beautifulsoup / JSON

1 个答案: