使用beautifulsoup在Python中进行足球网抓取

Question

问题

我设法从“ goal.com”中刮出了俱乐部的名称，但是现在我需要利用这些数据。我不知道如何从该数据中选择一个特定的俱乐部并使用它，因此我可以为与特定团队的下一场比赛进行倒计时。

代码

from requests import get
from bs4 import BeautifulSoup

#target site
url = "https://www.goal.com/en-in/team/real-madrid/fixtures-results/3kq9cckrnlogidldtdie2fkbl"

#get data from site
response = get(url)

#print data
print(response.status_code)

#get raw html data
match = BeautifulSoup(response.content, "html.parser")

#view the html data
#print(match.prettify)
#match_div = match.findAll('div')
#match_div = match.findAll('div', {"class":"match-data"})
#match_div = match.findAll('div', {"class":"team-away win"})
#match_div = match.find({"class":"team-name"})
#match_div = match.findAll('div', {"class":"team-away win"})
#opponent = match.find('span', {"class":"team-name"})
#opponent = match.find('span', {"class":"team-away win"})
opponent = match.findAll('span', {"class":"team-name"})

Answer 1

我喜欢使用xpath，它非常强大。 输入：

from requests import get
from bs4 import BeautifulSoup
from lxml import html
import datetime

#target site
url = "https://www.goal.com/en-in/team/real-madrid/fixtures-results/3kq9cckrnlogidldtdie2fkbl"

#get data from site
response = get(url)

#print status code
print(response.status_code)

#get raw html data
tree = html.fromstring(response.content)

#get the dates
dates = tree.xpath("//a[@class='match-main-data-link']/div/span[not(text())]/../time")
dates = [date.get('datetime') for date in dates]

#get the teams
teams = tree.xpath("//a[@class='match-main-data-link']/div/span[not(text())]/../../div/div/div/span[@class='team-name']")
teams = [team.text for team in teams]

print(dates)
print(teams)

输出：

200
['2019-03-31T18:45:00+00:00', '2019-04-03T19:30:00+00:00', '2019-04-06T14:15:00+00:00', '2019-04-15T19:00:00+00:00']
['Real Madrid', 'Huesca', 'Valencia', 'Real Madrid', 'Real Madrid', 'Eibar', u'Legan\xe9s', 'Real Madrid']

Answer 2

以下内容忽略了过去的比赛，并忽略了未来的日期和团队，并包括了tbc。因为我认为仅需要倒数计时，所以它会查看日期时间以确定将来的比赛。

from datetime import datetime
from bs4 import BeautifulSoup as bs
import requests
from dateutil import parser
import pytz

utc=pytz.UTC

r = requests.get('https://www.goal.com/en-in/team/real-madrid/fixtures-results/3kq9cckrnlogidldtdie2fkbl')
soup = bs(r.content, 'lxml')
items = soup.select('.match-main-data')
times = [item.find('time')['datetime'] if item.find('time') is not None else 'TBC' for item in items]
matches = [item['content'] for item in soup.select('[itemprop="name"][content]')]
results = list(zip(matches, times))
currentUTC = datetime.utcnow()
data = []

for result in results:
    if result[1] == 'TBC':
        data.append(result)
    else:
        dt = parser.parse(result[1])
        if dt > utc.localize(currentUTC):
            data.append(result)

print(data)