在div中抓取链接

Question

我正在使用Beautiful Soup在Python中创建一个Web爬虫。我想从某个div获取链接，我现在的代码不会打印任何内容。

import requests
from bs4 import BeautifulSoup

def spider(max_pages):
    page = 1
    while page <= max_pages:
        url = 'https://thenewboston.com/'
        source = requests.get(url)
        plain_text = source.text
        obj = BeautifulSoup(plain_text, "html5lib")

        for link in obj.find_all('div', {'class': 'videos-top-courses'}):
            href = 'https://thenewboston.com/', link.get('href')
            print(href)
        page += 1

spider(1)

Answer 1

您可以使用以下内容：

from bs4 import BeautifulSoup
from urllib2 import urlopen
soup = BeautifulSoup(urlopen("https://thenewboston.com/"),'html.parser')
videos = soup.findAll('td', {'class': 'video-icon-column'})
for td in videos:
    print td.a['href']

Answer 2

您必须找到<table>而不是<div>，之后您可以找到<a>来获取href

import requests
from bs4 import BeautifulSoup

def spider(max_pages):
    page = 1
    while page <= max_pages:
        url = 'https://thenewboston.com/'
        source = requests.get(url)
        plain_text = source.text
        obj = BeautifulSoup(plain_text, "html5lib")

        for table in obj.find_all('table', {'class': 'videos-top-courses'}):
            for a in table.find_all('a'):
                print(a.get('href'))
        page += 1

spider(1)