我正在使用Beautiful Soup在Python中创建一个Web爬虫。我想从某个div获取链接,我现在的代码不会打印任何内容。
import requests
from bs4 import BeautifulSoup
def spider(max_pages):
page = 1
while page <= max_pages:
url = 'https://thenewboston.com/'
source = requests.get(url)
plain_text = source.text
obj = BeautifulSoup(plain_text, "html5lib")
for link in obj.find_all('div', {'class': 'videos-top-courses'}):
href = 'https://thenewboston.com/', link.get('href')
print(href)
page += 1
spider(1)
答案 0 :(得分:0)
您可以使用以下内容:
from bs4 import BeautifulSoup
from urllib2 import urlopen
soup = BeautifulSoup(urlopen("https://thenewboston.com/"),'html.parser')
videos = soup.findAll('td', {'class': 'video-icon-column'})
for td in videos:
print td.a['href']
答案 1 :(得分:0)
您必须找到<table>
而不是<div>
,之后您可以找到<a>
来获取href
import requests
from bs4 import BeautifulSoup
def spider(max_pages):
page = 1
while page <= max_pages:
url = 'https://thenewboston.com/'
source = requests.get(url)
plain_text = source.text
obj = BeautifulSoup(plain_text, "html5lib")
for table in obj.find_all('table', {'class': 'videos-top-courses'}):
for a in table.find_all('a'):
print(a.get('href'))
page += 1
spider(1)