Web抓取排序表

时间:2018-02-26 14:00:01

标签: python python-3.x web-scraping beautifulsoup

我正在尝试提取以下网站的数据。我已经设法获取了我需要的数据,但是想要完成此操作的人希望选择访问不同的选项卡。

例如,数据可以按7天,本月,3个月和6个月进行分类。如何在这些标签之间切换?

非常感谢大家! :d

https://www.overbuff.com/heroes

1 个答案:

答案 0 :(得分:0)

该网站是动态的,因此在选择每个标记时通过脚本加载新数据。因此,您需要使用浏览器操作工具,例如selenium

from selenium import webdriver
import collections
import re
from bs4 import BeautifulSoup as soup
def get_hero_data(s):
  names = [a for i, [_, a] in enumerate(re.findall('/heroes/(.*?)">(.*?)</a>', s)) if i%2 != 0]
  numbers = re.findall("<span>(.*?)%</span>", s)
  final_numbers = [[re.findall('[\d\.]+$', a)[0], b, c, d] for a, b, c, d in [numbers[i:i+4] for i in range(0, len(numbers), 4)]]
  return [[a]+b for a, b in zip(names, final_numbers)]

hero_listing = {}
driver = webdriver.Chrome('path/to/driver')
driver.get('https://www.overbuff.com/heroes')
hero_listing['this month'] = get_hero_data(driver.page_source)
buttons = driver.find_elements_by_class_name('filter-option')
for i, t in enumerate([buttons[0], buttons[2], buttons[3]]):
   t.click()
   hero_listing[['this week', 'last 3 months', 'last 6 months'][i]] = get_hero_data(driver.page_source)

输出:

{'this week': [[u'Moira', u'10.44', u'51.39', u'2.42', u'8.64'], [u'D.Va', u'9.31', u'52.42', u'2.35', u'10.08'], [u'Mercy', u'7.61', u'50.47', u'2.61', u'2.14'], [u'Reinhardt', u'6.58', u'51.46', u'2.70', u'9.04'], [u'Roadhog', u'5.41', u'49.64', u'2.08', u'9.30'], [u'Genji', u'5.35', u'50.35', u'2.02', u'11.80'], [u'Zenyatta', u'5.34', u'53.25', u'2.20', u'8.57'], [u'Ana', u'4.91', u'46.90', u'2.28', u'10.39'], [u'Junkrat', u'4.45', u'51.21', u'2.76', u'12.08'], [u'L\xfacio', u'4.42', u'51.22', u'1.94', u'10.23'], [u'McCree', u'4.20', u'48.23', u'1.85', u'11.62'], [u'Tracer', u'4.03', u'49.23', u'1.93', u'9.51'], [u'Winston', u'3.95', u'48.58', u'2.13', u'6.46'], [u'Soldier: 76', u'3.91', u'49.95', u'2.31', u'10.84'], [u'Zarya', u'3.23', u'47.79', u'2.29', u'10.99'], [u'Orisa', u'2.93', u'52.61', u'2.64', u'10.85'], [u'Pharah', u'2.90', u'50.23', u'2.09', u'12.99'], [u'Widowmaker', u'2.57', u'48.16', u'2.09', u'9.63'], [u'Reaper', u'2.38', u'49.77', u'2.11', u'16.33'], [u'Hanzo', u'1.98', u'45.91', u'2.16', u'13.63'], [u'Symmetra', u'0.83', u'59.44', u'4.07', u'7.53'], [u'Doomfist', u'0.81', u'48.29', u'1.68', u'7.37'], [u'Torbj\xf6rn', u'0.68', u'55.48', u'4.27', u'13.65'], [u'Mei', u'0.64', u'47.47', u'2.46', u'8.15'], [u'Bastion', u'0.62', u'48.59', u'2.70', u'16.44'], [u'Sombra', u'0.53', u'45.10', u'2.31', u'13.26']], 'this month': [[u'Moira', u'9.89', u'51.47', u'2.45', u'8.96'], [u'D.Va', u'9.35', u'52.51', u'2.40', u'10.47'], [u'Mercy', u'8.31', u'50.48', u'2.63', u'2.45'], [u'Reinhardt', u'6.15', u'51.35', u'2.74', u'9.19'], [u'Zenyatta', u'5.63', u'53.29', u'2.25', u'9.01'], [u'Roadhog', u'5.62', u'49.72', u'2.06', u'9.72'], [u'Genji', u'5.19', u'50.51', u'2.06', u'12.26'], [u'Ana', u'4.87', u'46.59', u'2.35', u'10.88'], [u'Junkrat', u'4.59', u'51.40', u'2.81', u'12.78'], [u'McCree', u'4.29', u'48.36', u'1.91', u'12.01'], [u'L\xfacio', u'4.10', u'51.36', u'1.93', u'10.66'], [u'Winston', u'4.02', u'48.74', u'2.12', u'6.84'], [u'Soldier: 76', u'3.90', u'50.10', u'2.35', u'11.36'], [u'Tracer', u'3.87', u'49.34', u'1.97', u'9.87'], [u'Orisa', u'3.28', u'53.01', u'2.64', u'11.41'], [u'Zarya', u'3.19', u'47.80', u'2.37', u'11.40'], [u'Pharah', u'2.96', u'50.26', u'2.11', u'13.26'], [u'Widowmaker', u'2.55', u'48.45', u'2.10', u'10.02'], [u'Reaper', u'2.36', u'49.80', u'2.07', u'16.86'], [u'Hanzo', u'1.91', u'46.05', u'2.21', u'14.13'], [u'Symmetra', u'0.83', u'59.90', u'4.19', u'7.71'], [u'Doomfist', u'0.75', u'48.44', u'1.79', u'7.41'], [u'Torbj\xf6rn', u'0.67', u'55.84', u'4.42', u'14.49'], [u'Bastion', u'0.61', u'49.17', u'2.61', u'17.19'], [u'Mei', u'0.59', u'48.34', u'2.41', u'8.57'], [u'Sombra', u'0.51', u'45.56', u'2.34', u'13.73']], 'last 6 months': [[u'Mercy', u'11.71', u'51.99', u'2.63', u'3.08'], [u'D.Va', u'8.66', u'52.70', u'2.38', u'11.22'], [u'Moira', u'8.49', u'51.01', u'2.43', u'8.82'], [u'Reinhardt', u'6.44', u'51.46', u'2.74', u'9.61'], [u'Roadhog', u'5.39', u'50.04', u'2.01', u'10.88'], [u'Zenyatta', u'4.89', u'53.11', u'2.25', u'9.91'], [u'Junkrat', u'4.86', u'52.13', u'2.77', u'15.09'], [u'Genji', u'4.82', u'50.30', u'2.05', u'13.40'], [u'Ana', u'4.46', u'45.60', u'2.34', u'11.57'], [u'McCree', u'4.35', u'48.64', u'1.92', u'13.52'], [u'Soldier: 76', u'4.34', u'50.23', u'2.35', u'12.68'], [u'Winston', u'3.96', u'48.88', u'2.07', u'7.26'], [u'Zarya', u'3.73', u'48.18', u'2.37', u'12.09'], [u'Tracer', u'3.59', u'49.10', u'1.93', u'10.90'], [u'L\xfacio', u'3.38', u'50.80', u'1.83', u'10.59'], [u'Pharah', u'3.15', u'50.86', u'2.08', u'14.57'], [u'Orisa', u'3.04', u'53.14', u'2.66', u'12.36'], [u'Widowmaker', u'2.41', u'48.51', u'2.10', u'11.12'], [u'Reaper', u'2.36', u'50.10', u'1.99', u'18.23'], [u'Hanzo', u'1.99', u'46.14', u'2.21', u'15.63'], [u'Symmetra', u'0.86', u'60.06', u'4.24', u'7.85'], [u'Doomfist', u'0.72', u'48.43', u'1.77', u'8.44'], [u'Torbj\xf6rn', u'0.68', u'55.74', u'4.42', u'15.23'], [u'Bastion', u'0.65', u'50.14', u'2.58', u'18.92'], [u'Mei', u'0.57', u'48.57', u'2.43', u'9.33'], [u'Sombra', u'0.50', u'45.14', u'2.36', u'14.04']], 'last 3 months': [[u'Mercy', u'11.71', u'51.99', u'2.63', u'3.08'], [u'D.Va', u'8.66', u'52.70', u'2.38', u'11.22'], [u'Moira', u'8.49', u'51.01', u'2.43', u'8.82'], [u'Reinhardt', u'6.44', u'51.46', u'2.74', u'9.61'], [u'Roadhog', u'5.39', u'50.04', u'2.01', u'10.88'], [u'Zenyatta', u'4.89', u'53.11', u'2.25', u'9.91'], [u'Junkrat', u'4.86', u'52.13', u'2.77', u'15.09'], [u'Genji', u'4.82', u'50.30', u'2.05', u'13.40'], [u'Ana', u'4.46', u'45.60', u'2.34', u'11.57'], [u'McCree', u'4.35', u'48.64', u'1.92', u'13.52'], [u'Soldier: 76', u'4.34', u'50.23', u'2.35', u'12.68'], [u'Winston', u'3.96', u'48.88', u'2.07', u'7.26'], [u'Zarya', u'3.73', u'48.18', u'2.37', u'12.09'], [u'Tracer', u'3.59', u'49.10', u'1.93', u'10.90'], [u'L\xfacio', u'3.38', u'50.80', u'1.83', u'10.59'], [u'Pharah', u'3.15', u'50.86', u'2.08', u'14.57'], [u'Orisa', u'3.04', u'53.14', u'2.66', u'12.36'], [u'Widowmaker', u'2.41', u'48.51', u'2.10', u'11.12'], [u'Reaper', u'2.36', u'50.10', u'1.99', u'18.23'], [u'Hanzo', u'1.99', u'46.14', u'2.21', u'15.63'], [u'Symmetra', u'0.86', u'60.06', u'4.24', u'7.85'], [u'Doomfist', u'0.72', u'48.43', u'1.77', u'8.44'], [u'Torbj\xf6rn', u'0.68', u'55.74', u'4.42', u'15.23'], [u'Bastion', u'0.65', u'50.14', u'2.58', u'18.92'], [u'Mei', u'0.57', u'48.57', u'2.43', u'9.33'], [u'Sombra', u'0.50', u'45.14', u'2.36', u'14.04']]}