Question

我要通过网络剪贴该网站“ http://mbsweblist.fsco.gov.on.ca/agents.aspx”。我有代理商的姓氏。使用姓氏搜索时，它会返回许可证ID，该许可证ID是一个超链接，可将您带到其许可证信息（例如到期日期）的另一页。

这是我到目前为止的代码。但是一次只能搜索1个名称。我该如何搜索和检索许可证名称和到期日期，例如说一个1000个名称？

import requests
from bs4 import BeautifulSoup

def get_result_page_ontario(name):
r = requests.post("http://mbsweblist.fsco.gov.on.ca/agents.aspx",
data={
'ctl00$ctl00$MainPlaceHolder$Content4$bkmbname:': 'crossley',
'_EVENTTARGET': '',
'__EVENTARGUMENT': '',
'__LASTFOCUS': '',
'__VIEWSTATE': '/wEPDwULLTEwMzk1Nzk2NDAPZBYCZg9kFgJmD2QWBAIBD2QWAgIBDxYCHgRUZXh0BcUHPGxpbmsgcmVsPSdzdHlsZXNoZWV0JyBocmVmPSdodHRwOi8vd3d3LmZzY28uZ292Lm9uLmNhLy9TdHlsZSBMaWJyYXJ5L0ZTQ08vSW50ZXJuZXQvQ1NTL21hc3Rlci5jc3MnIHR5cGU9J3RleHQvY3NzJyBtZWRpYT0nc2NyZWVuJyAvPjxsaW5rIHJlbD0nc3R5bGVzaGVldCcgaHJlZj0naHR0cDovL3d3dy5mc2NvLmdvdi5vbi5jYS8vU3R5bGUgTGlicmFyeS9GU0NPL0ludGVybmV0L0NTUy9jaGFuZ2VtZS5jc3MnIHR5cGU9J3RleHQvY3NzJyBtZWRpYT0nc2NyZWVuJy8+PCEtLVtpZiBsdGUgSUUgNl0+PGxpbmsgcmVsPSdzdHlsZXNoZWV0JyBocmVmPSdodHRwOi8vd3d3LmZzY28uZ292Lm9uLmNhLy9TdHlsZSBMaWJyYXJ5L0ZTQ08vSW50ZXJuZXQvQ1NTL2llNi5jc3MnIHR5cGU9J3RleHQvY3NzJyBtZWRpYT0nc2NyZWVuJyAvPiA8IVtlbmRpZl0tLT48bGluayByZWw9J3N0eWxlc2hlZXQnIGhyZWY9J2h0dHA6Ly93d3cuZnNjby5nb3Yub24uY2EvL1N0eWxlIExpYnJhcnkvRlNDTy9JbnRlcm5ldC9DU1MvcHJpbnQuY3NzJyB0eXBlPSd0ZXh0L2NzcycgbWVkaWE9J3ByaW50JyAvPjxsaW5rIHJlbD0nc3R5bGVzaGVldCcgaHJlZj0naHR0cDovL3d3dy5mc2NvLmdvdi5vbi5jYS8vU3R5bGUgTGlicmFyeS9GU0NPL0ludGVybmV0L0NTUy9tb2JpbGUuY3NzJyB0eXBlPSd0ZXh0L2NzcycgbWVkaWE9J2hhbmRoZWxkJyAvPjxsaW5rIHJlbD0nc3R5bGVzaGVldCcgaHJlZj0naHR0cDovL3d3dy5mc2NvLmdvdi5vbi5jYS8vU3R5bGUgTGlicmFyeS9GU0NPL0ludGVybmV0L0NTUy9GU0NPQ3VzdG9tLmNzcycgdHlwZT0ndGV4dC9jc3MnIG1lZGlhPSdzY3JlZW4nIC8+PGxpbmsgcmVsPSdzdHlsZXNoZWV0JyBocmVmPSdodHRwOi8vd3d3LmZzY28uZ292Lm9uLmNhLy9TdHlsZSBMaWJyYXJ5L0ZTQ08vSW50ZXJuZXQvQ1NTL2dlbmVyYWwuY3NzJyB0eXBlPSd0ZXh0L2NzcycgbWVkaWE9J3NjcmVlbicgLz5kAgMPZBYKAgkPFgIeBGhyZWYFHmh0dHA6Ly93d3cuZnNjby5nb3Yub24uY2EvL2VuLxYCAgEPDxYCHghJbWFnZVVybAVKaHR0cDovL3d3dy5mc2NvLmdvdi5vbi5jYS8vU3R5bGUgTGlicmFyeS9GU0NPL0ludGVybmV0L0ltYWdlcy9GU0NPbG9nby5naWZkZAILD2QWCAIBDxYCHwEFHmh0dHA6Ly93d3cuZnNjby5nb3Yub24uY2EvL2VuL2QCAw8WAh8BBTZodHRwOi8vd3d3LmZzY28uZ292Lm9uLmNhLy9lbi9BYm91dC9QYWdlcy9kZWZhdWx0LmFzcHhkAgUPFgIfAQU8aHR0cDovL3d3dy5mc2NvLmdvdi5vbi5jYS8vZW4vQWJvdXQvUGFnZXMvZGVmYXVsdC5hc3B4I3N1cGVyZAIHDxYCHwEFPmh0dHA6Ly93d3cuZnNjby5nb3Yub24uY2EvL2VuL0Fib3V0L2NvbnRhY3QvUGFnZXMvZGVmYXVsdC5hc3B4ZAIPD2QWAgIBD2QWAgIBD2QWCgICD2QWAgIBD2QWAmYPEGRkFgFmZAIDD2QWAgIBD2QWAmYPDxYEHglCYWNrQ29sb3IKpAEeBF8hU0ICCGRkAgUPZBYEZg8PFgIfAAUXQWdlbnQvQnJva2VyIExhc3QgTmFtZTpkZAIBD2QWAmYPDxYEHwMKpAEfBAIIZGQCBg9kFgICAQ9kFgJmDw8WBB8DCqQBHwQCCGRkAgcPZBYCAgEPZBYCZg8QDxYEHgdDaGVja2VkaB4HRW5hYmxlZGhkZGRkAhMPFgIfAQU+aHR0cDovL3d3dy5mc2NvLmdvdi5vbi5jYS8vZW4vQWJvdXQvY29udGFjdC9QYWdlcy9kZWZhdWx0LmFzcHhkAhUPFgIfAQUjaHR0cDovL3d3dy5mc2NvLmdvdi5vbi5jYS8vZW4vSGVscC9kZHbfkxqmdOpuM/SlhkyMUMPoqe7xpqvhLZfOLed46aNe',
'__VIEWSTATEGENERATOR': '160FAD78',
'__EVENTVALIDATION': '/wEdAA7Q4cvANrpN5o7qvL/AjrRj3ieM1gYOLXKttt+dfEvWRlC30MDmLdG0SqBLRp4Edr0smFtAlmZS0w2+VR/uBTUgaQpcj9uHtwyf+rB2XgM9KzV/VOD8+NmupvzRtXx7cILCclsqUAusKL6yu6LPJYCYN93eHCeJb+Wv6Dc0KUw/tN8+BEUTySkHJ91vQ/nzu4CsVp8wE0Bpab2MDGOxbDBR3HNdVeUhWlxmX6SwVRp9GtD5VgtZgtwF9KTW5gMitXmBcXMJkDk9iOnoeSz/z5VWv/AwskRm5Qo6YdBnxt7SdQEcL98iN0RCUjhr/FmBpke28iIjJEQtlWEoAG7jfIg+',
'ctl00$ctl00$MainPlaceHolder$Content4$searchoption': 'Agents or Broker',
'ctl00$ctl00$MainPlaceHolder$Content4$bkmbno': '',
'ctl00$ctl00$MainPlaceHolder$Content4$bkmbname': name,
'ctl00$ctl00$MainPlaceHolder$Content4$agbkcity': '',
'ctl00$ctl00$MainPlaceHolder$Content4$srButton': 'Search',
'ctl00$ctl00$hLocal': 'en',
'ctl00$ctl00$hIsWide': '0'
})
return r.text

def parse_result_page_ontario(page):
soup = BeautifulSoup(page, 'html.parser')
allA = soup.find_all('a', href=True)
licenses = []
for a in allA:
if('ShowLicence.aspx' in a['href']):
    licenses.append(a.text)
return licenses

def parse_license_page_ontario(license):
r = requests.get("http://mbsweblist.fsco.gov.on.ca/ShowLicence.aspx?" + 
license + "~")
soup = BeautifulSoup(r.text, 'html.parser')
return soup.find("span", {"id": 
"MainPlaceHolder_Content4_cragexpiry"}).text

name = 'crossley'
page = get_result_page_ontario(name)
licenses = parse_result_page_ontario(page)
for l in licenses:
print(l)
print(parse_license_page_ontario(l))

Answer 1

您可以使用selenium遍历座席姓名列表，输入每个姓名，然后恢复所需的信息：

from bs4 import BeautifulSoup as soup
import re, time, requests
from selenium import webdriver
names = ['crossley']
def get_name_data(_html:str):
   _d = soup(_html, 'html.parser').find('table', {'id':'MainPlaceHolder_Content4_resultsGrid'})
   if not _d:
     yield []
   else:
     yield from [(lambda x:[x[0].a['href'], *[c.text for c in x]])(i.find_all('td')) for i in _d.find_all('tr')[1:]]

_final_dict = {}
d = webdriver.Chrome('path/to/chromedriver')
for name in names:
  d.get('http://mbsweblist.fsco.gov.on.ca/agents.aspx')
  inputElement = d.find_element_by_id("MainPlaceHolder_Content4_bkmbname")
  inputElement.send_keys(name)
  _submit = d.find_element_by_id('MainPlaceHolder_Content4_srButton')
  _submit.click()
  time.sleep(1)
  for _href, num, name, _ in get_name_data(d.page_source):
    _new_result = soup(requests.get(f'http://mbsweblist.fsco.gov.on.ca/{_href}').text, 'html.parser')
    results = [[c.text for c in i.find_all('td')] for i in _new_result.find('table', {'id':'MainPlaceHolder_Content4_Table2'}).find_all('tr')]
  _final_dict[name] = results

parsed_data = {re.sub('\s+$', '', a):dict(map(lambda x:[x[0][:-1], x[-1]], b)) for a, [_, *b, _] in _final_dict.items()}

输出：

{'CROSSLEY, KEVIN': {'Agent/Broker Name': 'CROSSLEY, KEVIN', 'Licence #': 'M17001352', 'Brokerage Name': 'Mortgage Sense Inc.                                                                                 ', 'Licence Class': 'Agent', 'Status': 'Authorized to Sell', 'Issue Date': 'April 1, 2018', 'Expiry Date': 'March 31, 2020'}}

因此，对于CROSSLEY, KEVIN：

print([parsed_data['CROSSLEY, KEVIN']['Licence #'], parsed_data['CROSSLEY, KEVIN']['Expiry Date']])

输出：

['M17001352', 'March 31, 2020']

Answer 2

sample

这是我尝试使用姓氏作为搜索键提取的信息。

python-网站，填写表格，搜索和提取数据

2 个答案: