我有兴趣从“自然语言处理”项目的医疗文档网页中获取文本。我要抓取的网页文档文本没有设计任何语义标记，只是带有粗体标题的一大块文本。获得帮助后，从第一页开始，我感兴趣的是，我实现了以下代码以从网页中获取文档文本：
import requests
import re
from bs4 import BeautifulSoup, Tag, NavigableString, Comment
url = 'https://www.mtsamples.com/site/pages/sample.asp?Type=24- Gastroenterology&Sample=2332-Abdominal%20Abscess%20I&D'
res = requests.get(url)
res.raise_for_status()
html = res.text
soup = BeautifulSoup(html, 'html.parser')
title_el = soup.find('h1')
page_title = title_el.text.strip()
first_hr = title_el.find_next_sibling('hr')
description_title = title_el.find_next_sibling('b',
text=re.compile('description', flags=re.I))
description_text_parts = []
for s in description_title.next_siblings:
if s is first_hr:
break
if isinstance(s, Tag):
description_text_parts.append(s.text.strip())
elif isinstance(s, NavigableString):
description_text_parts.append(str(s).strip())
description_text = '\n'.join(p for p in description_text_parts if p.strip())
# titles are all bold and uppercase
titles = [b for b in first_hr.find_next_siblings('b') if b.text.strip().isupper()]
docs = []
for t in titles:
text_parts = []
for s in t.next_siblings:
# go until next title
if s in titles:
break
if isinstance(s, Comment):
continue
if isinstance(s, Tag):
if s.name == 'div':
break
text_parts.append(s.text.strip())
elif isinstance(s, NavigableString):
text_parts.append(str(s).strip())
text = '\n'.join(p for p in text_parts if p.strip())
docs.append({
'title': t.text.strip(),
'text': text
})
这会将我的文档文本作为字典添加，并用title字段键和text值分隔到名为docs的列表中。此时，以上示例中抓取的网页将是docs列表中的唯一元素。
我有兴趣创建一个循环，以从位于https://www.mtsamples.com/site/pages/browse.asp?type=24-Gastroenterology&page=1的网页上的胃肠病学部分添加所有医疗文件记录。有23个单独的页面，每个页面按字母顺序包含许多不同的医学文档，总共包含230个医学文档。我想知道执行此循环的最佳方法是什么？同样，我的目标是将每个医疗文档附加到docs列表中，如我先前代码中的第一个示例所示。任何帮助将不胜感激！
答案 0 :(得分：0)
对于可靠的解决方案，只要存在“下一个”（while）按钮，就可以使用>循环来连续地迭代链接。在每次迭代时，您都可以抓取下一页并提取带有链接和说明的标题，以供以后使用：
import requests, re
from bs4 import BeautifulSoup as soup
def parse_page(_d):
data = filter(lambda x:x != '\n', [i for i in _d.find('table', {'id':'Browse'}).td.contents if isinstance(i, str) or i.name == 'a'])
_next = next(data, None)
while _next is not None:
new_d, _n = {'title':_next.text, 'link':_next['href']}, next(data, None)
if hasattr(_n, 'text'):
yield new_d
yield {'title':_n.text, 'link':_n['href'], 'about':next(data)}
else:
yield {**new_d, 'about':_n}
_next = next(data, None)
d, r = soup(requests.get('https://www.mtsamples.com/site/pages/browse.asp?type=24-Gastroenterology&page=1').text, 'html.parser'), []
r.append(list(parse_page(d)))
_c = [i for i in d.find('div', {'class':'Contrast'}).find_all('a') if i.text == '>']
while _c:
d = soup(requests.get(f'https://www.mtsamples.com{_c[0]["href"]}').text, 'html.parser')
r.append(list(parse_page(d)))
_c = [i for i in d.find('div', {'class':'Contrast'}).find_all('a') if i.text == '>']
输出（由于SO的字符限制，r的第一个元素）：
[{'title': 'Abdominal Abscess I&D;', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2332-Abdominal Abscess I&D', 'about': 'Incision and drainage (I&D;) of abdominal abscess, excisional debridement of nonviable and viable skin, subcutaneous tissue and muscle, then removal of foreign body.'}, {'title': 'Abdominal Exploration', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2038-Abdominal Exploration', 'about': 'Congenital chylous ascites and chylothorax and rule out infradiaphragmatic lymphatic leak. Diffuse intestinal and mesenteric lymphangiectasia. '}, {'title': 'Abdominal Pain - Consult', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=1921-Abdominal Pain - Consult', 'about': 'The patient presented to the emergency room last evening with approximately 7- to 8-day history of abdominal pain which has been persistent.'}, {'title': 'Abscess Excision', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2330-Abscess Excision', 'about': 'Excision of abscess, removal of foreign body. Repair of incisional hernia. Recurrent re-infected sebaceous cyst of abdomen. Abscess secondary to retained foreign body and incisional hernia.'}, {'title': 'Admission History & Physical - Nausea', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=1152-Admission History & Physical - Nausea', 'about': 'Patient status post gastric bypass surgery, developed nausea and right upper quadrant pain.'}, {'title': 'Adrenalectomy & Umbilical Hernia Repair', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2464-Adrenalectomy & Umbilical Hernia Repair', 'about': 'Laparoscopic hand-assisted left adrenalectomy and umbilical hernia repair. Patient with a 5.5-cm diameter nonfunctioning mass in his right adrenal.'}, {'title': 'Air Under Diaphragm - Consult', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2166-Air Under Diaphragm - Consult', 'about': 'Possible free air under the diaphragm. On a chest x-ray for what appeared to be shortness of breath she was found to have what was thought to be free air under the right diaphragm. No intra-abdominal pathology.'}, {'title': 'Appendectomy', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=1975-Appendectomy', 'about': 'Appendicitis, nonperforated. Appendectomy. A transverse right lower quadrant incision was made directly over the point of maximal tenderness. '}, {'title': 'Appendectomy - 1', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2298-Appendectomy - 1', 'about': 'Acute appendicitis, gangrenous. Appendectomy.'}, {'title': 'Appendectomy - Laparoscopic ', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2728-Appendectomy - Laparoscopic ', 'about': 'Laparoscopic appendectomy and peritoneal toilet and photos. Pelvic inflammatory disease and periappendicitis.'}]
答案 1 :(得分：0)
只需找到所有分页URL，然后遍历所有页面，找到文档URL并提取文档。这是为您提供的全面解决方案。
这将同时浏览页面并批量提取所有页面中的文档
import requests
from bs4 import BeautifulSoup, Tag, Comment, NavigableString
from urllib.parse import urljoin
from pprint import pprint
import itertools
import concurrent
from concurrent.futures import ThreadPoolExecutor
BASE_URL = 'https://www.mtsamples.com'
def make_soup(url: str) -> BeautifulSoup:
res = requests.get(url)
res.raise_for_status()
html = res.text
soup = BeautifulSoup(html, 'html.parser')
return soup
def make_soup_parallel(urls: list) -> list:
workers = min(10, len(urls))
with ThreadPoolExecutor(max_workers=workers) as e:
return list(e.map(make_soup, urls))
def find_pagination_urls(soup: BeautifulSoup) -> list:
urls = set()
for a in soup.select('.Contrast a'):
if not a.text.isnumeric():
continue
url = urljoin(BASE_URL, a['href'])
urls.add(url)
return sorted(list(urls), key=lambda u: int(u.split('page=')[1]))
def find_document_urls(soup: BeautifulSoup) -> list:
urls = []
for a in soup.select('#Browse a'):
url = urljoin(BASE_URL, a['href'])
urls.append(url)
return urls
def find_all_doc_urls() -> list:
index_url = 'https://www.mtsamples.com/site/pages/browse.asp?type=24-Gastroenterology&page=1'
index_soup = make_soup(index_url)
next_pages = find_pagination_urls(index_soup)
doc_urls = []
for soup in make_soup_parallel(next_pages):
doc_urls.extend(find_document_urls(index_soup))
return doc_urls
def extract_docs(soup: BeautifulSoup) -> list:
title_el = soup.find('h1')
first_hr = title_el.find_next_sibling('hr')
# titles are all bold and uppercase
titles = [b for b in first_hr.find_next_siblings('b') if b.text.strip().isupper()]
docs = []
for t in titles:
text_parts = []
for s in t.next_siblings:
# go until next title
if s in titles:
break
if isinstance(s, Comment):
continue
if isinstance(s, Tag):
if s.name == 'div':
break
text_parts.append(s.text.strip())
elif isinstance(s, NavigableString):
text_parts.append(str(s).strip())
text = '\n'.join(p for p in text_parts if p.strip())
docs.append({
'title': t.text.strip(),
'text': text
})
return docs
def batch(it, n: int):
it = [iter(it)] * n
return itertools.zip_longest(*it, fillvalue=None)
docs = []
doc_urls = find_all_doc_urls()
for b in batch(doc_urls, 5):
batch_urls = list(filter(bool, b))
for soup in make_soup_parallel(batch_urls):
docs.extend(extract_docs(soup))
pprint(docs)
输出：
[{'text': 'Abdominal wall abscess.', 'title': 'PREOPERATIVE DIAGNOSIS:'},
{'text': 'Abdominal wall abscess.', 'title': 'POSTOPERATIVE DIAGNOSIS:'},
{'text': 'Incision and drainage (I&D) of abdominal abscess, excisional '
'debridement of nonviable and viable skin, subcutaneous tissue and '
'muscle, then removal of foreign body.',
'title': 'PROCEDURE:'},
{'text': 'LMA.', 'title': 'ANESTHESIA:'},
...