Question

我想进行爬网，但是我遇到了一些麻烦，我需要打开商品的每个链接并获取其信息，然后将其保存在.html页面中的每个商品中。现在我只能在页面上打印所有链接

from bs4 import BeautifulSoup as soup
from urllib.request import urlopen as uReq
import requests
import urllib3
import ssl
from requests import request

urllib3.disable_warnings()

try:
    _create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
        pass
else:
    ssl._create_default_https_context = _create_unverified_https_context

PYTHONHTTPSVERIFY=0
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) 
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
headers = {'User-Agent': user_agent}
t = request('GET', url=my_url, headers=headers, verify=False).text
uClient = uReq(my_url)
page_html = uClient.read()
uClient.close()
page_soup = soup(page_html, "html.parser")

containers = page_soup.findAll("div", {"class": 'product'})

filename = "web.html"
f= open(filename, "w")

for containers in page_soup.findAll('div', attrs={'class': 'product'}):
    f.write(containers.a['href'] + '\n')
f.close()

Answer 1

您似乎正在尝试从第一个URL获取URL列表，然后希望从每个URL中获取一些信息。为此，需要对每个URL进行请求，并且每个URL都需要单独的BeautifulSoup解析。

拥有子页面后，即可提取信息，例如产品名称及其价格。

最后，您可以打印此信息或将其写入文件。最简单的方法是将其编写为CSV文件。在此示例中，我展示了如何将URL，名称和价格写成一行。 CSV库会自动正确设置格式：

from urllib3.exceptions import InsecureRequestWarning
from bs4 import BeautifulSoup
import requests
import csv

requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
my_url = 'https://franke-market.com.ua/moyki.html?on_page=100'
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'}
req = requests.get(my_url, headers=headers, verify=False)
soup = BeautifulSoup(req.content, "html.parser")

with open("products.csv", "w", newline="") as f_output:
    csv_output = csv.writer(f_output)
    csv_output.writerow(['URL', 'Title', 'Price'])

    # Find the URLs for all the products
    for div in soup.find_all('div', attrs={'class': 'product'})[:5]:
        url = div.a['href']
        print(url)

        # For each URL, get the sub page and get the name and price
        req_sub = requests.get(url, headers=headers, verify=False)
        soup_sub = BeautifulSoup(req_sub.content, "html.parser")        
        title = soup_sub.find('h1', class_='title').text
        price_info = soup_sub.find('div', class_='price_info').span.text

        # Write the url, name and price as a CSV file
        csv_output.writerow([url, title, price_info])

为您和output.csv个文件开始：

URL,Title,Price
https://franke-market.com.ua/franke_rol_610-38_101_0267_707_.html,Franke ROL 610-38 (101.0267.707),91795
https://franke-market.com.ua/franke-pmn-611i-101.0255.790.html,Franke Pamira PMN 611i (101.0255.790),57935
https://franke-market.com.ua/franke_pxl_611-60_101_0330_655_.html,Franke PXL 611-60 (101.0330.655),93222
https://franke-market.com.ua/franke-ron-610-41-101.0255.783.html,Franke Ron 610-41 (101.0255.783),57939
https://franke-market.com.ua/franke_pxl_611-78_101_0330_657_.html,Franke PXL 611-78 (101.0330.657),93223

然后可以将此文件打开到电子表格应用程序中。

无法检索网站

1 个答案: