Question

我有一个保存在csv文件中的搜索条件列表。我想循环浏览每个搜索条件，以在网站上生成相应的搜索结果。对于生成的每组搜索结果（链接），我想点击链接，然后从生成的新页面中获取数据。不幸的是，我遇到了进入每个链接的问题。如果有人愿意提供一些见解，那么非常感激。

import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup

# read list of CAS Numbers to be searched
data = pd.read_csv("NPRI CACs.csv", names=["CAS Number", "Chemical Name"])
data.dropna()
CAS = data["CAS Number"]

# Parameters to be called
url = 'http://www.lifelabs.msdss.com/Login.aspx?ReturnUrl=%2fMainMenu.aspx%3ffm%3d0%26tb%3d0'

# Sign into SafeTec
browser = webdriver.Firefox()
browser.get(url)
browser.find_element_by_class_name("text").click()

# Conduct MSDS Searches on SafeTec
for i in range(10):
    try:
        Ingredient_CAS_Number = browser.find_element_by_id("placeBody_dynField48_txtTextBox")
        Ingredient_CAS_Number.send_keys(CAS[i])
        browser.find_element_by_id("placeBody_linkSearchBottom").click()

        list_links = browser.find_elements_by_css_selector("a[href*='MSDSDetail']")
        links = []
        for j in range(len(list_links)):
            links.append(list_links[j].get_attribute('href'))

        Product_Name = []
        for link in links:
            browser.get(link)
            product = browser.find_element_by_id("placeBody_dynField1_txtTextBox")
            Product_Name.append(product)
        print(Product_Name)

        browser.get(url)
    except:
        print(CAS[i])
        continue

Answer 1

我设法用下面的代码解决了这个问题。虽然，解决方案有点不合适......

import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup

# read list of CAS Numbers to be searched
data = pd.read_csv("NPRI CACs.csv", names=["CAS Number", "Chemical Name"])
data.dropna()
CAS = data["CAS Number"]

# Parameters to be called
url = 'http://www.lifelabs.msdss.com/Login.aspx?ReturnUrl=%2fMainMenu.aspx%3ffm%3d0%26tb%3d0'

# Sign into SafeTec
browser = webdriver.Firefox()
browser.get(url)
browser.find_element_by_class_name("text").click()

# Conduct MSDS Searches on SafeTec
for i in range(2):

        Ingredient_CAS_Number = browser.find_element_by_id("placeBody_dynField48_txtTextBox")
        Ingredient_CAS_Number.send_keys(CAS[i])
        browser.find_element_by_id("placeBody_linkSearchBottom").click()

        list_links = browser.find_elements_by_css_selector("a[href*='MSDSDetail']")
        all_results = []
        for j in list_links:
            result = j.text
            all_results.append(result)

        for i in range(len(all_results)):
            browser.find_element_by_link_text(all_results[i]).click()
            browser.back()


        browser.get(url)

Python Web Scraping：在Ajax站点上逐一单击链接

1 个答案: