Question

from selenium import webdriver
from bs4 import BeautifulSoup as bs
import time

url = 'https://curecity.in/vendor-list.php?category=Doctor&filters_location=Jaipur&filters%5Bsubareas_global%5D=&filters_speciality='

driver = webdriver.Chrome('C:\chromedriver.exe')
driver.get(url)
driver.maximize_window()


next_page_number=1
next_page = True
while next_page == True:
     soup = bs(driver.page_source, 'html.parser')
     for link in soup.find_all('div',class_='col-md-9 feature-info'):
        link1 = link.find('a')
        print(link1['href'])
     try:
        driver.find_element_by_link_text(">").click()
        next_page_number+=1
        time.sleep(1)
    except:
        print ('No more pages')
        next_page=False

driver.close()

我正在尝试单击下一页，但是上面编写的代码正在单击备用页面。我想从第一页一直单击到最后一页。

Answer 1

我在此页面上发现了两个问题。

1）它加载页面的速度非常慢，因此甚至在获取数据和click()按钮之前，我不得不睡10秒钟。

2）按钮>的工作原理与我预期的不同-它跳了3页（即使我在浏览器中手动单击它），因此我搜索了具有下一页编号的按钮，然后单击它。

 driver.find_element_by_xpath('//a[@data-page="{}"]'.format(next_page_number)).click()

完整代码。即使没有BeautifulSoup

也可以使用

from selenium import webdriver
#from bs4 import BeautifulSoup as bs
import time

url = 'https://curecity.in/vendor-list.php?category=Doctor&filters_location=Jaipur&filters%5Bsubareas_global%5D=&filters_speciality='

driver = webdriver.Chrome('C:\chromedriver.exe')
#driver = webdriver.Firefox()
driver.maximize_window()

driver.get(url)
next_page_number = 1

while True:

    print('page:', next_page_number)
    time.sleep(10)

    #soup = bs(driver.page_source, 'html.parser')
    #for link in soup.find_all('div',class_='col-md-9 feature-info'):
    #    link1 = link.find('a')
    #    print(link1['href'])

    for link in driver.find_elements_by_xpath('//div[@class="col-md-2 feature-icon"]/a'):
        print(link.get_attribute('href'))

    try:
        next_page_number += 1
        driver.find_element_by_xpath('//a[@data-page="{}"]'.format(next_page_number)).click()
    except:
        print('No more pages')
        break # exit loop

#driver.close()

硒webdriver：我要单击下一页直到最后一页

1 个答案: