我正试图废弃此网站
www.united-church.ca/search/locator/all?keyw=&mission_units_ucc_ministry_type_advanced=10&locll=
我确实抓了它,但是我无法抓取电子邮件地址 你能帮我把它报废吗? 我在用草木
# -*- coding: utf-8 -*-
import scrapy
from ..items import ChurchItem
class ChurchSpiderSpider(scrapy.Spider):
name = 'church_spider'
page_number = 1
start_urls = ['https://www.united-church.ca/search/locator/all?keyw=&mission_units_ucc_ministry_type_advanced=10&locll=']
def parse(self, response):
items = ChurchItem()
container = response.css(".icon-ministry")
for t in container:
church_name = t.css(".field-name-locator-ministry-title a::text").extract()
church_phone = t.css(".field-name-field-phone::text").extract()
church_address = t.css(".thoroughfare::text").extract()
church_email = t.css('a.spamspan::text').extract()
items["church_name"] = church_name
items["church_phone"] = church_phone
items["church_address"] = church_address
items["church_email"] = church_email
yield items
# next_page = 'https://www.united-church.ca/search/locator/all?keyw=&mission_units_ucc_ministry_type_advanced=10&locll=&page=' + str(ChurchSpiderSpider.page_number)
# if ChurchSpiderSpider.page_number <= 110:
# ChurchSpiderSpider.page_number += 1
# yield response.follow(next_page, callback=self.parse)
有时church_email变成这样[at]
那是问的那个人的完整代码
答案 0 :(得分:1)
使用美丽的汤
获取电子邮件的一种简单方法是使用x2
查找div,然后将奇数显示替换为适当的电子邮件格式。
例如:
class=field-name-field-mu-email'
答案 1 :(得分:0)
您可以尝试使用Selenium进行网络抓取,我尝试了这段代码,并给出了理想的结果。
from selenium import webdriver
from bs4 import BeautifulSoup
driver = webdriver.Chrome("chromedriver")
driver.get("https://www.united-church.ca/search/locator/all?keyw=&mission_units_ucc_ministry_type_advanced=10&locll=")
content = driver.page_source
soup = BeautifulSoup(content)
for all_emails in soup.find_all('a',class_="spamspan"):
print(all_emails.text)
结果:
alpcharge@sasktel.net
guc-eug@bellnet.ca
pioneerpastoralcharge@gmail.com
acmeunitedchurch@gmail.com
cmcphers@lakeheadu.ca
mbm@kos.net
tommaclaren@gmail.com
agassizunited@shaw.ca
buchurch@xplornet.com
dmitchell008@yahoo.ca
karen.charlie62@gmail.com
trinityucbdn@westman.wave.ca
gepc.ucc.mail@gmail.com
monacampbell181@gmail.com
herbklaehn@gmail.com