我在app.scrapinghub.com上收到以下错误,但在我的本地计算机上工作正常。
注意:我在python(Scrapy框架)中使用请求模块发送请求并使用BeautifulSoup来解析响应
from scrapy.spider import Spider
import requests,json
from bs4 import BeautifulSoup
from datetime import datetime
from datetime import timedelta
from Scrapy_Project.pipelines import MySQLPipeline
class exampleSpider(Spider):
name='test'
start_urls=['http://www.example.com']
custom_settings = {
'ITEM_PIPELINES': {
'Scrapy_Project.pipelines.MySQLPipeline': None
}
}
def parse(self, response):
current_date=datetime.today()
today=current_date.strftime('%m/%d/%Y')
tommrrow_date= current_date+timedelta(days=1)
tommrrow=tommrrow_date.strftime('%m/%d/%Y')
date_list=[today,tommrrow]
data_lists=[]
for date in date_list:
m_show_time=[]
url='http://www.example.com?id=123'
page = requests.get(url) ///
soup = BeautifulSoup(page.content, 'html.parser')
movie_list=soup.find_all('exampe_info')
for index, item in enumerate(data_lists):
name=item.find('title').get_text()
x_time=item.find('starttime').get_text()
result= {'name': name ,'show_date':date}
yield result
示例代码:
ICollector