我在Amazon EC2上设置了Scrapyd服务器。我已经将scrapy项目成功地部署到服务器上,但是一旦我计划了一次蜘蛛运行,它就会立即运行并完成工作,而不会刮任何项目。
我还设置了另一台服务器,并尝试了一下,但是没有运气。
卷曲http://my.ec2/schedule.json -d project = default -d spider = somespider
yum install -y gcc
yum install -y openssl-devel
yum install python3
yum install -y python3-devel.x86_64
pip3 install python-dateutil
pip3 install Scrapy==1.5.1
pip3 install scrapyd
pip3 install scrapyd_client
pip3 install dateparser
pip3 install pyyaml
pip3 install botocore
export PATH=$PATH:/usr/local/bin
yum install -y git
cd /home/ec2-user
echo "[scrapyd]
eggs_dir = eggs
logs_dir =
items_dir =
jobs_to_keep = 5
dbs_dir = dbs
max_proc = 0
max_proc_per_cpu = 4
finished_to_keep = 250
poll_interval = 5.0
bind_address = 0.0.0.0
http_port = 6800
debug = off
runner = scrapyd.runner
application = scrapyd.app.application
launcher = scrapyd.launcher.Launcher
webroot = scrapyd.website.Root
[services]
schedule.json = scrapyd.webservice.Schedule
cancel.json = scrapyd.webservice.Cancel
addversion.json = scrapyd.webservice.AddVersion
listprojects.json = scrapyd.webservice.ListProjects
listversions.json = scrapyd.webservice.ListVersions
listspiders.json = scrapyd.webservice.ListSpiders
delproject.json = scrapyd.webservice.DeleteProject
delversion.json = scrapyd.webservice.DeleteVersion
listjobs.json = scrapyd.webservice.ListJobs
daemonstatus.json = scrapyd.webservice.DaemonStatus" > scrapyd.conf
scrapyd```
{"node_name": "my.ec2", "status": "ok", "pending": [], "running": [], "finished": [{"id": "abcd", "spider": "rishtml",
"start_time": "2019-09-14 19:33:42.667420", "end_time": "2019-09-14 19:33:43.563293"}]}