Question

我想在ActiveMQ使用者中运行我的抓蜘蛛（在onMessage中）

这是我的爬虫蜘蛛代码

import scrapy

导入json

selogerSpider类（scrapy.Spider）：

name = "realtor"


custom_settings = {
    'DOWNLOADER_MIDDLEWARES': {
        'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
        'scrapy_fake_useragent.middleware.RandomUserAgentMiddleware': 400,
    }
}

def start_requests(self):
    with open("annonces.txt", "r") as file:
        for line in file.readlines():
            yield scrapy.Request(line)

def parse(self, response):
  try:
    name = response.css(".agence-link::text").extract_first()
    address = response.css(".agence-adresse::text").extract_first()

    XPATH_siren = ".//div[@class='legalNoticeAgency']//p/text()"
    siren = response.xpath(XPATH_siren).extract_first()

    XPATH_website = ".//div[@class='agence-links']//a/@href"
    site = response.xpath(XPATH_website).extract()

    XPATH_phone = ".//div[@class='contact g-row-50']//div[@class='g-col g-50 u-pad-0']//button[@class='btn-phone b-btn b-second fi fi-phone tagClick']/@data-phone"
    phone = response.xpath(XPATH_phone).extract_first()

    agencyObj = WebRealTor(name, address, siren, site, phone)
    realtorJson = json.dumps(agencyObj.__dict__)
    with open('data.json', 'a') as outfile:
        json.dump(realtorJson, outfile)
  except KeyError :
    print(" KeyError")
class WebRealTor(object):
def __init__(self, nom,adresse,siren,site,phone):
    self.Nom = nom
    self.Adresse = adresse
    self.Site_web = site
    self.Numero_Siren= siren
    self.Numero_telephone = phone
def jsonDefault(object):
    return object.__dict__

我的监听器activemq代码导入时间导入系统导入脚印

class MyListener(stomp.ConnectionListener):
def on_error(self, headers, message):
    print('received an error "%s"' % message)

def on_message(self, headers, message):
    print('received a message "%s"' % message)
try:
conn = stomp.Connection()
conn.set_listener('', MyListener())
conn.start()
conn.connect('admin', 'password', wait=True)

conn.subscribe(destination='/queue/test', id=1, ack='auto')

conn.send(body=' '.join(sys.argv[1:]), destination='/queue/test')

time.sleep(2)
conn.disconnect()

IOError除外，例如e：打印 “ I / O错误（{0}）：{1}”。format（e.errno，e.strerror）

从Python脚本调用Scrapy Spider

0 个答案: