Question

我已经读过how to add a coroutine to a running asyncio loop?，但不是我想要的

基本上，我需要一个守护线程来订阅redis通道，并且可以添加动态回调方法，我的解决方案是将Thread类子类化，并创建一个事件循环并永远运行，但是在循环运行后，我无法调用对象的任何方法

redis.py

#! /usr/bin/env python
# -*- coding: utf-8 -*-
import redis
import os
import asyncio
import aioredis
from threading import Thread
from collections import defaultdict

assert os.getenv('REDIS_HOST') is not None
assert os.getenv('REDIS_PORT') is not None

class RedisClient(Thread):
    def __init__(self, loop):
        super(RedisClient, self).__init__()
        self.callbacks = defaultdict(list)
        self.channels = {}
        self.loop = loop

    async def pubsub(self):
        address = 'redis://{}:{}'.format(os.getenv('REDIS_HOST'), os.getenv('REDIS_PORT'))
        self.sub = await aioredis.create_redis(address)

    def sync_add_callback(self, channel, callback):
        self.loop.create_task(self.add_callback(channel, callback))

    async def add_callback(self, channel, callback):
        self.callbacks[channel].append(callback)

        if channel not in self.channels or self.channels[channel] is None:
            channels = await self.sub.subscribe(channel)
            ch1 = channels[0]
            assert isinstance(ch1, aioredis.Channel)
            self.channels[channel] = ch1

            async def async_reader(channel):
                while await channel.wait_message():
                    msg = await channel.get(encoding='utf-8')
                    # ... process message ...
                    print(msg)
                    print(channel.name)
                    for c in self.callbacks[channel.name.decode('utf-8')]:
                        c(channel.name, msg)

            tsk1 = asyncio.ensure_future(async_reader(ch1))

    def remove_callback(self, channel, callback):
        self.callbacks[channel].remove(callback)

    def run(self):
        asyncio.set_event_loop(self.loop)
        loop.run_until_complete(self.pubsub())


# Create the new loop and worker thread
loop = asyncio.new_event_loop()
redis_client = RedisClient(loop)
redis_client.start()

用法：

def test(channel, msg):
    print('{}{}'.format(channel, msg))

from redis import redis_client
redis_client.sync_add_callback('test', test)

也许我的解决方案不是Python的好习惯？

更新1：

我已经尝试过一种解决方案，并且效果很好，但是一开始，我想重用sub实例，该方法可以作为模块来订阅不同的频道，但是每个订阅都应该拥有自己的频道sub，也就是说每个订阅者都必须创建自己的Redis连接

解决方案：

#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import asyncio
import aioredis
from threading import Thread

assert os.getenv('REDIS_HOST') is not None
assert os.getenv('REDIS_PORT') is not None

class RedisClient(Thread):
    def __init__(self, channel, callback, *args, **kwargs):
        super(RedisClient, self).__init__(*args, **kwargs)
        self.daemon = True
        self.channel = channel
        self.callback = callback

    async def pubsub(self):
        address = 'redis://{}:{}'.format(os.getenv('REDIS_HOST'), os.getenv('REDIS_PORT'))
        sub = await aioredis.create_redis(address)

        channels = await sub.subscribe(self.channel)
        ch1 = channels[0]
        assert isinstance(ch1, aioredis.Channel)

        async def async_reader(channel):
            while await channel.wait_message():
                msg = await channel.get(encoding='utf-8')
                self.callback(channel.name.decode('utf-8'), msg)

        await async_reader(ch1)

    def run(self):
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        loop.run_until_complete(self.pubsub())

更新2：

最终，它运行良好

#! /usr/bin/env python
# -*- coding: utf-8 -*-
import redis
import os
import asyncio
import aioredis
from threading import Thread
from collections import defaultdict

assert os.getenv('REDIS_HOST') is not None
assert os.getenv('REDIS_PORT') is not None

class RedisClient(Thread):
    def __init__(self, loop):
        super(RedisClient, self).__init__()
        self.callbacks = defaultdict(list)
        self.channels = {}
        self.loop = loop
        self.sub = None

    async def pubsub(self):
        print('test3')
        address = 'redis://{}:{}'.format(os.getenv('REDIS_HOST'), os.getenv('REDIS_PORT'))
        self.sub = await aioredis.create_redis(address)

    def sync_add_callback(self, channel, callback):
        print('ahhhhhhhhh')
        asyncio.run_coroutine_threadsafe(self.add_callback(channel, callback), self.loop)

    async def add_callback(self, channel, callback):
        print('test2')
        if not self.sub:
            await self.pubsub()
        self.callbacks[channel].append(callback)

        if channel not in self.channels or self.channels[channel] is None:
            channels = await self.sub.subscribe(channel)
            ch1 = channels[0]
            assert isinstance(ch1, aioredis.Channel)
            self.channels[channel] = ch1

            async def async_reader(channel):
                while await channel.wait_message():
                    msg = await channel.get(encoding='utf-8')
                    # ... process message ...
                    print(msg)
                    print(channel.name)
                    print(self.callbacks[channel.name])
                    for c in self.callbacks[channel.name.decode('utf-8')]:
                        c(channel.name, msg)

            tsk1 = asyncio.ensure_future(async_reader(ch1))

    def remove_callback(self, channel, callback):
        self.callbacks[channel].remove(callback)

    def run(self):
        asyncio.set_event_loop(self.loop)
        loop.run_forever()


# Create the new loop and worker thread
loop = asyncio.new_event_loop()
redis_client = RedisClient(loop)
redis_client.start()

Answer 1

让我在这里向您展示使用aiohttp的类似情况。

async def listen_to_redis(app):
    try:
        sub = await aioredis.create_redis(('localhost', 6379), loop=app.loop)
        ch, *_ = await sub.subscribe('news')
        async for msg in ch.iter(encoding='utf-8'):
            # Forward message to all connected websockets:
            for ws in app['websockets']:
                ws.send_str('{}: {}'.format(ch.name, msg))
    except asyncio.CancelledError:
        pass
    finally:
        await sub.unsubscribe(ch.name)
        await sub.quit()


async def start_background_tasks(app):
    app['redis_listener'] = app.loop.create_task(listen_to_redis(app))


async def cleanup_background_tasks(app):
    app['redis_listener'].cancel()
    await app['redis_listener']


app = web.Application()
app.on_startup.append(start_background_tasks)
app.on_cleanup.append(cleanup_background_tasks)
web.run_app(app)

Answer 2

如果该想法是要从其他线程中调用sync_add_callback，则其实现应如下所示：

def sync_add_callback(self, channel, callback):
    asyncio.run_coroutine_threadsafe(self.add_callback(channel, callback), self.loop)

请注意，回调将在事件循环线程中被调用，因此它们不应自己使用阻塞调用。

如何在运行的事件循环中添加协程？

2 个答案: