Question

我正在尝试在Twisted中创建一个简单的分布式作业客户端/服务器系统。基本上步骤是：

使用一些作业和相关文件启动JobServer
启动JobClient实例，它们连接到JobServer并请求作业
服务器提供JobClient作业并通过TCP发送序列化JSON
经过大量计算后，JobClient会发回一个结果并等待新工作
冲洗并重复

但是我在本地计算机上调试协议时遇到问题。

JobServer.py

from twisted.application import internet, service
from twisted.internet import reactor, protocol, defer
from twisted.protocols import basic
from twisted.protocols.basic import Int32StringReceiver
from twisted.web import client
import random
import json
import base64
from logger import JobLogger

class JobServerProtocol(Int32StringReceiver):

    log = JobLogger("server.log")

    def connectionMade(self):
        self.log.write("Connected to client")
        self.sendJob(None)

    def stringReceived(self, msg):
        self.log.write("Recieved job from client: %s" % msg)
        self.sendJob(msg)

    def sendJob(self, msg):
        d = self.factory.getJob(msg)

        def onError(err):
            self.transport.write("Internal server error")
        d.addErrback(onError)

        def sendString(newjob_dict):
            encoded_str = json.dumps(newjob_dict)
            self.transport.write(encoded_str)
            self.log.write("Sending job to client: %s" % encoded_str)
        d.addCallback(sendString)

    def lengthLimitExceeded(self, msg):
        self.transport.loseConnection()

class JobServerFactory(protocol.ServerFactory):
    protocol = JobServerProtocol

    def __init__(self, jobs, files):
        assert len(jobs) == len(files)
        self.jobs = jobs
        self.files = files
        self.results = []

    def getJob(self, msg):

        # on startup the client will not have a message to send
        if msg:
            # recreate pickled msg
            msg_dict = json.loads(msg)
            self.results.append((msg_dict['result'], msg_dict['jidx']))

        # if we're all done, let the client know
        if len(self.jobs) == 0:
            job = None
            jidx = -1
            encoded = ""
        else:
            # get new job for client to process
            jidx = random.randint(0, len(self.jobs) - 1)
            job = self.jobs[jidx]
            del self.jobs[jidx]

            # get file
            with open(self.files[jidx], 'r') as f:
                filecontents = f.read()
            encoded = base64.b64encode(filecontents)

        # create dict object to send to client
        response_msg = {
            "job" : job,
            "index" : jidx,
            "file" : encoded
        }

        return defer.succeed(response_msg)

# args for factory
files = ['test.txt', 'test.txt', 'test.txt']
jobs = ["4*4-5", "2**2-5", "2/9*2/3"]

application = service.Application('jobservice')
factory = JobServerFactory(jobs=jobs, files=files)
internet.TCPServer(12345, factory).setServiceParent(
    service.IServiceCollection(application))

JobClient.py

from twisted.internet import reactor, protocol
from twisted.protocols.basic import Int32StringReceiver
import json
import time
from logger import JobLogger

class JobClientProtocol(Int32StringReceiver):

    log = JobLogger("client.log")

    def stringReceived(self, msg):

        # unpack job from server
        server_msg_dict = json.loads(msg)
        job = server_msg_dict["job"]
        index = server_msg_dict["index"]
        filestring = server_msg_dict["file"]

        if index == -1:
            # we're done with all tasks
            self.transport.loseConnection()

        self.log.write("Recieved job %d from server with file '%s'" % (index, filestring))

        # do something with file 
        # job from the server...
        time.sleep(5)
        result = { "a" : 1, "b" : 2, "c" : 3}
        result_msg = { "result" : result, "jidx" : index }

        self.log.write("Completed job %d from server with result '%s'" % (index, result))

        # serialize and tell server
        result_str = json.dumps(result_msg)
        self.transport.write(encoded_str)

    def lengthLimitExceeded(self, msg):
        self.transport.loseConnection()

class JobClientFactory(protocol.ClientFactory):

    def buildProtocol(self, addr):
        p = JobClientProtocol()
        p.factory = self
        return p

reactor.connectTCP("127.0.0.1", 12345, JobClientFactory())
reactor.run()

logging.py

class JobLogger(object):
    def __init__(self, filename):
        self.log = open(filename, 'a')

    def write(self, string):
        self.log.write("%s\n" % string)

    def close(self):
        self.log.close()

运行，仅在一个客户端进行本地测试：

$ twistd -y JobServer.py -l ./jobserver.log --pidfile=./jobserver.pid
$ python JobClient.py

我遇到的问题：

客户端和服务器.log文件无法可靠地写入 - 有时直到我终止该进程后才会写入。
在客户端连接并且服务器发回消息后协议卡住。这条消息似乎永远不会传达给客户。

一般情况下，我希望这些协议确保任何一方的操作都可以花费任何时间，但也许我没有正确设计。

Answer 1

客户端和服务器.log文件无法可靠地写入 - 有时直到我终止该进程后才会写入。

如果您希望及时在磁盘上显示字节，则可能需要在文件对象上调用flush。

在客户端连接并且服务器发回消息后协议卡住。这条消息似乎永远不会传达给客户。

服务器不向客户端发送int32字符串：它直接调用transport.write。客户端感到困惑，因为这些看起来像是非常长的int32字符串。例如，“内部服务器错误”的前四个字节解码为整数1702129225，因此如果服务器上出现错误并且这些字节被发送到客户端，则客户端将等待大约2GB的数据，然后继续。

改为使用Int32StringReceiver.sendString。

扭曲的客户/服务器沟通困境

1 个答案: