ZeroMQ的大多数示例和测试都在一个方向上使用通道。显然，一些ZeroMQ模式可以用于双向通信，但这会带来性能成本吗？
我创建了一个示例来说明我们计划如何使用ZeroMQ，完全异步，客户端和服务器可以随时从任何线程发送请求/回复。我试图尽可能简单，但请尽量忽略错误处理和关机过程。 请评论任何其他问题，使用您可能找到的代码 。
客户端可以运行方式，可以是同步的，在哪里序列化请求，也可以是异步的，它会创建一个单独的线程来尽快发送请求。请求很小，只是创建请求时的时间戳和一个简单的字符串。服务器接收请求，添加更多字符串数据并简单地返回答复。然后，客户端在回复中提取时间戳并计算往返时间。
现在，当以异步模式运行时，在收到几乎所有回复之前，不会完成发送线程。 为什么这样，我期待单独的发送线程更快完成？
RTT时间相当高。
在异步模式下，可以通过为每150个请求添加 sleep(1) 来改进RTT，而不会花费太多时间发送/接收所有消息。 这是否表明ZeroMQ没有有效处理队列？
在同步模式下，RTT虽然较低，但仍然非常高，只需处理一个请求，然后影响总吞吐量。
在Windows 8工作站上本地运行时的输出示例：
Send thread finished, 100000 requests sent in us=1047525
Received 100000 replies, last 10000 took us=95440, averate RTT=36228
Work thread finished, 100000 replies received in us=1077641
下面是客户端和服务器的代码。
TIA
客户代码：
// ZMQClient.cpp : Defines the entry point for the console application.
//
#include "stdafx.h"
#include <profileapi.h>
#include <atomic>
#include <handleapi.h>
#include <process.h>
#include "zmq.hpp"
#include <queue>
using namespace zmq;
using namespace std;
//#define SYNCHRONOUS
namespace
{
LARGE_INTEGER ui64StartCounter, ui64StopCounter, Frequency;
HANDLE g_zmqThread = INVALID_HANDLE_VALUE;
HANDLE g_workThread = INVALID_HANDLE_VALUE;
HANDLE g_sendThread = INVALID_HANDLE_VALUE;
SRWLOCK g_queueLock;
queue<message_t*> g_incomingQueue;
HANDLE g_queueUpdated = INVALID_HANDLE_VALUE;
bool g_runThreads = true;
context_t g_context(1);
const unsigned int g_numOfIterations = 100000;
const unsigned int g_statisticsInterval = 10000;
}
static DWORD WINAPI ZmqThreadStatic(PVOID pThreadParameter)
{
if (pThreadParameter == nullptr) return ERROR_INVALID_PARAMETER;
PCSTR connectionAddress = (PCSTR)pThreadParameter;
socket_t sendSocket(g_context, ZMQ_PULL);
sendSocket.bind("inproc://sendsocket");
printf("Connecting to: %s\n", connectionAddress);
zmq::socket_t clientSocket(g_context, ZMQ_DEALER);
clientSocket.connect(connectionAddress);
zmq_pollitem_t pollItems[] = {
{ sendSocket, 0, ZMQ_POLLIN, 0 },
{ clientSocket, 0, ZMQ_POLLIN, 0 }
};
message_t zMsgt;
int more;
size_t size = sizeof(more);
while (g_runThreads)
{
zmq_poll(pollItems, 2, -1);
if (pollItems[0].revents & ZMQ_POLLIN)
{
while (true)
{
sendSocket.recv(&zMsgt, 0);
sendSocket.getsockopt(ZMQ_RCVMORE, &more, &size);
clientSocket.send(zMsgt, more ? ZMQ_SNDMORE : 0);
if (more == 0) break;
}
}
if (pollItems[1].revents & ZMQ_POLLIN)
{
while (true)
{
clientSocket.recv(&zMsgt, 0);
message_t* newMsg = new message_t;
newMsg->move(&zMsgt);
AcquireSRWLockExclusive(&g_queueLock);
g_incomingQueue.push(newMsg);
ReleaseSRWLockExclusive(&g_queueLock);
SetEvent(g_queueUpdated);
clientSocket.getsockopt(ZMQ_RCVMORE, &more, &size);
if (more == 0) break;
}
}
}
clientSocket.close();
printf("ZMQ thread finished\n");
return NO_ERROR;
}
static DWORD WINAPI WorkThreadStatic(PVOID pThreadParameter)
{
socket_t sendSocket(g_context, ZMQ_PUSH);
sendSocket.connect("inproc://sendsocket");
LARGE_INTEGER ui64StartCounter, ui64StopCounter, ui64TotalCounter, ui64RTTCounter, Frequency;
QueryPerformanceFrequency((PLARGE_INTEGER)&Frequency);
ui64TotalCounter.QuadPart = 0;
ui64RTTCounter.QuadPart = 0;
unsigned int iterations = 0;
#ifdef SYNCHRONOUS
LARGE_INTEGER ui64CurrentCounter;
printf("Work thread sending synchronous requests...\n");
string dataStr = "Hello World, " + to_string(iterations);
QueryPerformanceCounter(&ui64CurrentCounter);
message_t outMsg(sizeof(ui64CurrentCounter) + dataStr.size());
memcpy(outMsg.data(), &ui64CurrentCounter, sizeof(ui64CurrentCounter));
memcpy((char*)outMsg.data() + sizeof(ui64CurrentCounter), dataStr.c_str(), dataStr.size());
sendSocket.send(outMsg, 0);
#endif
bool isWorking = true;
while (isWorking && g_runThreads)
{
switch (WaitForSingleObjectEx(g_queueUpdated, INFINITE, FALSE))
{
case WAIT_OBJECT_0:
while (!g_incomingQueue.empty())
{
AcquireSRWLockExclusive(&g_queueLock);
message_t* msg = g_incomingQueue.front();
g_incomingQueue.pop();
ReleaseSRWLockExclusive(&g_queueLock);
if (iterations == 0)
{
printf("First request, starting timer...\n");
QueryPerformanceCounter(&ui64StartCounter);
}
QueryPerformanceCounter(&ui64StopCounter);
LARGE_INTEGER* sendTime = (LARGE_INTEGER*)msg->data();
ui64StopCounter.QuadPart -= sendTime->QuadPart;
ui64RTTCounter.QuadPart += ui64StopCounter.QuadPart;
delete msg;
if (++iterations % g_statisticsInterval == 0)
{
QueryPerformanceCounter(&ui64StopCounter);
ui64StopCounter.QuadPart -= ui64StartCounter.QuadPart;
ui64StopCounter.QuadPart *= 1000000;
ui64StopCounter.QuadPart /= Frequency.QuadPart;
ui64RTTCounter.QuadPart /= 10000;
ui64RTTCounter.QuadPart *= 1000000;
ui64RTTCounter.QuadPart /= Frequency.QuadPart;
printf("Received %d replies, last %d took us=%lu, averate RTT=%lu\n", iterations, g_statisticsInterval, ui64StopCounter.QuadPart, ui64RTTCounter.QuadPart);
ui64TotalCounter.QuadPart += ui64StopCounter.QuadPart;
ui64RTTCounter.QuadPart = 0;
QueryPerformanceCounter(&ui64StartCounter);
}
if (iterations >= g_numOfIterations)
{
isWorking = false;
}
#ifdef SYNCHRONOUS
else
{
string dataStr = "Hello World, " + to_string(iterations);
QueryPerformanceCounter(&ui64CurrentCounter);
message_t outMsg(sizeof(ui64CurrentCounter) + dataStr.size());
memcpy(outMsg.data(), &ui64CurrentCounter, sizeof(ui64CurrentCounter));
memcpy((char*)outMsg.data() + sizeof(ui64CurrentCounter), dataStr.c_str(), dataStr.size());
sendSocket.send(outMsg, 0);
}
#endif
}
break;
default:
isWorking = false;
break;
}
}
printf("Work thread finished, %d replies received in us=%lu\n", iterations, ui64TotalCounter.QuadPart);
return NO_ERROR;
}
static DWORD WINAPI SendThreadStatic(PVOID pThreadParameter)
{
socket_t sendSocket(g_context, ZMQ_PUSH);
sendSocket.connect("inproc://sendsocket");
LARGE_INTEGER ui64CurrentCounter, ui64StartCounter, ui64StopCounter, ui64TotalCounter, Frequency;
QueryPerformanceFrequency((PLARGE_INTEGER)&Frequency);
ui64TotalCounter.QuadPart = 0;
printf("Send thread sending asynchronous requests...\n");
unsigned int iterations;
QueryPerformanceCounter(&ui64TotalCounter);
QueryPerformanceCounter(&ui64StartCounter);
for (iterations = 0; iterations < g_numOfIterations; ++iterations)
{
string dataStr = "Hello World, " + to_string(iterations);
QueryPerformanceCounter(&ui64CurrentCounter);
message_t outMsg(sizeof(ui64CurrentCounter) + dataStr.size());
memcpy(outMsg.data(), &ui64CurrentCounter, sizeof(ui64CurrentCounter));
memcpy((char*)outMsg.data() + sizeof(ui64CurrentCounter), dataStr.c_str(), dataStr.size());
sendSocket.send(outMsg, 0);
if ((iterations + 1) % g_statisticsInterval == 0)
{
QueryPerformanceCounter(&ui64StopCounter);
ui64StopCounter.QuadPart -= ui64StartCounter.QuadPart;
ui64StopCounter.QuadPart *= 1000000;
ui64StopCounter.QuadPart /= Frequency.QuadPart;
printf("Sent %d requests, last %d took us=%lu\n", iterations + 1, g_statisticsInterval, ui64StopCounter.QuadPart);
QueryPerformanceCounter(&ui64StartCounter);
}
if ((iterations + 1) % 150 == 0)
{
// Sleep(1);
}
}
QueryPerformanceCounter(&ui64StopCounter);
ui64StopCounter.QuadPart -= ui64TotalCounter.QuadPart;
ui64StopCounter.QuadPart *= 1000000;
ui64StopCounter.QuadPart /= Frequency.QuadPart;
printf("Send thread finished, %d requests sent in us=%lu\n", iterations, ui64StopCounter.QuadPart);
return NO_ERROR;
}
int _tmain(int argc, _TCHAR* argv[])
{
QueryPerformanceFrequency((PLARGE_INTEGER)&Frequency);
g_queueUpdated = CreateEvent(nullptr, FALSE, FALSE, nullptr);
PCTSTR serverAddress = argc > 1 ? argv[1] : _T("127.0.0.1");
PCTSTR port = argc > 2 ? argv[2] : _T("14767");
wstring connectionAddress = _T("tcp://");
connectionAddress += serverAddress;
connectionAddress += _T(":");
connectionAddress += port;
string address = string(connectionAddress.begin(), connectionAddress.end());
g_zmqThread = (HANDLE)_beginthreadex(nullptr, 0, (unsigned int(__stdcall *)(PVOID)) ZmqThreadStatic, (void*)address.c_str(), 0, (unsigned *)nullptr);
Sleep(20); // Allow the new thread to bind internal socket
g_workThread = (HANDLE)_beginthreadex(nullptr, 0, (unsigned int(__stdcall *)(PVOID)) WorkThreadStatic, nullptr, 0, (unsigned *)nullptr);
#ifndef SYNCHRONOUS
Sleep(20); // Allow the new thread to bind internal socket
g_sendThread = (HANDLE)_beginthreadex(nullptr, 0, (unsigned int(__stdcall *)(PVOID)) SendThreadStatic, nullptr, 0, (unsigned *)nullptr);
#endif
getchar();
printf("Closing down...\n");
g_runThreads = false;
SetEvent(g_queueUpdated);
CloseHandle(g_queueUpdated);
g_queueUpdated = INVALID_HANDLE_VALUE;
g_context.close();
printf("Successfully shutdown.\n");
getchar();
return 0;
}
服务器代码：
// ZMQServer.cpp : Defines the entry point for the console application.
//
#include "stdafx.h"
#include <profileapi.h>
#include <atomic>
#include <handleapi.h>
#include <process.h>
#include "zmq.hpp"
#include <queue>
using namespace zmq;
using namespace std;
namespace
{
HANDLE g_zmqThread = INVALID_HANDLE_VALUE;
HANDLE g_workThread = INVALID_HANDLE_VALUE;
SRWLOCK g_queueLock;
queue< pair<string, message_t*> > g_incomingQueue;
HANDLE g_queueUpdated = INVALID_HANDLE_VALUE;
bool g_threadsRunning = true;
context_t g_context(1);
context_t g_localContext(0);
const unsigned int g_statisticsInterval = 10000;
}
static DWORD WINAPI ZmqThreadStatic(PVOID pThreadParameter)
{
socket_t incomingSocket(g_localContext, ZMQ_DEALER);
incomingSocket.connect("inproc://backend");
socket_t sendSocket(g_localContext, ZMQ_PULL);
sendSocket.connect("inproc://sendsocket");
zmq_pollitem_t pollItems[] = {
{ sendSocket, 0, ZMQ_POLLIN, 0 },
{ incomingSocket, 0, ZMQ_POLLIN, 0 }
};
printf("Starting to handle incoming and outgoing data...\n");
unsigned int iterations = 0;
message_t zMsgt;
int more;
size_t size = sizeof(more);
while (g_threadsRunning)
{
zmq_poll(pollItems, 2, -1);
if (pollItems[0].revents & ZMQ_POLLIN)
{
while (true)
{
sendSocket.recv(&zMsgt, 0);
sendSocket.getsockopt(ZMQ_RCVMORE, &more, &size);
incomingSocket.send(zMsgt, more ? ZMQ_SNDMORE : 0);
if (more == 0) break;
}
}
if (pollItems[1].revents & ZMQ_POLLIN)
{
while (true)
{
incomingSocket.recv(&zMsgt, 0);
string strId(static_cast<char*>(zMsgt.data()), zMsgt.size());
incomingSocket.recv(&zMsgt);
message_t* newMsg = new message_t;
newMsg->move(&zMsgt);
AcquireSRWLockExclusive(&g_queueLock);
g_incomingQueue.push(make_pair(strId, newMsg));
ReleaseSRWLockExclusive(&g_queueLock);
SetEvent(g_queueUpdated);
incomingSocket.getsockopt(ZMQ_RCVMORE, &more, &size);
if (more == 0) break;
}
}
}
printf("Stopped handling incoming and outgoing data.\n");
return NO_ERROR;
}
static DWORD WINAPI WorkThreadStatic(PVOID pThreadParameter)
{
socket_t sendSocket(g_localContext, ZMQ_PUSH);
sendSocket.bind("inproc://sendsocket");
LARGE_INTEGER ui64StartCounter, ui64StopCounter, Frequency;
QueryPerformanceFrequency((PLARGE_INTEGER)&Frequency);
printf("Starting to process requests...\n");
unsigned int iterations = 0;
while (g_threadsRunning)
{
switch (WaitForSingleObjectEx(g_queueUpdated, INFINITE, FALSE))
{
case WAIT_OBJECT_0:
{
AcquireSRWLockExclusive(&g_queueLock);
while(!g_incomingQueue.empty())
{
pair<string, message_t*> pair_data(g_incomingQueue.front());
g_incomingQueue.pop();
ReleaseSRWLockExclusive(&g_queueLock);
string in((char*)pair_data.second->data(), pair_data.second->size());
delete pair_data.second;
string out = in + " - Reply";
// This code below doesn't work when several threads are used, since the id end up in one thread and the data in another...
message_t outMsg(out.size());
memcpy(outMsg.data(), out.c_str(), out.size());
message_t id(pair_data.first.size());
memcpy(id.data(), pair_data.first.c_str(), pair_data.first.size());
sendSocket.send(id, ZMQ_SNDMORE);
sendSocket.send(outMsg, 0);
if (iterations == 0)
{
printf("First request, starting timer...\n");
QueryPerformanceCounter(&ui64StartCounter);
}
if (++iterations % g_statisticsInterval == 0)
{
QueryPerformanceCounter(&ui64StopCounter);
ui64StopCounter.QuadPart -= ui64StartCounter.QuadPart;
ui64StopCounter.QuadPart *= 1000000;
ui64StopCounter.QuadPart /= Frequency.QuadPart;
printf("Received %d requests, last %d took us=%lu\n", iterations, g_statisticsInterval, ui64StopCounter.QuadPart);
QueryPerformanceCounter(&ui64StartCounter);
}
AcquireSRWLockExclusive(&g_queueLock);
}
ReleaseSRWLockExclusive(&g_queueLock);
break;
}
default:
g_threadsRunning = false;
break;
}
}
sendSocket.close();
printf("Stopped processing requests.\n");
return NO_ERROR;
}
BOOL WINAPI ControlCHandlerRoutine( _In_ DWORD dwCtrlType)
{
g_threadsRunning = false;
SetEvent(g_queueUpdated);
g_localContext.close();
return NO_ERROR;
}
int _tmain(int argc, _TCHAR* argv[])
{
PCTSTR port = argc > 1 ? argv[1] : _T("14767");
wstring serverAddress = _T("tcp://*:");
serverAddress += port;
string address = string(serverAddress.begin(), serverAddress.end());
g_queueUpdated = CreateEvent(nullptr, FALSE, FALSE, nullptr);
socket_t frontend(g_context, ZMQ_ROUTER);
frontend.bind(address.c_str());
socket_t backend(g_localContext, ZMQ_DEALER);
backend.bind("inproc://backend");
g_workThread = (HANDLE)_beginthreadex(nullptr, 0, (unsigned int(__stdcall *)(PVOID)) WorkThreadStatic, nullptr, 0, (unsigned *)nullptr);
g_zmqThread = (HANDLE)_beginthreadex(nullptr, 0, (unsigned int(__stdcall *)(PVOID)) ZmqThreadStatic, nullptr, 0, (unsigned *)nullptr);
printf("Listening on port: %ws\n", port);
SetConsoleCtrlHandler(ControlCHandlerRoutine, TRUE);
try
{
proxy(frontend, backend, nullptr);
}
catch (error_t& e)
{
if (e.num() != ERROR_WAIT_NO_CHILDREN)
{
printf("Proxy exception: %s\n", e.what());
}
}
printf("Closing down...\n");
CloseHandle(g_queueUpdated);
frontend.unbind(address.c_str());
frontend.close();
backend.close();
g_context.close();
printf("Successfully shutdown.\n");
return 0;
}