我们中有3人正致力于为基于epoll的服务器添加SSL，但经过3天的沮丧之后，我们决定向全世界寻求帮助。
问题是SSL_accept()总是返回SSL_ERROR_WANT_READ，我们为相关套接字的EPOLLIN和EPOLLOUT设置了epoll事件，但即使我们收到一个事件并调用SSL_accept它也只是返回想要阅读......太令人沮丧了。
似乎没有好的工作示例使用我们可以找到的带有epoll的openssl，请将代码示例发送给我们，如果有的话，我们花了几天时间来解决这个问题并阅读每个SE / SO消息并花费了openssl邮件列表上的小时数。
我们有一个基本的app结构，对于accept来说是这样的：
if (http_sock == source_fd || https_sock == source_fd) {
// new connection
req_type_t req_type = (http_sock == source_fd) ? HTTP : HTTPS;
int infd;
while (1) {
struct sockaddr_in in_addr;
socklen_t in_addr_len;
ZLOG_DBG(zlog_cat, "New %s request on source_fd: %d http_sock: %d https_sock: %d", req_type == HTTP ? "HTTP" : "HTTPS", source_fd, http_sock, https_sock);
in_addr_len = sizeof in_addr;
infd = accept4(source_fd, &in_addr, &in_addr_len, SOCK_NONBLOCK);
if (infd <= 0)
{
if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
/* We have processed all incoming
connections. */
break;
} else {
ZLOG_ERR(zlog_cat, "ERROR: accept");
break;
}
}
req = new_req_data(kds, req_type, &g_data, infd, &in_addr, in_addr_len, cert_q);
event.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
if (req->type == HTTPS) {
req->c_tls.want_ssl_accept = 1;
event.events = EPOLLIN | EPOLLRDHUP | EPOLLOUT | EPOLLET;
ssl_establish(req);
}
if ( (s = set_socket_blocking(req->client.fd, 1)) < 0)
err(EXIT_FAILURE, "[%s:%d] set_socket_blocking()", __FUNCTION__, __LINE__);
epoll_event_t *epoll_event = new_epoll_event_t(infd, req);
event.data.ptr = epoll_event;
// add new infd to our epoll listener
if ( (s = epoll_ctl(efd, EPOLL_CTL_ADD, req->client.fd, &event)) == -1 )
err(EXIT_FAILURE, "epoll_ctl(http) %s", strerror(errno));
}
ZLOG_DBG(zlog_cat, "New %s connection from %s:%d on FD %d mac_address: %s", req->type ? "HTTPS" : "HTTP",
inet_ntoa(req->client.sock.sin_addr), ntohs(req->client.sock.sin_port), req->client.fd, req->device->mac_address);
g_data.conn_counter++;
continue; // continue to next accept event
} // end new connection
ssl_establish()方法如下所示：
int ssl_establish(req_data_t *req) {
unsigned long e;
int ssl_err, rc;
// create openssl server context and ssl object
if ((req->s_tls.ctx = create_context(0)) == NULL) {
ZLOG_ERR(zlog_cat, "create_context() %s", strerror(errno));
return -1;
}
if ((req->s_tls.ssl = SSL_new(req->s_tls.ctx)) == NULL) {
ZLOG_ERR(zlog_cat, "SSL_new() %s", strerror(errno));
return -1;
}
// Establishing SSL/TLS connections with client
if (req->https_def_rsa != NULL && req->https_def_cert != NULL) {
req->c_tls.servername = req->https_def_domain;
req->c_tls.cert = req->https_def_cert;
req->c_tls.rsa_key = req->https_def_rsa;
req->c_tls.dest_ip = req->orig.sock.sin_addr.s_addr;
req->c_tls.dc_cache = req->dc_cache;
req->c_tls.q_entry = NULL;
if (req->c_tls.ctx == NULL ) {
if ((req->c_tls.ctx = create_context(1)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR create_context");
if (configure_context(&req->c_tls))
ZLOG_ERR(zlog_cat, "ERROR configure_context");
if ((req->c_tls.ssl = SSL_new(req->c_tls.ctx)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR ssl_new");
// SSL_set_mode(req->c_tls.ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); // This doesn't seem to do anything
if (SSL_set_fd(req->c_tls.ssl, req->client.fd) == 0)
ZLOG_ERR(zlog_cat, "ERROR ssl_set_fd");
}
req->client.ssl = req->c_tls.ssl;
}
if ((rc = SSL_accept(req->c_tls.ssl)) < 0) {
ssl_err = SSL_get_error(req->c_tls.ssl, rc);
switch (ssl_err) {
case SSL_ERROR_WANT_READ:
ZLOG_DBG(zlog_cat, "SSL_accept SSL_ERROR_WANT_READ");
req->c_tls.want_ssl_accept = 1;
return 0;
break;
case SSL_ERROR_WANT_WRITE:
ZLOG_DBG(zlog_cat, "SSL_accept SSL_ERROR_WANT_WRITE");
req->c_tls.want_ssl_accept = 1;
return 0;
break;
case SSL_ERROR_ZERO_RETURN:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_ZERO_RETURN");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
case SSL_ERROR_WANT_X509_LOOKUP:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_WANT_X509_LOOKUP");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
case SSL_ERROR_SYSCALL:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_SYSCALL");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
case SSL_ERROR_SSL:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_SSL");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
}
}
ZLOG_DBG(zlog_cat, "SSL_accept success");
req->c_tls.want_ssl_accept = 0;
return 1;
}
在pollin / pollout处理程序中，我们这样做：
if (req->type == HTTPS && req->c_tls.want_ssl_accept) {
ZLOG_DBG(zlog_cat, "ssl_establish EPOLLIN");
if ((rc = ssl_establish(req)) <= 0) {
ZLOG_DBG(zlog_cat, "ssl_establish EPOLLIN retry...");
continue;
}
}
我们的日志如下：
2016-08-03 23:26:32.224 New HTTPS connection from 192.168.1.195:52659 on FD 15 mac_address: e4:f8:9c:85:63:99
2016-08-03 23:26:32.224 epoll_wait returned 1
2016-08-03 23:26:32.224 epoll event number: 0 on fd: 15
2016-08-03 23:26:32.224 Reading data from Client, source_fd: 15 source_sock->fd: 15 dest_sock->fd: -1 mac_address: e4:f8:9c:85:63:99
2016-08-03 23:26:32.224 ssl_establish EPOLLIN
2016-08-03 23:26:32.484 SSL_accept SSL_ERROR_WANT_READ
2016-08-03 23:26:32.484 ssl_establish EPOLLIN retry...
2016-08-03 23:26:32.954 New HTTPS connection from 192.168.1.195:52660 on FD 16 mac_address: e4:f8:9c:85:63:99
2016-08-03 23:26:32.954 epoll_wait returned 1
2016-08-03 23:26:32.954 epoll event number: 0 on fd: 16
2016-08-03 23:26:32.954 Establishing SSL (EPOLLOUT)
2016-08-03 23:26:32.974 SSL_accept SSL_ERROR_WANT_READ
2016-08-03 23:26:32.974 ssl_establish EPOLLOUT retry...
答案 0 :(得分：1)
好的 - 我终于搞定了。如果有人偶然发现，请不要使用SSL_accept（）。
我在主结构中添加了一些状态跟踪，并更新了ssl_establish()方法，如下所示：
int ssl_establish(req_data_t *req, epoll_event_t *epoll_event ) {
unsigned long e;
int ssl_err, rc;
struct epoll_event event;
memset(&event, 0, sizeof(struct epoll_event));
event.events = req->client.events;
event.data.ptr = epoll_event;
if (!req->client.tcp_connected) {
struct pollfd pfd;
pfd.fd = req->client.fd;
pfd.events = POLLOUT | POLLERR;
int r = poll(&pfd, 1, 0);
if (r == 1 && pfd.revents == POLLOUT) {
ZLOG_DBG(zlog_cat, "tcp connected fd %d", req->client.fd);
req->client.tcp_connected = 1;
req->client.events = EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET;
event.events = req->client.events;
if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0)
ZLOG_ERR(zlog_cat, "ERROR: unable to modify epoll_ctl");
} else {
ZLOG_ERR(zlog_cat, "[%d | %d] ERROR poll fd return %d revents %d", req->client.fd, req->state, r, pfd.revents);
return -1;
}
}
// create openssl server context and ssl object
if (!req->s_tls.ctx)
if ((req->s_tls.ctx = create_context(0)) == NULL) {
ZLOG_ERR(zlog_cat, "create_context() %s", strerror(errno));
return -1;
}
if (!req->s_tls.ssl)
if ((req->s_tls.ssl = SSL_new(req->s_tls.ctx)) == NULL) {
ZLOG_ERR(zlog_cat, "SSL_new() %s", strerror(errno));
return -1;
}
req->orig.ssl = req->s_tls.ssl;
req->proxy.ssl = req->s_tls.ssl;
// Establishing SSL/TLS connections with client
if (req->https_def_rsa != NULL && req->https_def_cert != NULL) {
req->c_tls.servername = req->https_def_domain;
req->c_tls.cert = req->https_def_cert;
req->c_tls.rsa_key = req->https_def_rsa;
req->c_tls.dest_ip = req->orig.sock.sin_addr.s_addr;
req->c_tls.kudoso = req->kudoso;
req->c_tls.dc_cache = req->dc_cache;
req->c_tls.q_entry = NULL;
if (!req->errBio)
req->errBio = BIO_new_fd(2, BIO_NOCLOSE);
if (!req->c_tls.ctx) {
if ((req->c_tls.ctx = create_context(1)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR create_context");
if (configure_context(&req->c_tls))
ZLOG_ERR(zlog_cat, "ERROR configure_context");
}
if (!req->c_tls.ssl) {
if ((req->c_tls.ssl = SSL_new(req->c_tls.ctx)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR ssl_new");
// SSL_set_mode(req->c_tls.ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
if (SSL_set_fd(req->c_tls.ssl, req->client.fd) == 0)
ZLOG_ERR(zlog_cat, "ERROR ssl_set_fd");
SSL_set_accept_state(req->c_tls.ssl);
req->client.ssl = req->c_tls.ssl;
}
}
int r = SSL_do_handshake(req->c_tls.ssl);
if (r == 1) {
req->client.ssl_connected = 1;
ZLOG_DBG(zlog_cat, "[%d | %d] ssl connected", req->client.fd, req->state);
req->client.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
event.events = req->client.events;
if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0)
ZLOG_ERR(zlog_cat, "ERROR: unable to modify epoll_ctl");
return 1;
}
int err = SSL_get_error(req->c_tls.ssl, r);
int oldev = req->client.events;
if (err == SSL_ERROR_WANT_WRITE) {
req->client.events |= EPOLLOUT;
req->client.events &= ~EPOLLIN;
ZLOG_DBG(zlog_cat, "do_handshake return want write set events %d", req->client.events);
if (oldev == req->client.events) return 0;
} else if (err == SSL_ERROR_WANT_READ) {
req->client.events |= EPOLLIN;
req->client.events &= ~EPOLLOUT;
ZLOG_DBG(zlog_cat, "do_handshake return want read set events %d", req->client.events);
if (oldev == req->client.events) return 0;
} else {
ZLOG_ERR(zlog_cat, "ERROR SSL_do_handshake return %d error %d errno %d msg %s", r, err, errno, strerror(errno));
ERR_print_errors(req->errBio);
return -1;
}
event.events = req->client.events;
if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0)
ZLOG_ERR(zlog_cat, "[%d | %d] ERROR: unable to modify epoll_ctl", req->client.fd, req->state);
return 0;
}