client, server: implement experimental support for performing SSL operations in worker threads

client/PasswordListener.c

@@ -61,6 +61,11 @@ void remove_client (struct PasswordListenerClient *client)
 {
     PasswordListener *l = client->l;
     
+    // stop using any buffers before they get freed
+    if (l->ssl) {
+        BSSLConnection_ReleaseBuffers(&client->sslcon);
+    }
+    
     // free receiver
     SingleStreamReceiver_Free(&client->receiver);
     
@@ -121,7 +126,7 @@ void listener_handler (PasswordListener *l)
     
     if (l->ssl) {
         // create bottom NSPR file descriptor
-        if (!BSSLConnection_MakeBackend(&client->sock->bottom_prfd, send_if, recv_if)) {
+        if (!BSSLConnection_MakeBackend(&client->sock->bottom_prfd, send_if, recv_if, l->twd, l->ssl_flags)) {
             BLog(BLOG_ERROR, "BSSLConnection_MakeBackend failed");
             goto fail2;
         }
@@ -222,6 +227,11 @@ void client_receiver_handler (struct PasswordListenerClient *client)
     // remove password entry
     BAVL_Remove(&l->passwords, &pw_entry->tree_node);
     
+    // stop using any buffers before they get freed
+    if (l->ssl) {
+        BSSLConnection_ReleaseBuffers(&client->sslcon);
+    }
+    
     // free receiver
     SingleStreamReceiver_Free(&client->receiver);
     
@@ -246,7 +256,7 @@ void client_receiver_handler (struct PasswordListenerClient *client)
     return;
 }
 
-int PasswordListener_Init (PasswordListener *l, BReactor *bsys, BAddr listen_addr, int max_clients, int ssl, CERTCertificate *cert, SECKEYPrivateKey *key)
+int PasswordListener_Init (PasswordListener *l, BReactor *bsys, BThreadWorkDispatcher *twd, BAddr listen_addr, int max_clients, int ssl, int ssl_flags, CERTCertificate *cert, SECKEYPrivateKey *key)
 {
     ASSERT(BConnection_AddressSupported(listen_addr))
     ASSERT(max_clients > 0)
@@ -254,7 +264,9 @@ int PasswordListener_Init (PasswordListener *l, BReactor *bsys, BAddr listen_add
     
     // init arguments
     l->bsys = bsys;
+    l->twd = twd;
     l->ssl = ssl;
+    l->ssl_flags = ssl_flags;
     
     // allocate client entries
     if (!(l->clients_data = (struct PasswordListenerClient *)BAllocArray(max_clients, sizeof(struct PasswordListenerClient)))) {

client/PasswordListener.h

@@ -73,7 +73,9 @@ struct PasswordListenerClient;
  */
 typedef struct {
     BReactor *bsys;
+    BThreadWorkDispatcher *twd;
     int ssl;
+    int ssl_flags;
     PRFileDesc model_dprfd;
     PRFileDesc *model_prfd;
     struct PasswordListenerClient *clients_data;
@@ -105,15 +107,19 @@ struct PasswordListenerClient {
  * 
  * @param l the object
  * @param bsys reactor we live in
+ * @param twd thread work dispatcher. May be NULL if ssl_flags does not request performing SSL
+ *            operations in threads.
  * @param listen_addr address to listen on. Must be supported according to {@link BConnection_AddressSupported}.
  * @param max_clients maximum number of client to hold until they are identified.
  *                    Must be >0.
  * @param ssl whether to use TLS. Must be 1 or 0.
+ * @param ssl_flags flags passed down to {@link BSSLConnection_MakeBackend}. May be used to
+ *                  request performing SSL operations in threads.
  * @param cert if using TLS, the server certificate
  * @param key if using TLS, the private key
  * @return 1 on success, 0 on failure
  */
-int PasswordListener_Init (PasswordListener *l, BReactor *bsys, BAddr listen_addr, int max_clients, int ssl, CERTCertificate *cert, SECKEYPrivateKey *key) WARN_UNUSED;
+int PasswordListener_Init (PasswordListener *l, BReactor *bsys, BThreadWorkDispatcher *twd, BAddr listen_addr, int max_clients, int ssl, int ssl_flags, CERTCertificate *cert, SECKEYPrivateKey *key) WARN_UNUSED;
 
 /**
  * Frees the object.

client/PeerChat.c

@@ -193,8 +193,8 @@ static void ssl_recv_decoder_handler_error (PeerChat *o)
     return;
 }
 
-int PeerChat_Init (PeerChat *o, peerid_t peer_id, int ssl_mode, CERTCertificate *ssl_cert, SECKEYPrivateKey *ssl_key,
-                   uint8_t *ssl_peer_cert, int ssl_peer_cert_len, BPendingGroup *pg, void *user,
+int PeerChat_Init (PeerChat *o, peerid_t peer_id, int ssl_mode, int ssl_flags, CERTCertificate *ssl_cert, SECKEYPrivateKey *ssl_key,
+                   uint8_t *ssl_peer_cert, int ssl_peer_cert_len, BPendingGroup *pg, BThreadWorkDispatcher *twd, void *user,
                    BLog_logfunc logfunc,
                    PeerChat_handler_error handler_error,
                    PeerChat_handler_message handler_message)
@@ -244,7 +244,7 @@ int PeerChat_Init (PeerChat *o, peerid_t peer_id, int ssl_mode, CERTCertificate
         StreamPacketSender_Init(&o->ssl_sp_sender, send_buf_output, pg);
         
         // init SSL bottom prfd
-        if (!BSSLConnection_MakeBackend(&o->ssl_bottom_prfd, StreamPacketSender_GetInput(&o->ssl_sp_sender), SimpleStreamBuffer_GetOutput(&o->ssl_recv_buf))) {
+        if (!BSSLConnection_MakeBackend(&o->ssl_bottom_prfd, StreamPacketSender_GetInput(&o->ssl_sp_sender), SimpleStreamBuffer_GetOutput(&o->ssl_recv_buf), twd, ssl_flags)) {
             PeerLog(o, BLOG_ERROR, "BSSLConnection_MakeBackend failed");
             goto fail2;
         }
@@ -366,6 +366,11 @@ void PeerChat_Free (PeerChat *o)
     DebugObject_Free(&o->d_obj);
     DebugError_Free(&o->d_err);
     
+    // stop using any buffers before they get freed
+    if (o->ssl_mode != PEERCHAT_SSL_NONE) {
+        BSSLConnection_ReleaseBuffers(&o->ssl_con);
+    }
+    
     PacketBuffer_Free(&o->send_buf);
     BufferWriter_Free(&o->send_writer);
     if (o->ssl_mode != PEERCHAT_SSL_NONE) {

client/PeerChat.h

@@ -109,8 +109,8 @@ typedef struct {
     DebugObject d_obj;
 } PeerChat;
 
-int PeerChat_Init (PeerChat *o, peerid_t peer_id, int ssl_mode, CERTCertificate *ssl_cert, SECKEYPrivateKey *ssl_key,
-                   uint8_t *ssl_peer_cert, int ssl_peer_cert_len, BPendingGroup *pg, void *user,
+int PeerChat_Init (PeerChat *o, peerid_t peer_id, int ssl_mode, int ssl_flags, CERTCertificate *ssl_cert, SECKEYPrivateKey *ssl_key,
+                   uint8_t *ssl_peer_cert, int ssl_peer_cert_len, BPendingGroup *pg, BThreadWorkDispatcher *twd, void *user,
                    BLog_logfunc logfunc,
                    PeerChat_handler_error handler_error,
                    PeerChat_handler_message handler_message) WARN_UNUSED;

client/StreamPeerIO.c

@@ -104,7 +104,7 @@ void connector_handler (StreamPeerIO *pio, int is_error)
         BConnection_RecvAsync_Init(&pio->connect.sock.con);
         
         // create bottom NSPR file descriptor
-        if (!BSSLConnection_MakeBackend(&pio->connect.sock.bottom_prfd, BConnection_SendAsync_GetIf(&pio->connect.sock.con), BConnection_RecvAsync_GetIf(&pio->connect.sock.con))) {
+        if (!BSSLConnection_MakeBackend(&pio->connect.sock.bottom_prfd, BConnection_SendAsync_GetIf(&pio->connect.sock.con), BConnection_RecvAsync_GetIf(&pio->connect.sock.con), pio->twd, pio->ssl_flags)) {
             PeerLog(pio, BLOG_ERROR, "BSSLConnection_MakeBackend failed");
             goto fail1;
         }
@@ -230,6 +230,11 @@ void pwsender_handler (StreamPeerIO *pio)
     ASSERT(pio->mode == MODE_CONNECT)
     ASSERT(pio->connect.state == CONNECT_STATE_SENDING)
     
+    // stop using any buffers before they get freed
+    if (pio->ssl) {
+        BSSLConnection_ReleaseBuffers(&pio->connect.sslcon);
+    }
+    
     // free password sender
     SingleStreamSender_Free(&pio->connect.pwsender);
     
@@ -345,6 +350,11 @@ void free_io (StreamPeerIO *pio)
 {
     ASSERT(pio->sock)
     
+    // stop using any buffers before they get freed
+    if (pio->ssl) {
+        BSSLConnection_ReleaseBuffers(&pio->sslcon);
+    }
+    
     // reset decoder
     PacketProtoDecoder_Reset(&pio->input_decoder);
     
@@ -496,6 +506,9 @@ void reset_state (StreamPeerIO *pio)
                 case CONNECT_STATE_SENT:
                 case CONNECT_STATE_SENDING:
                     if (pio->connect.state == CONNECT_STATE_SENDING) {
+                        if (pio->ssl) {
+                            BSSLConnection_ReleaseBuffers(&pio->connect.sslcon);
+                        }
                         SingleStreamSender_Free(&pio->connect.pwsender);
                         if (!pio->ssl) {
                             BConnection_SendAsync_Free(&pio->connect.sock.con);
@@ -539,7 +552,9 @@ void reset_and_report_error (StreamPeerIO *pio)
 int StreamPeerIO_Init (
     StreamPeerIO *pio,
     BReactor *reactor,
+    BThreadWorkDispatcher *twd,
     int ssl,
+    int ssl_flags,
     uint8_t *ssl_peer_cert,
     int ssl_peer_cert_len,
     int payload_mtu,
@@ -557,8 +572,10 @@ int StreamPeerIO_Init (
     
     // init arguments
     pio->reactor = reactor;
+    pio->twd = twd;
     pio->ssl = ssl;
     if (pio->ssl) {
+        pio->ssl_flags = ssl_flags;
         pio->ssl_peer_cert = ssl_peer_cert;
         pio->ssl_peer_cert_len = ssl_peer_cert_len;
     }

client/StreamPeerIO.h

@@ -74,7 +74,9 @@ typedef void (*StreamPeerIO_handler_error) (void *user);
 typedef struct {
     // common arguments
     BReactor *reactor;
+    BThreadWorkDispatcher *twd;
     int ssl;
+    int ssl_flags;
     uint8_t *ssl_peer_cert;
     int ssl_peer_cert_len;
     int payload_mtu;
@@ -138,7 +140,11 @@ typedef struct {
  *
  * @param pio the object
  * @param reactor reactor we live in
+ * @param twd thread work dispatcher. May be NULL if ssl_flags does not request performing SSL
+ *            operations in threads.
  * @param ssl if nonzero, SSL will be used for peer connection
+ * @param ssl_flags flags passed down to {@link BSSLConnection_MakeBackend}. May be used to
+ *                  request performing SSL operations in threads.
  * @param ssl_peer_cert if using SSL, the certificate we expect the peer to have
  * @param ssl_peer_cert_len if using SSL, the length of the certificate
  * @param payload_mtu maximum packet size as seen from the user. Must be >=0.
@@ -153,7 +159,9 @@ typedef struct {
 int StreamPeerIO_Init (
     StreamPeerIO *pio,
     BReactor *reactor,
+    BThreadWorkDispatcher *twd,
     int ssl,
+    int ssl_flags,
     uint8_t *ssl_peer_cert,
     int ssl_peer_cert_len,
     int payload_mtu,

client/client.c

@@ -56,6 +56,7 @@
 #include <nspr_support/BSSLConnection.h>
 #include <server_connection/ServerConnection.h>
 #include <tuntap/BTap.h>
+#include <threadwork/BThreadWork.h>
 
 #ifndef BADVPN_USE_WINAPI
 #include <base/BLog_syslog.h>
@@ -93,6 +94,8 @@ struct {
     int loglevel;
     int loglevels[BLOG_NUM_CHANNELS];
     int threads;
+    int use_threads_for_ssl_handshake;
+    int use_threads_for_ssl_data;
     int ssl;
     char *nssdb;
     char *client_cert_name;
@@ -217,6 +220,8 @@ static int parse_arguments (int argc, char *argv[]);
 // processes certain command line options
 static int process_arguments (void);
 
+static int ssl_flags (void);
+
 // handler for program termination request
 static void signal_handler (void *unused);
 
@@ -416,6 +421,45 @@ int main (int argc, char *argv[])
     
     BLog(BLOG_NOTICE, "initializing "GLOBAL_PRODUCT_NAME" "PROGRAM_NAME" "GLOBAL_VERSION);
     
+    if (options.ssl) {
+        // init NSPR
+        PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0);
+        
+        // register local NSPR file types
+        if (!DummyPRFileDesc_GlobalInit()) {
+            BLog(BLOG_ERROR, "DummyPRFileDesc_GlobalInit failed");
+            goto fail01;
+        }
+        if (!BSSLConnection_GlobalInit()) {
+            BLog(BLOG_ERROR, "BSSLConnection_GlobalInit failed");
+            goto fail01;
+        }
+        
+        // init NSS
+        if (NSS_Init(options.nssdb) != SECSuccess) {
+            BLog(BLOG_ERROR, "NSS_Init failed (%d)", (int)PR_GetError());
+            goto fail01;
+        }
+        
+        // set cipher policy
+        if (NSS_SetDomesticPolicy() != SECSuccess) {
+            BLog(BLOG_ERROR, "NSS_SetDomesticPolicy failed (%d)", (int)PR_GetError());
+            goto fail02;
+        }
+        
+        // init server cache
+        if (SSL_ConfigServerSessionIDCache(0, 0, 0, NULL) != SECSuccess) {
+            BLog(BLOG_ERROR, "SSL_ConfigServerSessionIDCache failed (%d)", (int)PR_GetError());
+            goto fail02;
+        }
+        
+        // open server certificate and private key
+        if (!open_nss_cert_and_key(options.client_cert_name, &client_cert, &client_key)) {
+            BLog(BLOG_ERROR, "Cannot open certificate and key");
+            goto fail03;
+        }
+    }
+    
     // initialize network
     if (!BNetwork_GlobalInit()) {
         BLog(BLOG_ERROR, "BNetwork_GlobalInit failed");
@@ -457,51 +501,12 @@ int main (int argc, char *argv[])
         }
     }
     
-    if (options.ssl) {
-        // init NSPR
-        PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0);
-        
-        // register local NSPR file types
-        if (!DummyPRFileDesc_GlobalInit()) {
-            BLog(BLOG_ERROR, "DummyPRFileDesc_GlobalInit failed");
-            goto fail5;
-        }
-        if (!BSSLConnection_GlobalInit()) {
-            BLog(BLOG_ERROR, "BSSLConnection_GlobalInit failed");
-            goto fail5;
-        }
-        
-        // init NSS
-        if (NSS_Init(options.nssdb) != SECSuccess) {
-            BLog(BLOG_ERROR, "NSS_Init failed (%d)", (int)PR_GetError());
-            goto fail5;
-        }
-        
-        // set cipher policy
-        if (NSS_SetDomesticPolicy() != SECSuccess) {
-            BLog(BLOG_ERROR, "NSS_SetDomesticPolicy failed (%d)", (int)PR_GetError());
-            goto fail6;
-        }
-        
-        // init server cache
-        if (SSL_ConfigServerSessionIDCache(0, 0, 0, NULL) != SECSuccess) {
-            BLog(BLOG_ERROR, "SSL_ConfigServerSessionIDCache failed (%d)", (int)PR_GetError());
-            goto fail6;
-        }
-        
-        // open server certificate and private key
-        if (!open_nss_cert_and_key(options.client_cert_name, &client_cert, &client_key)) {
-            BLog(BLOG_ERROR, "Cannot open certificate and key");
-            goto fail7;
-        }
-    }
-    
     // init listeners
     int num_listeners = 0;
     if (options.transport_mode == TRANSPORT_MODE_TCP) {
         while (num_listeners < num_bind_addrs) {
             struct bind_addr *addr = &bind_addrs[num_listeners];
-            if (!PasswordListener_Init(&listeners[num_listeners], &ss, addr->addr, TCP_MAX_PASSWORD_LISTENER_CLIENTS, options.peer_ssl, client_cert, client_key)) {
+            if (!PasswordListener_Init(&listeners[num_listeners], &ss, &twd, addr->addr, TCP_MAX_PASSWORD_LISTENER_CLIENTS, options.peer_ssl, ssl_flags(), client_cert, client_key)) {
                 BLog(BLOG_ERROR, "PasswordListener_Init failed");
                 goto fail8;
             }
@@ -553,7 +558,7 @@ int main (int argc, char *argv[])
     LinkedList1_Init(&waiting_relay_peers);
     
     // start connecting to server
-    if (!ServerConnection_Init(&server, &ss, server_addr, SC_KEEPALIVE_INTERVAL, SERVER_BUFFER_MIN_PACKETS, options.ssl, client_cert, client_key, server_name, NULL,
+    if (!ServerConnection_Init(&server, &ss, &twd, server_addr, SC_KEEPALIVE_INTERVAL, SERVER_BUFFER_MIN_PACKETS, options.ssl, ssl_flags(), client_cert, client_key, server_name, NULL,
                                server_handler_error, server_handler_ready, server_handler_newclient, server_handler_endclient, server_handler_message
     )) {
         BLog(BLOG_ERROR, "ServerConnection_Init failed");
@@ -570,9 +575,12 @@ int main (int argc, char *argv[])
     BLog(BLOG_NOTICE, "entering event loop");
     BReactor_Exec(&ss);
     
-    // allow freeing server queue flows
     if (server_ready) {
+        // allow freeing server queue flows
         PacketPassFairQueue_PrepareFree(&server_queue);
+        
+        // make ServerConnection stop using buffers from peers before they are freed
+        ServerConnection_ReleaseBuffers(&server);
     }
     
     // free peers
@@ -604,28 +612,29 @@ fail8:
             PasswordListener_Free(&listeners[num_listeners]);
         }
     }
-    if (options.ssl) {
-        CERT_DestroyCertificate(client_cert);
-        SECKEY_DestroyPrivateKey(client_key);
-fail7:
-        ASSERT_FORCE(SSL_ShutdownServerSessionIDCache() == SECSuccess)
-fail6:
-        SSL_ClearSessionCache();
-        ASSERT_FORCE(NSS_Shutdown() == SECSuccess)
-fail5:
-        ASSERT_FORCE(PR_Cleanup() == PR_SUCCESS)
-        PL_ArenaFinish();
-    }
     if (BThreadWorkDispatcher_UsingThreads(&twd)) {
         BSecurity_GlobalFreeThreadSafe();
     }
 fail4:
+    // NOTE: BThreadWorkDispatcher must be freed before NSPR and stuff
     BThreadWorkDispatcher_Free(&twd);
 fail3:
     BSignal_Finish();
 fail2:
     BReactor_Free(&ss);
 fail1:
+    if (options.ssl) {
+        CERT_DestroyCertificate(client_cert);
+        SECKEY_DestroyPrivateKey(client_key);
+fail03:
+        ASSERT_FORCE(SSL_ShutdownServerSessionIDCache() == SECSuccess)
+fail02:
+        SSL_ClearSessionCache();
+        ASSERT_FORCE(NSS_Shutdown() == SECSuccess)
+fail01:
+        ASSERT_FORCE(PR_Cleanup() == PR_SUCCESS)
+        PL_ArenaFinish();
+    }
     BLog(BLOG_NOTICE, "exiting");
     BLog_Free();
 fail0:
@@ -659,6 +668,8 @@ void print_help (const char *name)
         "        [--loglevel <0-5/none/error/warning/notice/info/debug>]\n"
         "        [--channel-loglevel <channel-name> <0-5/none/error/warning/notice/info/debug>] ...\n"
         "        [--threads <integer>]\n"
+        "        [--use-threads-for-ssl-handshake]\n"
+        "        [--use-threads-for-ssl-data]\n"
         "        [--ssl --nssdb <string> --client-cert-name <string>]\n"
         "        [--server-name <string>]\n"
         "        --server-addr <addr>\n"
@@ -716,6 +727,8 @@ int parse_arguments (int argc, char *argv[])
         options.loglevels[i] = -1;
     }
     options.threads = 0;
+    options.use_threads_for_ssl_handshake = 0;
+    options.use_threads_for_ssl_data = 0;
     options.ssl = 0;
     options.nssdb = NULL;
     options.client_cert_name = NULL;
@@ -826,6 +839,12 @@ int parse_arguments (int argc, char *argv[])
             options.threads = atoi(argv[i + 1]);
             i++;
         }
+        else if (!strcmp(arg, "--use-threads-for-ssl-handshake")) {
+            options.use_threads_for_ssl_handshake = 1;
+        }
+        else if (!strcmp(arg, "--use-threads-for-ssl-data")) {
+            options.use_threads_for_ssl_data = 1;
+        }
         else if (!strcmp(arg, "--ssl")) {
             options.ssl = 1;
         }
@@ -1283,6 +1302,18 @@ int process_arguments (void)
     return 1;
 }
 
+int ssl_flags (void)
+{
+    int flags = 0;
+    if (options.use_threads_for_ssl_handshake) {
+        flags |= BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE;
+    }
+    if (options.use_threads_for_ssl_data) {
+        flags |= BSSLCONNECTION_FLAG_THREADWORK_IO;
+    }
+    return flags;
+}
+
 void signal_handler (void *unused)
 {
     BLog(BLOG_NOTICE, "termination requested");
@@ -1385,7 +1416,7 @@ void peer_add (peerid_t id, int flags, const uint8_t *cert, int cert_len)
     }
     
     // init chat
-    if (!PeerChat_Init(&peer->chat, peer->id, chat_ssl_mode, client_cert, client_key, peer->cert, peer->cert_len, BReactor_PendingGroup(&ss), peer,
+    if (!PeerChat_Init(&peer->chat, peer->id, chat_ssl_mode, ssl_flags(), client_cert, client_key, peer->cert, peer->cert_len, BReactor_PendingGroup(&ss), &twd, peer,
         (BLog_logfunc)peer_logfunc,
         (PeerChat_handler_error)peer_chat_handler_error,
         (PeerChat_handler_message)peer_chat_handler_message
@@ -1614,7 +1645,7 @@ int peer_init_link (struct peer_data *peer)
     } else {
         // init StreamPeerIO
         if (!StreamPeerIO_Init(
-            &peer->pio.tcp.pio, &ss, options.peer_ssl,
+            &peer->pio.tcp.pio, &ss, &twd, options.peer_ssl, ssl_flags(),
             (options.peer_ssl ? peer->cert : NULL),
             (options.peer_ssl ? peer->cert_len : -1),
             data_mtu,

flooder/flooder.c

@@ -277,7 +277,7 @@ int main (int argc, char *argv[])
     
     // start connecting to server
     if (!ServerConnection_Init(
-        &server, &ss, server_addr, SC_KEEPALIVE_INTERVAL, SERVER_BUFFER_MIN_PACKETS, options.ssl, client_cert, client_key, server_name, NULL,
+        &server, &ss, NULL, server_addr, SC_KEEPALIVE_INTERVAL, SERVER_BUFFER_MIN_PACKETS, options.ssl, 0, client_cert, client_key, server_name, NULL,
         server_handler_error, server_handler_ready, server_handler_newclient, server_handler_endclient, server_handler_message
     )) {
         BLog(BLOG_ERROR, "ServerConnection_Init failed");
@@ -292,6 +292,7 @@ int main (int argc, char *argv[])
     BReactor_Exec(&ss);
     
     if (server_ready) {
+        ServerConnection_ReleaseBuffers(&server);
         SinglePacketBuffer_Free(&flood_buffer);
         PacketProtoEncoder_Free(&flood_encoder);
         PacketRecvInterface_Free(&flood_source);

nspr_support/BSSLConnection.c

@@ -40,10 +40,20 @@
 
 #include <generated/blog_channel_BSSLConnection.h>
 
+#define THREADWORK_STATE_NONE 0
+#define THREADWORK_STATE_HANDSHAKE 1
+#define THREADWORK_STATE_READ 2
+#define THREADWORK_STATE_WRITE 3
+
+static void backend_threadwork_start (struct BSSLConnection_backend *b, int op);
+static int backend_threadwork_do_io (struct BSSLConnection_backend *b);
 static void connection_init_job_handler (BSSLConnection *o);
 static void connection_init_up (BSSLConnection *o);
 static void connection_try_io (BSSLConnection *o);
+static void connection_threadwork_func_work (void *user);
+static void connection_threadwork_handler_done (void *user);
 static void connection_recv_job_handler (BSSLConnection *o);
+static void connection_try_handshake (BSSLConnection *o);
 static void connection_try_send (BSSLConnection *o);
 static void connection_try_recv (BSSLConnection *o);
 static void connection_send_if_handler_send (BSSLConnection *o, uint8_t *data, int data_len);
@@ -65,6 +75,13 @@ static PRStatus method_close (PRFileDesc *fd)
 {
     struct BSSLConnection_backend *b = (struct BSSLConnection_backend *)fd->secret;
     ASSERT(!b->con)
+    ASSERT(b->threadwork_state == THREADWORK_STATE_NONE)
+    
+    // free mutexes
+    if ((b->flags & BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE) || (b->flags & BSSLCONNECTION_FLAG_THREADWORK_IO)) {
+        BMutex_Free(&b->recv_buf_mutex);
+        BMutex_Free(&b->send_buf_mutex);
+    }
     
     // free backend
     free(b);
@@ -80,17 +97,25 @@ static PRInt32 method_read (PRFileDesc *fd, void *buf, PRInt32 amount)
     struct BSSLConnection_backend *b = (struct BSSLConnection_backend *)fd->secret;
     ASSERT(amount > 0)
     
+    if (b->threadwork_state != THREADWORK_STATE_NONE) {
+        BMutex_Lock(&b->recv_buf_mutex);
+    }
+    
     // if we are receiving into buffer or buffer has no data left, refuse recv
     if (b->recv_busy || b->recv_pos == b->recv_len) {
-        // start receiving if not already
-        if (!b->recv_busy) {
-            // set recv busy
-            b->recv_busy = 1;
-            
-            // receive into buffer
-            StreamRecvInterface_Receiver_Recv(b->recv_if, b->recv_buf, BSSLCONNECTION_BUF_SIZE);
+        if (b->threadwork_state != THREADWORK_STATE_NONE) {
+            b->threadwork_want_recv = 1;
+            BMutex_Unlock(&b->recv_buf_mutex);
+        } else {
+            // start receiving if not already
+            if (!b->recv_busy) {
+                // set recv busy
+                b->recv_busy = 1;
+                
+                // receive into buffer
+                StreamRecvInterface_Receiver_Recv(b->recv_if, b->recv_buf, BSSLCONNECTION_BUF_SIZE);
+            }
         }
-        
         PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
         return -1;
     }
@@ -106,6 +131,10 @@ static PRInt32 method_read (PRFileDesc *fd, void *buf, PRInt32 amount)
     // update buffer
     b->recv_pos += amount;
     
+    if (b->threadwork_state != THREADWORK_STATE_NONE) {
+        BMutex_Unlock(&b->recv_buf_mutex);
+    }
+    
     return amount;
 }
 
@@ -114,8 +143,18 @@ static PRInt32 method_write (PRFileDesc *fd, const void *buf, PRInt32 amount)
     struct BSSLConnection_backend *b = (struct BSSLConnection_backend *)fd->secret;
     ASSERT(amount > 0)
     
+    if (b->threadwork_state != THREADWORK_STATE_NONE) {
+        BMutex_Lock(&b->send_buf_mutex);
+    }
+    
+    ASSERT(!b->send_busy || b->send_pos < b->send_len)
+    
     // if there is data in buffer, refuse send
     if (b->send_pos < b->send_len) {
+        if (b->threadwork_state != THREADWORK_STATE_NONE) {
+            b->threadwork_want_send = 1;
+            BMutex_Unlock(&b->send_buf_mutex);
+        }
         PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
         return -1;
     }
@@ -130,8 +169,13 @@ static PRInt32 method_write (PRFileDesc *fd, const void *buf, PRInt32 amount)
     b->send_pos = 0;
     b->send_len = amount;
     
-    // start sending
-    StreamPassInterface_Sender_Send(b->send_if, b->send_buf + b->send_pos, b->send_len - b->send_pos);
+    if (b->threadwork_state != THREADWORK_STATE_NONE) {
+        BMutex_Unlock(&b->send_buf_mutex);
+    } else {
+        // start sending
+        b->send_busy = 1;
+        StreamPassInterface_Sender_Send(b->send_if, b->send_buf + b->send_pos, b->send_len - b->send_pos);
+    }
     
     return amount;
 }
@@ -277,20 +321,35 @@ static PRIOMethods methods = {
 
 static void backend_send_if_handler_done (struct BSSLConnection_backend *b, int data_len)
 {
+    ASSERT(b->send_busy)
     ASSERT(b->send_len > 0)
     ASSERT(b->send_pos < b->send_len)
     ASSERT(data_len > 0)
     ASSERT(data_len <= b->send_len - b->send_pos)
     
+    if (b->threadwork_state != THREADWORK_STATE_NONE) {
+        BMutex_Lock(&b->send_buf_mutex);
+    }
+    
     // update buffer
     b->send_pos += data_len;
     
     // send more if needed
     if (b->send_pos < b->send_len) {
         StreamPassInterface_Sender_Send(b->send_if, b->send_buf + b->send_pos, b->send_len - b->send_pos);
+        if (b->threadwork_state != THREADWORK_STATE_NONE) {
+            BMutex_Unlock(&b->send_buf_mutex);
+        }
         return;
     }
     
+    // set send not busy
+    b->send_busy = 0;
+    
+    if (b->threadwork_state != THREADWORK_STATE_NONE) {
+        BMutex_Unlock(&b->send_buf_mutex);
+    }
+    
     // notify connection
     if (b->con && !b->con->have_error) {
         connection_try_io(b->con);
@@ -304,11 +363,19 @@ static void backend_recv_if_handler_done (struct BSSLConnection_backend *b, int
     ASSERT(data_len > 0)
     ASSERT(data_len <= BSSLCONNECTION_BUF_SIZE)
     
+    if (b->threadwork_state != THREADWORK_STATE_NONE) {
+        BMutex_Lock(&b->recv_buf_mutex);
+    }
+    
     // init buffer
     b->recv_busy = 0;
     b->recv_pos = 0;
     b->recv_len = data_len;
     
+    if (b->threadwork_state != THREADWORK_STATE_NONE) {
+        BMutex_Unlock(&b->recv_buf_mutex);
+    }
+    
     // notify connection
     if (b->con && !b->con->have_error) {
         connection_try_io(b->con);
@@ -316,6 +383,39 @@ static void backend_recv_if_handler_done (struct BSSLConnection_backend *b, int
     }
 }
 
+static void backend_threadwork_start (struct BSSLConnection_backend *b, int op)
+{
+    ASSERT(b->con)
+    ASSERT(b->threadwork_state == THREADWORK_STATE_NONE)
+    ASSERT(op == THREADWORK_STATE_HANDSHAKE || op == THREADWORK_STATE_READ || op == THREADWORK_STATE_WRITE)
+    
+    b->threadwork_state = op;
+    b->threadwork_want_recv = 0;
+    b->threadwork_want_send = 0;
+    BThreadWork_Init(&b->threadwork, b->twd, connection_threadwork_handler_done, b->con, connection_threadwork_func_work, b->con);
+}
+
+static int backend_threadwork_do_io (struct BSSLConnection_backend *b)
+{
+    ASSERT(b->con)
+    ASSERT(b->threadwork_state == THREADWORK_STATE_NONE)
+    
+    int io_ready = (b->threadwork_want_recv && !b->recv_busy && b->recv_pos < b->recv_len) ||
+                   (b->threadwork_want_send && b->send_pos == b->send_len);
+    
+    if (b->threadwork_want_recv && b->recv_pos == b->recv_len && !b->recv_busy) {
+        b->recv_busy = 1;
+        StreamRecvInterface_Receiver_Recv(b->recv_if, b->recv_buf, BSSLCONNECTION_BUF_SIZE);
+    }
+    
+    if (b->send_pos < b->send_len && !b->send_busy) {
+        b->send_busy = 1;
+        StreamPassInterface_Sender_Send(b->send_if, b->send_buf + b->send_pos, b->send_len - b->send_pos);
+    }
+    
+    return io_ready;
+}
+
 static void connection_report_error (BSSLConnection *o)
 {
     ASSERT(!o->have_error)
@@ -333,12 +433,15 @@ static void connection_init_job_handler (BSSLConnection *o)
     ASSERT(!o->have_error)
     ASSERT(!o->up)
     
-    connection_try_io(o);
-    return;
+    connection_try_handshake(o);
 }
 
 static void connection_init_up (BSSLConnection *o)
 {
+    // unset init job
+    // (just in the impossible case that handshake completed before the init job executed)
+    BPending_Unset(&o->init_job);
+    
     // init send interface
     StreamPassInterface_Init(&o->send_if, (StreamPassInterface_handler_send)connection_send_if_handler_send, o, o->pg);
     
@@ -364,28 +467,7 @@ static void connection_try_io (BSSLConnection *o)
     ASSERT(!o->have_error)
     
     if (!o->up) {
-        // unset init job (in case backend called us before it executed)
-        BPending_Unset(&o->init_job);
-        
-        // try handshake
-        SECStatus res = SSL_ForceHandshake(o->prfd);
-        if (res == SECFailure) {
-            PRErrorCode error = PR_GetError();
-            if (error == PR_WOULD_BLOCK_ERROR) {
-                return;
-            }
-            
-            BLog(BLOG_ERROR, "SSL_ForceHandshake failed (%"PRIi32")", error);
-            
-            connection_report_error(o);
-            return;
-        }
-        
-        // init up
-        connection_init_up(o);
-        
-        // report up
-        o->handler(o->user, BSSLCONNECTION_EVENT_UP);
+        connection_try_handshake(o);
         return;
     }
     
@@ -404,6 +486,163 @@ static void connection_try_io (BSSLConnection *o)
     }
 }
 
+static void connection_threadwork_func_work (void *user)
+{
+    BSSLConnection *o = user;
+    struct BSSLConnection_backend *b = o->backend;
+    ASSERT(b->threadwork_state != THREADWORK_STATE_NONE)
+    
+    switch (b->threadwork_state) {
+        case THREADWORK_STATE_HANDSHAKE:
+            b->threadwork_result_sec = SSL_ForceHandshake(o->prfd);
+            break;
+        case THREADWORK_STATE_WRITE:
+            b->threadwork_result_pr = PR_Write(o->prfd, o->send_data, o->send_len);
+            break;
+        case THREADWORK_STATE_READ:
+            b->threadwork_result_pr = PR_Read(o->prfd, o->recv_data, o->recv_avail);
+            break;
+        default:
+            ASSERT(0);
+    }
+    
+    b->threadwork_error = PR_GetError();
+}
+
+static void connection_threadwork_handler_done (void *user)
+{
+    BSSLConnection *o = user;
+    struct BSSLConnection_backend *b = o->backend;
+    ASSERT(b->threadwork_state != THREADWORK_STATE_NONE)
+    
+    // remember what operation the threadwork was performing
+    int op = b->threadwork_state;
+    
+    // free threadwork
+    BThreadWork_Free(&b->threadwork);
+    b->threadwork_state = THREADWORK_STATE_NONE;
+    
+    // start any necessary backend I/O operations, and determine if any of the requested
+    // backend I/O that was not available at the time is now available
+    int io_ready = backend_threadwork_do_io(b);
+    
+    switch (op) {
+        case THREADWORK_STATE_HANDSHAKE: {
+            ASSERT(!o->up)
+            ASSERT((b->flags & BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE))
+            
+            if (b->threadwork_result_sec == SECFailure) {
+                if (b->threadwork_error == PR_WOULD_BLOCK_ERROR) {
+                    if (io_ready) {
+                        // requested backend I/O got ready, try again
+                        backend_threadwork_start(o->backend, THREADWORK_STATE_HANDSHAKE);
+                    }
+                    return;
+                }
+                BLog(BLOG_ERROR, "SSL_ForceHandshake failed (%"PRIi32")", b->threadwork_error);
+                connection_report_error(o);
+                return;
+            }
+            
+            // init up
+            connection_init_up(o);
+            
+            // report up
+            o->handler(o->user, BSSLCONNECTION_EVENT_UP);
+            return;
+        } break;
+        
+        case THREADWORK_STATE_WRITE: {
+            ASSERT(o->up)
+            ASSERT((b->flags & BSSLCONNECTION_FLAG_THREADWORK_IO))
+            ASSERT(o->send_len > 0)
+            
+            PRInt32 result = b->threadwork_result_pr;
+            PRErrorCode error = b->threadwork_error;
+            
+            if (result < 0) {
+                if (error == PR_WOULD_BLOCK_ERROR) {
+                    if (io_ready) {
+                        // requested backend I/O got ready, try again
+                        backend_threadwork_start(o->backend, THREADWORK_STATE_WRITE);
+                    } else if (o->recv_avail > 0) {
+                        // don't forget about receiving
+                        backend_threadwork_start(o->backend, THREADWORK_STATE_READ);
+                    }
+                    return;
+                }
+                BLog(BLOG_ERROR, "PR_Write failed (%"PRIi32")", error);
+                connection_report_error(o);
+                return;
+            }
+            
+            ASSERT(result > 0)
+            ASSERT(result <= o->send_len)
+            
+            // set no send data
+            o->send_len = -1;
+            
+            // don't forget about receiving
+            if (o->recv_avail > 0) {
+                backend_threadwork_start(o->backend, THREADWORK_STATE_READ);
+            }
+            
+            // finish send operation
+            StreamPassInterface_Done(&o->send_if, result);
+        } break;
+        
+        case THREADWORK_STATE_READ: {
+            ASSERT(o->up)
+            ASSERT((b->flags & BSSLCONNECTION_FLAG_THREADWORK_IO))
+            ASSERT(o->recv_avail > 0)
+            
+            PRInt32 result = b->threadwork_result_pr;
+            PRErrorCode error = b->threadwork_error;
+            
+            if (result < 0) {
+                if (error == PR_WOULD_BLOCK_ERROR) {
+                    if (io_ready) {
+                        // requested backend I/O got ready, try again
+                        backend_threadwork_start(o->backend, THREADWORK_STATE_READ);
+                    } else if (o->send_len > 0) {
+                        // don't forget about sending
+                        backend_threadwork_start(o->backend, THREADWORK_STATE_WRITE);
+                    }
+                    return;
+                }
+                BLog(BLOG_ERROR, "PR_Read failed (%"PRIi32")", error);
+                connection_report_error(o);
+                return;
+            }
+            
+            if (result == 0) {
+                BLog(BLOG_ERROR, "PR_Read returned 0");
+                connection_report_error(o);
+                return;
+            }
+            
+            ASSERT(result > 0)
+            ASSERT(result <= o->recv_avail)
+            
+            // set no recv data
+            o->recv_avail = -1;
+            
+            // don't forget about sending
+            if (o->send_len > 0) {
+                backend_threadwork_start(o->backend, THREADWORK_STATE_WRITE);
+            }
+            
+            // finish receive operation
+            StreamRecvInterface_Done(&o->recv_if, result);
+        } break;
+        
+        default:
+            ASSERT(0);
+    }
+    
+    return;
+}
+
 static void connection_recv_job_handler (BSSLConnection *o)
 {
     DebugObject_Access(&o->d_obj);
@@ -415,12 +654,53 @@ static void connection_recv_job_handler (BSSLConnection *o)
     return;
 }
 
+static void connection_try_handshake (BSSLConnection *o)
+{
+    ASSERT(!o->have_error)
+    ASSERT(!o->up)
+    
+    // continue in threadwork if requested
+    if ((o->backend->flags & BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE)) {
+        if (o->backend->threadwork_state == THREADWORK_STATE_NONE) {
+            backend_threadwork_start(o->backend, THREADWORK_STATE_HANDSHAKE);
+        }
+        return;
+    }
+    
+    // try handshake
+    SECStatus res = SSL_ForceHandshake(o->prfd);
+    if (res == SECFailure) {
+        PRErrorCode error = PR_GetError();
+        if (error == PR_WOULD_BLOCK_ERROR) {
+            return;
+        }
+        BLog(BLOG_ERROR, "SSL_ForceHandshake failed (%"PRIi32")", error);
+        connection_report_error(o);
+        return;
+    }
+    
+    // init up
+    connection_init_up(o);
+    
+    // report up
+    o->handler(o->user, BSSLCONNECTION_EVENT_UP);
+    return;
+}
+
 static void connection_try_send (BSSLConnection *o)
 {
     ASSERT(!o->have_error)
     ASSERT(o->up)
     ASSERT(o->send_len > 0)
     
+    // continue in threadwork if requested
+    if ((o->backend->flags & BSSLCONNECTION_FLAG_THREADWORK_IO)) {
+        if (o->backend->threadwork_state == THREADWORK_STATE_NONE) {
+            backend_threadwork_start(o->backend, THREADWORK_STATE_WRITE);
+        }
+        return;
+    }
+    
     // send
     PRInt32 res = PR_Write(o->prfd, o->send_data, o->send_len);
     if (res < 0) {
@@ -428,9 +708,7 @@ static void connection_try_send (BSSLConnection *o)
         if (error == PR_WOULD_BLOCK_ERROR) {
             return;
         }
-        
         BLog(BLOG_ERROR, "PR_Write failed (%"PRIi32")", error);
-        
         connection_report_error(o);
         return;
     }
@@ -454,6 +732,14 @@ static void connection_try_recv (BSSLConnection *o)
     // unset recv job
     BPending_Unset(&o->recv_job);
     
+    // continue in threadwork if requested
+    if ((o->backend->flags & BSSLCONNECTION_FLAG_THREADWORK_IO)) {
+        if (o->backend->threadwork_state == THREADWORK_STATE_NONE) {
+            backend_threadwork_start(o->backend, THREADWORK_STATE_READ);
+        }
+        return;
+    }
+    
     // recv
     PRInt32 res = PR_Read(o->prfd, o->recv_data, o->recv_avail);
     if (res < 0) {
@@ -461,16 +747,13 @@ static void connection_try_recv (BSSLConnection *o)
         if (error == PR_WOULD_BLOCK_ERROR) {
             return;
         }
-        
         BLog(BLOG_ERROR, "PR_Read failed (%"PRIi32")", error);
-        
         connection_report_error(o);
         return;
     }
     
     if (res == 0) {
         BLog(BLOG_ERROR, "PR_Read returned 0");
-        
         connection_report_error(o);
         return;
     }
@@ -493,6 +776,11 @@ static void connection_send_if_handler_send (BSSLConnection *o, uint8_t *data, i
     ASSERT(o->send_len == -1)
     ASSERT(data_len > 0)
     
+#ifndef NDEBUG
+    ASSERT(!o->releasebuffers_called)
+    o->user_io_started = 1;
+#endif
+    
     // limit amount for PR_Write
     if (data_len > INT32_MAX) {
         data_len = INT32_MAX;
@@ -504,7 +792,6 @@ static void connection_send_if_handler_send (BSSLConnection *o, uint8_t *data, i
     
     // start sending
     connection_try_send(o);
-    return;
 }
 
 static void connection_recv_if_handler_recv (BSSLConnection *o, uint8_t *data, int data_len)
@@ -515,6 +802,11 @@ static void connection_recv_if_handler_recv (BSSLConnection *o, uint8_t *data, i
     ASSERT(o->recv_avail == -1)
     ASSERT(data_len > 0)
     
+#ifndef NDEBUG
+    ASSERT(!o->releasebuffers_called)
+    o->user_io_started = 1;
+#endif
+    
     // limit amount for PR_Read
     if (data_len > INT32_MAX) {
         data_len = INT32_MAX;
@@ -526,7 +818,6 @@ static void connection_recv_if_handler_recv (BSSLConnection *o, uint8_t *data, i
     
     // start receiving
     connection_try_recv(o);
-    return;
 }
 
 int BSSLConnection_GlobalInit (void)
@@ -543,20 +834,46 @@ int BSSLConnection_GlobalInit (void)
     return 1;
 }
 
-int BSSLConnection_MakeBackend (PRFileDesc *prfd, StreamPassInterface *send_if, StreamRecvInterface *recv_if)
+int BSSLConnection_MakeBackend (PRFileDesc *prfd, StreamPassInterface *send_if, StreamRecvInterface *recv_if, BThreadWorkDispatcher *twd, int flags)
 {
     ASSERT(bprconnection_initialized)
+    ASSERT(!(flags & ~(BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE | BSSLCONNECTION_FLAG_THREADWORK_IO)))
+    ASSERT(!(flags & BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE) || twd)
+    ASSERT(!(flags & BSSLCONNECTION_FLAG_THREADWORK_IO) || twd)
+    
+    // don't do stuff in threads if threads aren't available
+    if (((flags & BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE) || (flags & BSSLCONNECTION_FLAG_THREADWORK_IO)) &&
+        !BThreadWorkDispatcher_UsingThreads(twd)
+    ) {
+        BLog(BLOG_WARNING, "SSL operations in threads requested but threads are not available");
+        flags &= ~(BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE | BSSLCONNECTION_FLAG_THREADWORK_IO);
+    }
     
     // allocate backend
     struct BSSLConnection_backend *b = (struct BSSLConnection_backend *)malloc(sizeof(*b));
     if (!b) {
         BLog(BLOG_ERROR, "malloc failed");
-        return 0;
+        goto fail0;
+    }
+    
+    // init mutexes
+    if ((flags & BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE) || (flags & BSSLCONNECTION_FLAG_THREADWORK_IO)) {
+        if (!BMutex_Init(&b->send_buf_mutex)) {
+            BLog(BLOG_ERROR, "BMutex_Init failed");
+            goto fail1;
+        }
+        
+        if (!BMutex_Init(&b->recv_buf_mutex)) {
+            BLog(BLOG_ERROR, "BMutex_Init failed");
+            goto fail2;
+        }
     }
     
     // init arguments
     b->send_if = send_if;
     b->recv_if = recv_if;
+    b->twd = twd;
+    b->flags = flags;
     
     // init interfaces
     StreamPassInterface_Sender_Init(b->send_if, (StreamPassInterface_handler_done)backend_send_if_handler_done, b);
@@ -566,6 +883,7 @@ int BSSLConnection_MakeBackend (PRFileDesc *prfd, StreamPassInterface *send_if,
     b->con = NULL;
     
     // init send buffer
+    b->send_busy = 0;
     b->send_len = 0;
     b->send_pos = 0;
     
@@ -574,6 +892,9 @@ int BSSLConnection_MakeBackend (PRFileDesc *prfd, StreamPassInterface *send_if,
     b->recv_pos = 0;
     b->recv_len = 0;
     
+    // set threadwork state
+    b->threadwork_state = THREADWORK_STATE_NONE;
+    
     // init prfd
     memset(prfd, 0, sizeof(*prfd));
     prfd->methods = &methods;
@@ -581,6 +902,15 @@ int BSSLConnection_MakeBackend (PRFileDesc *prfd, StreamPassInterface *send_if,
     prfd->identity = bprconnection_identity;
     
     return 1;
+    
+    if ((flags & BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE) || (flags & BSSLCONNECTION_FLAG_THREADWORK_IO)) {
+fail2:
+        BMutex_Free(&b->send_buf_mutex);
+    }
+fail1:
+    free(b);
+fail0:
+    return 0;
 }
 
 void BSSLConnection_Init (BSSLConnection *o, PRFileDesc *prfd, int force_handshake, BPendingGroup *pg, void *user,
@@ -600,6 +930,8 @@ void BSSLConnection_Init (BSSLConnection *o, PRFileDesc *prfd, int force_handsha
     
     // set backend
     o->backend = (struct BSSLConnection_backend *)(get_bottom(prfd)->secret);
+    ASSERT(!o->backend->con)
+    ASSERT(o->backend->threadwork_state == THREADWORK_STATE_NONE)
     
     // set have no error
     o->have_error = 0;
@@ -621,6 +953,11 @@ void BSSLConnection_Init (BSSLConnection *o, PRFileDesc *prfd, int force_handsha
     // set backend connection
     o->backend->con = o;
     
+#ifndef NDEBUG
+    o->user_io_started = 0;
+    o->releasebuffers_called = 0;
+#endif
+    
     DebugError_Init(&o->d_err, o->pg);
     DebugObject_Init(&o->d_obj);
 }
@@ -629,6 +966,10 @@ void BSSLConnection_Free (BSSLConnection *o)
 {
     DebugObject_Free(&o->d_obj);
     DebugError_Free(&o->d_err);
+#ifndef NDEBUG
+    ASSERT(o->releasebuffers_called || !o->user_io_started)
+#endif
+    ASSERT(o->backend->threadwork_state == THREADWORK_STATE_NONE)
     
     if (o->up) {
         // free recv job
@@ -648,6 +989,24 @@ void BSSLConnection_Free (BSSLConnection *o)
     o->backend->con = NULL;
 }
 
+void BSSLConnection_ReleaseBuffers (BSSLConnection *o)
+{
+    DebugObject_Access(&o->d_obj);
+#ifndef NDEBUG
+    ASSERT(!o->releasebuffers_called)
+#endif
+    
+    // wait for threadwork to finish
+    if (o->backend->threadwork_state != THREADWORK_STATE_NONE) {
+        BThreadWork_Free(&o->backend->threadwork);
+        o->backend->threadwork_state = THREADWORK_STATE_NONE;
+    }
+    
+#ifndef NDEBUG
+    o->releasebuffers_called = 1;
+#endif
+}
+
 StreamPassInterface * BSSLConnection_GetSendIf (BSSLConnection *o)
 {
     DebugObject_Access(&o->d_obj);

nspr_support/BSSLConnection.h

@@ -31,6 +31,7 @@
 #define BADVPN_BSSLCONNECTION_H
 
 #include <prio.h>
+#include <ssl.h>
 
 #include <misc/debug.h>
 #include <misc/debugerror.h>
@@ -38,12 +39,17 @@
 #include <base/BPending.h>
 #include <flow/StreamPassInterface.h>
 #include <flow/StreamRecvInterface.h>
+#include <threadwork/BThreadWork.h>
+#include <threadwork/BMutex.h>
 
 #define BSSLCONNECTION_EVENT_UP 1
 #define BSSLCONNECTION_EVENT_ERROR 2
 
 #define BSSLCONNECTION_BUF_SIZE 4096
 
+#define BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE (1 << 0)
+#define BSSLCONNECTION_FLAG_THREADWORK_IO (1 << 1)
+
 typedef void (*BSSLConnection_handler) (void *user, int event);
 
 struct BSSLConnection_backend;
@@ -64,6 +70,10 @@ typedef struct {
     int send_len;
     uint8_t *recv_data;
     int recv_avail;
+#ifndef NDEBUG
+    int user_io_started;
+    int releasebuffers_called;
+#endif
     DebugError d_err;
     DebugObject d_obj;
 } BSSLConnection;
@@ -71,22 +81,35 @@ typedef struct {
 struct BSSLConnection_backend {
     StreamPassInterface *send_if;
     StreamRecvInterface *recv_if;
+    BThreadWorkDispatcher *twd;
+    int flags;
     BSSLConnection *con;
     uint8_t send_buf[BSSLCONNECTION_BUF_SIZE];
+    int send_busy;
     int send_pos;
     int send_len;
     uint8_t recv_buf[BSSLCONNECTION_BUF_SIZE];
     int recv_busy;
     int recv_pos;
     int recv_len;
+    int threadwork_state;
+    int threadwork_want_recv;
+    int threadwork_want_send;
+    BThreadWork threadwork;
+    SECStatus threadwork_result_sec;
+    PRInt32 threadwork_result_pr;
+    PRErrorCode threadwork_error;
+    BMutex send_buf_mutex;
+    BMutex recv_buf_mutex;
 };
 
 int BSSLConnection_GlobalInit (void) WARN_UNUSED;
-int BSSLConnection_MakeBackend (PRFileDesc *prfd, StreamPassInterface *send_if, StreamRecvInterface *recv_if) WARN_UNUSED;
+int BSSLConnection_MakeBackend (PRFileDesc *prfd, StreamPassInterface *send_if, StreamRecvInterface *recv_if, BThreadWorkDispatcher *twd, int flags) WARN_UNUSED;
 
 void BSSLConnection_Init (BSSLConnection *o, PRFileDesc *prfd, int force_handshake, BPendingGroup *pg, void *user,
                           BSSLConnection_handler handler);
 void BSSLConnection_Free (BSSLConnection *o);
+void BSSLConnection_ReleaseBuffers (BSSLConnection *o);
 StreamPassInterface * BSSLConnection_GetSendIf (BSSLConnection *o);
 StreamRecvInterface * BSSLConnection_GetRecvIf (BSSLConnection *o);
 

nspr_support/CMakeLists.txt

@@ -2,4 +2,4 @@ add_library(nspr_support
     DummyPRFileDesc.c
     BSSLConnection.c
 )
-target_link_libraries(nspr_support system flow ${NSPR_LIBRARIES} ${NSS_LIBRARIES})
+target_link_libraries(nspr_support system flow threadwork ${NSPR_LIBRARIES} ${NSS_LIBRARIES})

server/server.c

@@ -63,6 +63,7 @@
 #include <system/BNetwork.h>
 #include <security/BRandom.h>
 #include <nspr_support/DummyPRFileDesc.h>
+#include <threadwork/BThreadWork.h>
 
 #ifndef BADVPN_USE_WINAPI
 #include <base/BLog_syslog.h>
@@ -86,6 +87,9 @@ struct {
     #endif
     int loglevel;
     int loglevels[BLOG_NUM_CHANNELS];
+    int threads;
+    int use_threads_for_ssl_handshake;
+    int use_threads_for_ssl_data;
     int ssl;
     char *nssdb;
     char *server_cert_name;
@@ -134,6 +138,9 @@ BIPAddr relay_predicate_raddr;
 // i/o system
 BReactor ss;
 
+// thread work dispatcher
+BThreadWorkDispatcher twd;
+
 // server certificate if using SSL
 CERTCertificate *server_cert;
 
@@ -172,6 +179,8 @@ static int parse_arguments (int argc, char *argv[]);
 // processes certain command line options
 static int process_arguments (void);
 
+static int ssl_flags (void);
+
 // handler for program termination request
 static void signal_handler (void *unused);
 
@@ -376,6 +385,57 @@ int main (int argc, char *argv[])
     
     BLog(BLOG_NOTICE, "initializing "GLOBAL_PRODUCT_NAME" "PROGRAM_NAME" "GLOBAL_VERSION);
     
+    if (options.ssl) {
+        // initialize NSPR
+        PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0);
+        
+        // initialize i/o layer types
+        if (!DummyPRFileDesc_GlobalInit()) {
+            BLog(BLOG_ERROR, "DummyPRFileDesc_GlobalInit failed");
+            goto fail01;
+        }
+        if (!BSSLConnection_GlobalInit()) {
+            BLog(BLOG_ERROR, "BSSLConnection_GlobalInit failed");
+            goto fail01;
+        }
+        
+        // initialize NSS
+        if (NSS_Init(options.nssdb) != SECSuccess) {
+            BLog(BLOG_ERROR, "NSS_Init failed (%d)", (int)PR_GetError());
+            goto fail01;
+        }
+        if (NSS_SetDomesticPolicy() != SECSuccess) {
+            BLog(BLOG_ERROR, "NSS_SetDomesticPolicy failed (%d)", (int)PR_GetError());
+            goto fail02;
+        }
+        
+        // initialize server cache
+        if (SSL_ConfigServerSessionIDCache(0, 0, 0, NULL) != SECSuccess) {
+            BLog(BLOG_ERROR, "SSL_ConfigServerSessionIDCache failed (%d)", (int)PR_GetError());
+            goto fail02;
+        }
+        
+        // open server certificate and private key
+        if (!open_nss_cert_and_key(options.server_cert_name, &server_cert, &server_key)) {
+            BLog(BLOG_ERROR, "Cannot open certificate and key");
+            goto fail03;
+        }
+        
+        // initialize model SSL fd
+        DummyPRFileDesc_Create(&model_dprfd);
+        if (!(model_prfd = SSL_ImportFD(NULL, &model_dprfd))) {
+            BLog(BLOG_ERROR, "SSL_ImportFD failed");
+            ASSERT_FORCE(PR_Close(&model_dprfd) == PR_SUCCESS)
+            goto fail04;
+        }
+        
+        // set server certificate
+        if (SSL_ConfigSecureServer(model_prfd, server_cert, server_key, NSS_FindCertKEAType(server_cert)) != SECSuccess) {
+            BLog(BLOG_ERROR, "SSL_ConfigSecureServer failed");
+            goto fail05;
+        }
+    }
+    
     // initialize network
     if (!BNetwork_GlobalInit()) {
         BLog(BLOG_ERROR, "BNetwork_GlobalInit failed");
@@ -429,63 +489,18 @@ int main (int argc, char *argv[])
         goto fail3;
     }
     
+    // init thread work dispatcher
+    if (!BThreadWorkDispatcher_Init(&twd, &ss, options.threads)) {
+        BLog(BLOG_ERROR, "BThreadWorkDispatcher_Init failed");
+        goto fail3a;
+    }
+    
     // setup signal handler
     if (!BSignal_Init(&ss, signal_handler, NULL)) {
         BLog(BLOG_ERROR, "BSignal_Init failed");
         goto fail4;
     }
     
-    if (options.ssl) {
-        // initialize NSPR
-        PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0);
-        
-        // initialize i/o layer types
-        if (!DummyPRFileDesc_GlobalInit()) {
-            BLog(BLOG_ERROR, "DummyPRFileDesc_GlobalInit failed");
-            goto fail5;
-        }
-        if (!BSSLConnection_GlobalInit()) {
-            BLog(BLOG_ERROR, "BSSLConnection_GlobalInit failed");
-            goto fail5;
-        }
-        
-        // initialize NSS
-        if (NSS_Init(options.nssdb) != SECSuccess) {
-            BLog(BLOG_ERROR, "NSS_Init failed (%d)", (int)PR_GetError());
-            goto fail5;
-        }
-        if (NSS_SetDomesticPolicy() != SECSuccess) {
-            BLog(BLOG_ERROR, "NSS_SetDomesticPolicy failed (%d)", (int)PR_GetError());
-            goto fail6;
-        }
-        
-        // initialize server cache
-        if (SSL_ConfigServerSessionIDCache(0, 0, 0, NULL) != SECSuccess) {
-            BLog(BLOG_ERROR, "SSL_ConfigServerSessionIDCache failed (%d)", (int)PR_GetError());
-            goto fail6;
-        }
-        
-        // open server certificate and private key
-        if (!open_nss_cert_and_key(options.server_cert_name, &server_cert, &server_key)) {
-            BLog(BLOG_ERROR, "Cannot open certificate and key");
-            goto fail7;
-        }
-        
-        // initialize model SSL fd
-        DummyPRFileDesc_Create(&model_dprfd);
-        if (!(model_prfd = SSL_ImportFD(NULL, &model_dprfd))) {
-            BLog(BLOG_ERROR, "SSL_ImportFD failed");
-            ASSERT_FORCE(PR_Close(&model_dprfd) == PR_SUCCESS)
-            goto fail8;
-        }
-        
-        // set server certificate
-        if (SSL_ConfigSecureServer(model_prfd, server_cert, server_key, NSS_FindCertKEAType(server_cert)) != SECSuccess) {
-            BLog(BLOG_ERROR, "SSL_ConfigSecureServer failed");
-            goto fail9;
-        }
-    }
-    
     // initialize number of clients
     clients_num = 0;
     
@@ -553,23 +568,10 @@ fail10:
         BListener_Free(&listeners[num_listeners]);
     }
     
-    if (options.ssl) {
-fail9:
-        ASSERT_FORCE(PR_Close(model_prfd) == PR_SUCCESS)
-fail8:
-        CERT_DestroyCertificate(server_cert);
-        SECKEY_DestroyPrivateKey(server_key);
-fail7:
-        ASSERT_FORCE(SSL_ShutdownServerSessionIDCache() == SECSuccess)
-fail6:
-        ASSERT_FORCE(NSS_Shutdown() == SECSuccess)
-fail5:
-        ASSERT_FORCE(PR_Cleanup() == PR_SUCCESS)
-        PL_ArenaFinish();
-    }
-    
     BSignal_Finish();
 fail4:
+    BThreadWorkDispatcher_Free(&twd);
+fail3a:
     BReactor_Free(&ss);
 fail3:
     if (options.relay_predicate) {
@@ -588,6 +590,20 @@ fail2:
         BPredicate_Free(&comm_predicate);
     }
 fail1:
+    if (options.ssl) {
+fail05:
+        ASSERT_FORCE(PR_Close(model_prfd) == PR_SUCCESS)
+fail04:
+        CERT_DestroyCertificate(server_cert);
+        SECKEY_DestroyPrivateKey(server_key);
+fail03:
+        ASSERT_FORCE(SSL_ShutdownServerSessionIDCache() == SECSuccess)
+fail02:
+        ASSERT_FORCE(NSS_Shutdown() == SECSuccess)
+fail01:
+        ASSERT_FORCE(PR_Cleanup() == PR_SUCCESS)
+        PL_ArenaFinish();
+    }
     BLog(BLOG_NOTICE, "exiting");
     BLog_Free();
 fail0:
@@ -612,6 +628,9 @@ void print_help (const char *name)
         #endif
         "        [--loglevel <0-5/none/error/warning/notice/info/debug>]\n"
         "        [--channel-loglevel <channel-name> <0-5/none/error/warning/notice/info/debug>] ...\n"
+        "        [--threads <integer>]\n"
+        "        [--use-threads-for-ssl-handshake]\n"
+        "        [--use-threads-for-ssl-data]\n"
         "        [--listen-addr <addr>] ...\n"
         "        [--ssl --nssdb <string> --server-cert-name <string>]\n"
         "        [--comm-predicate <string>]\n"
@@ -641,6 +660,9 @@ int parse_arguments (int argc, char *argv[])
     for (int i = 0; i < BLOG_NUM_CHANNELS; i++) {
         options.loglevels[i] = -1;
     }
+    options.threads = 0;
+    options.use_threads_for_ssl_handshake = 0;
+    options.use_threads_for_ssl_data = 0;
     options.ssl = 0;
     options.nssdb = NULL;
     options.server_cert_name = NULL;
@@ -725,6 +747,20 @@ int parse_arguments (int argc, char *argv[])
             options.loglevels[channel] = loglevel;
             i += 2;
         }
+        else if (!strcmp(arg, "--threads")) {
+            if (1 >= argc - i) {
+                fprintf(stderr, "%s: requires an argument\n", arg);
+                return 0;
+            }
+            options.threads = atoi(argv[i + 1]);
+            i++;
+        }
+        else if (!strcmp(arg, "--use-threads-for-ssl-handshake")) {
+            options.use_threads_for_ssl_handshake = 1;
+        }
+        else if (!strcmp(arg, "--use-threads-for-ssl-data")) {
+            options.use_threads_for_ssl_data = 1;
+        }
         else if (!strcmp(arg, "--ssl")) {
             options.ssl = 1;
         }
@@ -833,6 +869,18 @@ int process_arguments (void)
     return 1;
 }
 
+int ssl_flags (void)
+{
+    int flags = 0;
+    if (options.use_threads_for_ssl_handshake) {
+        flags |= BSSLCONNECTION_FLAG_THREADWORK_HANDSHAKE;
+    }
+    if (options.use_threads_for_ssl_data) {
+        flags |= BSSLCONNECTION_FLAG_THREADWORK_IO;
+    }
+    return flags;
+}
+
 void signal_handler (void *unused)
 {
     BLog(BLOG_NOTICE, "termination requested");
@@ -882,7 +930,7 @@ void listener_handler (BListener *listener)
     
     if (options.ssl) {
         // create bottom NSPR file descriptor
-        if (!BSSLConnection_MakeBackend(&client->bottom_prfd, BConnection_SendAsync_GetIf(&client->con), BConnection_RecvAsync_GetIf(&client->con))) {
+        if (!BSSLConnection_MakeBackend(&client->bottom_prfd, BConnection_SendAsync_GetIf(&client->con), BConnection_RecvAsync_GetIf(&client->con), &twd, ssl_flags())) {
             client_log(client, BLOG_ERROR, "BSSLConnection_MakeBackend failed");
             goto fail2;
         }

server_connection/ServerConnection.c

@@ -29,6 +29,7 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <stddef.h>
 
 #include <misc/debug.h>
 #include <misc/strdup.h>
@@ -67,6 +68,7 @@ void connector_handler (ServerConnection *o, int is_error)
 {
     DebugObject_Access(&o->d_obj);
     ASSERT(o->state == STATE_CONNECTING)
+    ASSERT(!o->buffers_released)
     
     // check connection attempt result
     if (is_error) {
@@ -91,7 +93,7 @@ void connector_handler (ServerConnection *o, int is_error)
     
     if (o->have_ssl) {
         // create bottom NSPR file descriptor
-        if (!BSSLConnection_MakeBackend(&o->bottom_prfd, send_iface, recv_iface)) {
+        if (!BSSLConnection_MakeBackend(&o->bottom_prfd, send_iface, recv_iface, o->twd, o->ssl_flags)) {
             BLog(BLOG_ERROR, "BSSLConnection_MakeBackend failed");
             goto fail0a;
         }
@@ -211,6 +213,7 @@ fail0:
 void pending_handler (ServerConnection *o)
 {
     ASSERT(o->state == STATE_WAITINIT)
+    ASSERT(!o->buffers_released)
     DebugObject_Access(&o->d_obj);
     
     // send hello
@@ -251,6 +254,7 @@ void connection_handler (ServerConnection *o, int event)
 {
     DebugObject_Access(&o->d_obj);
     ASSERT(o->state >= STATE_WAITINIT)
+    ASSERT(!o->buffers_released)
     
     if (event == BCONNECTION_EVENT_RECVCLOSED) {
         BLog(BLOG_INFO, "connection closed");
@@ -267,6 +271,7 @@ void sslcon_handler (ServerConnection *o, int event)
     DebugObject_Access(&o->d_obj);
     ASSERT(o->have_ssl)
     ASSERT(o->state >= STATE_WAITINIT)
+    ASSERT(!o->buffers_released)
     ASSERT(event == BSSLCONNECTION_EVENT_ERROR)
     
     BLog(BLOG_ERROR, "SSL error");
@@ -279,6 +284,7 @@ void decoder_handler_error (ServerConnection *o)
 {
     DebugObject_Access(&o->d_obj);
     ASSERT(o->state >= STATE_WAITINIT)
+    ASSERT(!o->buffers_released)
     
     BLog(BLOG_ERROR, "decoder error");
     
@@ -289,6 +295,7 @@ void decoder_handler_error (ServerConnection *o)
 void input_handler_send (ServerConnection *o, uint8_t *data, int data_len)
 {
     ASSERT(o->state >= STATE_WAITINIT)
+    ASSERT(!o->buffers_released)
     ASSERT(data_len >= 0)
     ASSERT(data_len <= SC_MAX_ENC)
     DebugObject_Access(&o->d_obj);
@@ -487,10 +494,12 @@ void end_packet (ServerConnection *o, uint8_t type)
 int ServerConnection_Init (
     ServerConnection *o,
     BReactor *reactor,
+    BThreadWorkDispatcher *twd,
     BAddr addr,
     int keepalive_interval,
     int buffer_size,
     int have_ssl,
+    int ssl_flags,
     CERTCertificate *client_cert,
     SECKEYPrivateKey *client_key,
     const char *server_name,
@@ -509,10 +518,12 @@ int ServerConnection_Init (
     
     // init arguments
     o->reactor = reactor;
+    o->twd = twd;
     o->keepalive_interval = keepalive_interval;
     o->buffer_size = buffer_size;
     o->have_ssl = have_ssl;
     if (have_ssl) {
+        o->ssl_flags = ssl_flags;
         o->client_cert = client_cert;
         o->client_key = client_key;
     }
@@ -545,6 +556,7 @@ int ServerConnection_Init (
     
     // set state
     o->state = STATE_CONNECTING;
+    o->buffers_released = 0;
     
     DebugError_Init(&o->d_err, BReactor_PendingGroup(o->reactor));
     DebugObject_Init(&o->d_obj);
@@ -565,6 +577,11 @@ void ServerConnection_Free (ServerConnection *o)
         // allow freeing queue flows
         PacketPassPriorityQueue_PrepareFree(&o->output_queue);
         
+        // stop using any buffers before they get freed
+        if (o->have_ssl && !o->buffers_released) {
+            BSSLConnection_ReleaseBuffers(&o->sslcon);
+        }
+        
         // free output user flow
         PacketPassPriorityQueueFlow_Free(&o->output_user_qflow);
         
@@ -612,6 +629,18 @@ void ServerConnection_Free (ServerConnection *o)
     free(o->server_name);
 }
 
+void ServerConnection_ReleaseBuffers (ServerConnection *o)
+{
+    DebugObject_Access(&o->d_obj);
+    ASSERT(!o->buffers_released)
+    
+    if (o->state > STATE_CONNECTING && o->have_ssl) {
+        BSSLConnection_ReleaseBuffers(&o->sslcon);
+    }
+    
+    o->buffers_released = 1;
+}
+
 PacketPassInterface * ServerConnection_GetSendInterface (ServerConnection *o)
 {
     ASSERT(o->state == STATE_COMPLETE)

server_connection/ServerConnection.h

@@ -117,8 +117,9 @@ typedef void (*ServerConnection_handler_message) (void *user, peerid_t peer_id,
  * Object used to communicate with a VPN chat server.
  */
 typedef struct {
-    // reactor
+    // global resources
     BReactor *reactor;
+    BThreadWorkDispatcher *twd;
     
     // keepalive interval
     int keepalive_interval;
@@ -129,6 +130,9 @@ typedef struct {
     // whether we use SSL
     int have_ssl;
     
+    // ssl flags
+    int ssl_flags;
+    
     // client certificate if using SSL
     CERTCertificate *client_cert;
 
@@ -157,6 +161,7 @@ typedef struct {
     
     // state
     int state;
+    int buffers_released;
     
     // whether an error is being reported
     int error;
@@ -207,10 +212,14 @@ typedef struct {
  *
  * @param o the object
  * @param reactor {@link BReactor} we live in
+ * @param twd thread work dispatcher. May be NULL if ssl_flags does not request performing SSL
+ *            operations in threads.
  * @param addr address to connect to
  * @param keepalive_interval keep-alive sending interval. Must be >0.
  * @param buffer_size minimum size of send buffer in number of packets. Must be >0.
  * @param have_ssl whether to use SSL for connecting to the server. Must be 1 or 0.
+ * @param ssl_flags flags passed down to {@link BSSLConnection_MakeBackend}. May be used to
+ *                  request performing SSL operations in threads.
  * @param client_cert if using SSL, client certificate to use. Must remain valid as
  *                    long as this object is alive.
  * @param client_key if using SSL, prvate ket to use. Must remain valid as
@@ -229,10 +238,12 @@ typedef struct {
 int ServerConnection_Init (
     ServerConnection *o,
     BReactor *reactor,
+    BThreadWorkDispatcher *twd,
     BAddr addr,
     int keepalive_interval,
     int buffer_size,
     int have_ssl,
+    int ssl_flags,
     CERTCertificate *client_cert,
     SECKEYPrivateKey *client_key,
     const char *server_name,
@@ -246,11 +257,22 @@ int ServerConnection_Init (
 
 /**
  * Frees the object.
+ * {@link ServerConnection_ReleaseBuffers} must have been called if the
+ * send interface obtained from {@link ServerConnection_GetSendInterface}
+ * was used.
  *
  * @param o the object
  */
 void ServerConnection_Free (ServerConnection *o);
 
+/**
+ * Stops using any buffers passed to the send interface obtained from
+ * {@link ServerConnection_GetSendInterface}. If the send interface
+ * has been used, this must be called at appropriate time before this
+ * object is freed.
+ */
+void ServerConnection_ReleaseBuffers (ServerConnection *o);
+
 /**
  * Returns an interface for sending data to the server (just one).
  * This goes directly into the link (i.e. TCP, possibly via SSL), so packets