Make peers use SSL when talking through the server.

Update protocol and bump protocol version. Compatibility is preserved such that a network with a new 
server can have both new and old clients, provided new clients use the --allow-peer-talk-without-ssl 
option, reducing the security benefit.

blog_channels.txt

@@ -83,3 +83,4 @@ BNetwork 4
 BConnection 4
 BSSLConnection 4
 BDatagram 4
+PeerChat 4

client/PeerChat.c

@@ -22,29 +22,185 @@
 
 #include <string.h>
 
+#include <ssl.h>
+#include <sslerr.h>
+
 #include <misc/byteorder.h>
+#include <base/BLog.h>
+#include <security/BRandom.h>
 
 #include "PeerChat.h"
 
+#include <generated/blog_channel_PeerChat.h>
+
+static void report_error (PeerChat *o)
+{
+    DebugError_AssertNoError(&o->d_err);
+    
+    DEBUGERROR(&o->d_err, o->handler_error(o->user))
+    return;
+}
+
 static void recv_job_handler (PeerChat *o)
 {
     DebugObject_Access(&o->d_obj);
+    DebugError_AssertNoError(&o->d_err);
     ASSERT(o->recv_data_len >= 0)
+    ASSERT(o->recv_data_len <= SC_MAX_MSGLEN)
     
     int data_len = o->recv_data_len;
     
     // set no received data
     o->recv_data_len = -1;
     
+#ifdef PEERCHAT_SIMULATE_ERROR
+    uint8_t x;
+    BRandom_randomize(&x, sizeof(x));
+    if (x < PEERCHAT_SIMULATE_ERROR) {
+        BLog(BLOG_ERROR, "simulate error");
+        report_error(o);
+        return;
+    }
+#endif
+    
+    if (o->ssl_mode != PEERCHAT_SSL_NONE) {
+        // buffer data
+        if (!SimpleStreamBuffer_Write(&o->ssl_recv_buf, o->recv_data, data_len)) {
+            BLog(BLOG_ERROR, "out of recv buffer");
+            report_error(o);
+            return;
+        }
+    } else {
+        // call message handler
+        o->handler_message(o->user, o->recv_data, data_len);
+        return;
+    }
+}
+
+static void ssl_con_handler (PeerChat *o, int event)
+{
+    DebugObject_Access(&o->d_obj);
+    DebugError_AssertNoError(&o->d_err);
+    ASSERT(o->ssl_mode == PEERCHAT_SSL_CLIENT || o->ssl_mode == PEERCHAT_SSL_SERVER)
+    ASSERT(event == BSSLCONNECTION_EVENT_ERROR)
+    
+    BLog(BLOG_ERROR, "SSL error");
+    
+    report_error(o);
+    return;
+}
+
+static SECStatus client_auth_data_callback (PeerChat *o, PRFileDesc *fd, CERTDistNames *caNames, CERTCertificate **pRetCert, SECKEYPrivateKey **pRetKey)
+{
+    DebugObject_Access(&o->d_obj);
+    ASSERT(o->ssl_mode == PEERCHAT_SSL_CLIENT)
+    
+    CERTCertificate *cert = CERT_DupCertificate(o->ssl_cert);
+    if (!cert) {
+        BLog(BLOG_ERROR, "CERT_DupCertificate failed");
+        goto fail0;
+    }
+    
+    SECKEYPrivateKey *key = SECKEY_CopyPrivateKey(o->ssl_key);
+    if (!key) {
+        BLog(BLOG_ERROR, "SECKEY_CopyPrivateKey failed");
+        goto fail1;
+    }
+    
+    *pRetCert = cert;
+    *pRetKey = key;
+    return SECSuccess;
+    
+fail1:
+    CERT_DestroyCertificate(cert);
+fail0:
+    return SECFailure;
+}
+
+static SECStatus auth_certificate_callback (PeerChat *o, PRFileDesc *fd, PRBool checkSig, PRBool isServer)
+{
+    DebugObject_Access(&o->d_obj);
+    ASSERT(o->ssl_mode == PEERCHAT_SSL_CLIENT || o->ssl_mode == PEERCHAT_SSL_SERVER)
+    
+    // This callback is used to bypass checking the server's domain name, as peers
+    // don't have domain names. We byte-compare the certificate to the one reported
+    // by the server anyway.
+    
+    SECStatus ret = SECFailure;
+    
+    CERTCertificate *cert = SSL_PeerCertificate(o->ssl_prfd);
+    if (!cert) {
+        BLog(BLOG_ERROR, "SSL_PeerCertificate failed");
+        PORT_SetError(SSL_ERROR_BAD_CERTIFICATE);
+        goto fail1;
+    }
+    
+    SECCertUsage cert_usage = (o->ssl_mode == PEERCHAT_SSL_CLIENT ? certUsageSSLServer : certUsageSSLClient);
+    
+    if (CERT_VerifyCertNow(CERT_GetDefaultCertDB(), cert, PR_TRUE, cert_usage, SSL_RevealPinArg(o->ssl_prfd)) != SECSuccess) {
+        goto fail2;
+    }
+    
+    // compare to certificate provided by the server
+    SECItem der = cert->derCert;
+    if (der.len != o->ssl_peer_cert_len || memcmp(der.data, o->ssl_peer_cert, der.len)) {
+        BLog(BLOG_ERROR, "peer certificate doesn't match");
+        PORT_SetError(SSL_ERROR_BAD_CERTIFICATE);
+        goto fail2;
+    }
+    
+    ret = SECSuccess;
+    
+fail2:
+    CERT_DestroyCertificate(cert);
+fail1:
+    return ret;
+}
+
+static void ssl_recv_if_handler_send (PeerChat *o, uint8_t *data, int data_len)
+{
+    DebugObject_Access(&o->d_obj);
+    DebugError_AssertNoError(&o->d_err);
+    ASSERT(o->ssl_mode == PEERCHAT_SSL_CLIENT || o->ssl_mode == PEERCHAT_SSL_SERVER)
+    ASSERT(data_len >= 0)
+    ASSERT(data_len <= SC_MAX_MSGLEN)
+    
+    // accept packet
+    PacketPassInterface_Done(&o->ssl_recv_if);
+    
     // call message handler
-    o->handler_message(o->user, o->recv_data, data_len);
+    o->handler_message(o->user, data, data_len);
+    return;
+}
+
+static void ssl_recv_decoder_handler_error (PeerChat *o)
+{
+    DebugObject_Access(&o->d_obj);
+    DebugError_AssertNoError(&o->d_err);
+    ASSERT(o->ssl_mode == PEERCHAT_SSL_CLIENT || o->ssl_mode == PEERCHAT_SSL_SERVER)
+    
+    BLog(BLOG_ERROR, "decoder error");
+    
+    report_error(o);
     return;
 }
 
-int PeerChat_Init (PeerChat *o, peerid_t peer_id, BPendingGroup *pg, void *user, PeerChat_handler_error handler_error,
-                                                                                 PeerChat_handler_message handler_message)
+int PeerChat_Init (PeerChat *o, peerid_t peer_id, int ssl_mode, CERTCertificate *ssl_cert, SECKEYPrivateKey *ssl_key,
+                   uint8_t *ssl_peer_cert, int ssl_peer_cert_len, BPendingGroup *pg, void *user,
+                   PeerChat_handler_error handler_error,
+                   PeerChat_handler_message handler_message)
 {
+    ASSERT(ssl_mode == PEERCHAT_SSL_NONE || ssl_mode == PEERCHAT_SSL_CLIENT || ssl_mode == PEERCHAT_SSL_SERVER)
+    ASSERT(ssl_mode == PEERCHAT_SSL_NONE || ssl_peer_cert_len >= 0)
+    ASSERT(handler_error)
+    ASSERT(handler_message)
+    
     // init arguments
+    o->ssl_mode = ssl_mode;
+    o->ssl_cert = ssl_cert;
+    o->ssl_key = ssl_key;
+    o->ssl_peer_cert = ssl_peer_cert;
+    o->ssl_peer_cert_len = ssl_peer_cert_len;
     o->user = user;
     o->handler_error = handler_error;
     o->handler_message = handler_message;
@@ -58,16 +214,120 @@ int PeerChat_Init (PeerChat *o, peerid_t peer_id, BPendingGroup *pg, void *user,
     // init PacketProto encoder
     PacketProtoEncoder_Init(&o->pp_encoder, SCOutmsgEncoder_GetOutput(&o->sc_encoder), pg);
     
-    // init received job
+    // init recv job
     BPending_Init(&o->recv_job, pg, (BPending_handler)recv_job_handler, o);
     
     // set no received data
     o->recv_data_len = -1;
     
+    if (o->ssl_mode != PEERCHAT_SSL_NONE) {
+        // init receive buffer
+        if (!SimpleStreamBuffer_Init(&o->ssl_recv_buf, PEERCHAT_SSL_RECV_BUF_SIZE, pg)) {
+            BLog(BLOG_ERROR, "SimpleStreamBuffer_Init failed");
+            goto fail1;
+        }
+        
+        // init SSL StreamPacketSender
+        StreamPacketSender_Init(&o->ssl_sp_sender, PacketCopier_GetInput(&o->copier), pg);
+        
+        // init SSL bottom prfd
+        if (!BSSLConnection_MakeBackend(&o->ssl_bottom_prfd, StreamPacketSender_GetInput(&o->ssl_sp_sender), SimpleStreamBuffer_GetOutput(&o->ssl_recv_buf))) {
+            BLog(BLOG_ERROR, "BSSLConnection_MakeBackend failed");
+            goto fail2;
+        }
+        
+        // init SSL prfd
+        if (!(o->ssl_prfd = SSL_ImportFD(NULL, &o->ssl_bottom_prfd))) {
+            ASSERT_FORCE(PR_Close(&o->ssl_bottom_prfd) == PR_SUCCESS)
+            BLog(BLOG_ERROR, "SSL_ImportFD failed");
+            goto fail2;
+        }
+        
+        // set client or server mode
+        if (SSL_ResetHandshake(o->ssl_prfd, (o->ssl_mode == PEERCHAT_SSL_SERVER ? PR_TRUE : PR_FALSE)) != SECSuccess) {
+            BLog(BLOG_ERROR, "SSL_ResetHandshake failed");
+            goto fail3;
+        }
+        
+        if (o->ssl_mode == PEERCHAT_SSL_SERVER) {
+            // set server certificate
+            if (SSL_ConfigSecureServer(o->ssl_prfd, o->ssl_cert, o->ssl_key, NSS_FindCertKEAType(o->ssl_cert)) != SECSuccess) {
+                BLog(BLOG_ERROR, "SSL_ConfigSecureServer failed");
+                goto fail3;
+            }
+            
+            // set require client certificate
+            if (SSL_OptionSet(o->ssl_prfd, SSL_REQUEST_CERTIFICATE, PR_TRUE) != SECSuccess) {
+                BLog(BLOG_ERROR, "SSL_OptionSet(SSL_REQUEST_CERTIFICATE) failed");
+                goto fail3;
+            }
+            if (SSL_OptionSet(o->ssl_prfd, SSL_REQUIRE_CERTIFICATE, PR_TRUE) != SECSuccess) {
+                BLog(BLOG_ERROR, "SSL_OptionSet(SSL_REQUIRE_CERTIFICATE) failed");
+                goto fail3;
+            }
+        } else {
+            // set client certificate callback
+            if (SSL_GetClientAuthDataHook(o->ssl_prfd, (SSLGetClientAuthData)client_auth_data_callback, o) != SECSuccess) {
+                BLog(BLOG_ERROR, "SSL_GetClientAuthDataHook failed");
+                goto fail3;
+            }
+        }
+        
+        // set verify peer certificate hook
+        if (SSL_AuthCertificateHook(o->ssl_prfd, (SSLAuthCertificate)auth_certificate_callback, o) != SECSuccess) {
+            BLog(BLOG_ERROR, "SSL_AuthCertificateHook failed");
+            goto fail3;
+        }
+        
+        // init SSL connection
+        BSSLConnection_Init(&o->ssl_con, o->ssl_prfd, 0, pg, o, (BSSLConnection_handler)ssl_con_handler);
+        
+        // init SSL PacketStreamSender
+        PacketStreamSender_Init(&o->ssl_ps_sender, BSSLConnection_GetSendIf(&o->ssl_con), sizeof(struct packetproto_header) + SC_MAX_MSGLEN, pg);
+        
+        // init SSL copier
+        PacketCopier_Init(&o->ssl_copier, SC_MAX_MSGLEN, pg);
+        
+        // init SSL encoder
+        PacketProtoEncoder_Init(&o->ssl_encoder, PacketCopier_GetOutput(&o->ssl_copier), pg);
+        
+        // init SSL buffer
+        if (!SinglePacketBuffer_Init(&o->ssl_buffer, PacketProtoEncoder_GetOutput(&o->ssl_encoder), PacketStreamSender_GetInput(&o->ssl_ps_sender), pg)) {
+            BLog(BLOG_ERROR, "SinglePacketBuffer_Init failed");
+            goto fail4;
+        }
+        
+        // init receive interface
+        PacketPassInterface_Init(&o->ssl_recv_if, SC_MAX_MSGLEN, (PacketPassInterface_handler_send)ssl_recv_if_handler_send, o, pg);
+        
+        // init receive decoder
+        if (!PacketProtoDecoder_Init(&o->ssl_recv_decoder, BSSLConnection_GetRecvIf(&o->ssl_con), &o->ssl_recv_if, pg, o, (PacketProtoDecoder_handler_error)ssl_recv_decoder_handler_error)) {
+            BLog(BLOG_ERROR, "PacketProtoDecoder_Init failed");
+            goto fail5;
+        }
+    }
+    
+    DebugError_Init(&o->d_err, pg);
     DebugObject_Init(&o->d_obj);
     return 1;
     
+    if (o->ssl_mode != PEERCHAT_SSL_NONE) {
+fail5:
+        PacketPassInterface_Free(&o->ssl_recv_if);
+        SinglePacketBuffer_Free(&o->ssl_buffer);
+fail4:
+        PacketProtoEncoder_Free(&o->ssl_encoder);
+        PacketCopier_Free(&o->ssl_copier);
+        PacketStreamSender_Free(&o->ssl_ps_sender);
+        BSSLConnection_Free(&o->ssl_con);
+fail3:
+        ASSERT_FORCE(PR_Close(o->ssl_prfd) == PR_SUCCESS)
+fail2:
+        StreamPacketSender_Free(&o->ssl_sp_sender);
+        SimpleStreamBuffer_Free(&o->ssl_recv_buf);
+    }
 fail1:
+    BPending_Free(&o->recv_job);
     PacketProtoEncoder_Free(&o->pp_encoder);
     SCOutmsgEncoder_Free(&o->sc_encoder);
     PacketCopier_Free(&o->copier);
@@ -77,17 +337,23 @@ fail1:
 void PeerChat_Free (PeerChat *o)
 {
     DebugObject_Free(&o->d_obj);
+    DebugError_Free(&o->d_err);
     
-    // free received job
+    if (o->ssl_mode != PEERCHAT_SSL_NONE) {
+        PacketProtoDecoder_Free(&o->ssl_recv_decoder);
+        PacketPassInterface_Free(&o->ssl_recv_if);
+        SinglePacketBuffer_Free(&o->ssl_buffer);
+        PacketProtoEncoder_Free(&o->ssl_encoder);
+        PacketCopier_Free(&o->ssl_copier);
+        PacketStreamSender_Free(&o->ssl_ps_sender);
+        BSSLConnection_Free(&o->ssl_con);
+        ASSERT_FORCE(PR_Close(o->ssl_prfd) == PR_SUCCESS)
+        StreamPacketSender_Free(&o->ssl_sp_sender);
+        SimpleStreamBuffer_Free(&o->ssl_recv_buf);
+    }
     BPending_Free(&o->recv_job);
-    
-    // free PacketProto encoder
     PacketProtoEncoder_Free(&o->pp_encoder);
-    
-    // free SC encoder
     SCOutmsgEncoder_Free(&o->sc_encoder);
-    
-    // free copier
     PacketCopier_Free(&o->copier);
 }
 
@@ -95,7 +361,11 @@ PacketPassInterface * PeerChat_GetSendInput (PeerChat *o)
 {
     DebugObject_Access(&o->d_obj);
     
-    return PacketCopier_GetInput(&o->copier);
+    if (o->ssl_mode != PEERCHAT_SSL_NONE) {
+        return PacketCopier_GetInput(&o->ssl_copier);
+    } else {
+        return PacketCopier_GetInput(&o->copier);
+    }
 }
 
 PacketRecvInterface * PeerChat_GetSendOutput (PeerChat *o)
@@ -108,6 +378,7 @@ PacketRecvInterface * PeerChat_GetSendOutput (PeerChat *o)
 void PeerChat_InputReceived (PeerChat *o, uint8_t *data, int data_len)
 {
     DebugObject_Access(&o->d_obj);
+    DebugError_AssertNoError(&o->d_err);
     ASSERT(o->recv_data_len == -1)
     ASSERT(data_len >= 0)
     ASSERT(data_len <= SC_MAX_MSGLEN)

client/PeerChat.h

@@ -23,34 +23,80 @@
 #ifndef BADVPN_PEERCHAT_H
 #define BADVPN_PEERCHAT_H
 
+#include <cert.h>
+#include <keyhi.h>
+
 #include <protocol/packetproto.h>
 #include <protocol/scproto.h>
 #include <misc/debug.h>
+#include <misc/debugerror.h>
 #include <base/DebugObject.h>
 #include <base/BPending.h>
 #include <flow/SinglePacketSender.h>
 #include <flow/PacketProtoEncoder.h>
 #include <flow/PacketCopier.h>
+#include <flow/StreamPacketSender.h>
+#include <flow/PacketStreamSender.h>
+#include <flow/SinglePacketBuffer.h>
+#include <flow/PacketProtoDecoder.h>
+#include <nspr_support/BSSLConnection.h>
 #include <client/SCOutmsgEncoder.h>
+#include <client/SimpleStreamBuffer.h>
+
+#define PEERCHAT_SSL_NONE 0
+#define PEERCHAT_SSL_CLIENT 1
+#define PEERCHAT_SSL_SERVER 2
+
+#define PEERCHAT_SSL_RECV_BUF_SIZE 4096
+
+//#define PEERCHAT_SIMULATE_ERROR 40
 
 typedef void (*PeerChat_handler_error) (void *user);
 typedef void (*PeerChat_handler_message) (void *user, uint8_t *data, int data_len);
 
 typedef struct {
+    int ssl_mode;
+    CERTCertificate *ssl_cert;
+    SECKEYPrivateKey *ssl_key;
+    uint8_t *ssl_peer_cert;
+    int ssl_peer_cert_len;
     void *user;
     PeerChat_handler_error handler_error;
     PeerChat_handler_message handler_message;
+    
+    // common
     PacketProtoEncoder pp_encoder;
     SCOutmsgEncoder sc_encoder;
     PacketCopier copier;
     BPending recv_job;
     uint8_t *recv_data;
     int recv_data_len;
+    
+    // SSL transport
+    StreamPacketSender ssl_sp_sender;
+    SimpleStreamBuffer ssl_recv_buf;
+    
+    // SSL connection
+    PRFileDesc ssl_bottom_prfd;
+    PRFileDesc *ssl_prfd;
+    BSSLConnection ssl_con;
+    
+    // SSL higher layer
+    PacketStreamSender ssl_ps_sender;
+    SinglePacketBuffer ssl_buffer;
+    PacketProtoEncoder ssl_encoder;
+    PacketCopier ssl_copier;
+    PacketProtoDecoder ssl_recv_decoder;
+    PacketPassInterface ssl_recv_if;
+    
+    DebugError d_err;
     DebugObject d_obj;
 } PeerChat;
 
-int PeerChat_Init (PeerChat *o, peerid_t peer_id, BPendingGroup *pg, void *user, PeerChat_handler_error handler_error,
-                                                                                 PeerChat_handler_message handler_message) WARN_UNUSED;
+int PeerChat_Init (PeerChat *o, peerid_t peer_id, int ssl_mode, CERTCertificate *ssl_cert, SECKEYPrivateKey *ssl_key,
+                   uint8_t *ssl_peer_cert, int ssl_peer_cert_len, BPendingGroup *pg, void *user,
+                   PeerChat_handler_error handler_error,
+                   PeerChat_handler_message handler_message) WARN_UNUSED;
 void PeerChat_Free (PeerChat *o);
 PacketPassInterface * PeerChat_GetSendInput (PeerChat *o);
 PacketRecvInterface * PeerChat_GetSendOutput (PeerChat *o);

client/client.c

@@ -112,6 +112,7 @@ struct {
     int max_groups;
     int igmp_group_membership_interval;
     int igmp_last_member_query_time;
+    int allow_peer_talk_without_ssl;
 } options;
 
 // bind addresses
@@ -222,6 +223,8 @@ static void peer_log (struct peer_data *peer, int level, const char *fmt, ...);
 // see if we are the master relative to this peer
 static int peer_am_master (struct peer_data *peer);
 
+static void peer_free_chat (struct peer_data *peer);
+
 // initializes the link
 static int peer_init_link (struct peer_data *peer);
 
@@ -255,7 +258,8 @@ static void peer_unregister_need_relay (struct peer_data *peer);
 // handle a link setup failure
 static void peer_reset (struct peer_data *peer);
 
-// handle incoming peer messages
+// chat handlers
+static void peer_chat_handler_error (struct peer_data *peer);
 static void peer_chat_handler_message (struct peer_data *peer, uint8_t *data, int data_len);
 
 // handlers for different message types
@@ -334,10 +338,12 @@ static void server_handler_message (void *user, peerid_t peer_id, uint8_t *data,
 static void peer_job_send_seed_after_binding (struct peer_data *peer);
 static void peer_job_init (struct peer_data *peer);
 
-static struct server_flow * server_flow_init (peerid_t peer_id, PacketRecvInterface *input);
+static struct server_flow * server_flow_init (peerid_t peer_id);
 static void server_flow_free (struct server_flow *flow);
 static void server_flow_die (struct server_flow *flow);
 static void server_flow_qflow_handler_busy (struct server_flow *flow);
+static void server_flow_connect (struct server_flow *flow, PacketRecvInterface *input);
+static void server_flow_disconnect (struct server_flow *flow);
 
 int main (int argc, char *argv[])
 {
@@ -667,6 +673,7 @@ void print_help (const char *name)
         "        [--max-groups <num>]\n"
         "        [--igmp-group-membership-interval <ms>]\n"
         "        [--igmp-last-member-query-time <ms>]\n"
+        "        [--allow-peer-talk-without-ssl]\n"
         "Address format is a.b.c.d:port (IPv4) or [addr]:port (IPv6).\n",
         name
     );
@@ -716,6 +723,7 @@ int parse_arguments (int argc, char *argv[])
     options.max_groups = PEER_DEFAULT_MAX_GROUPS;
     options.igmp_group_membership_interval = DEFAULT_IGMP_GROUP_MEMBERSHIP_INTERVAL;
     options.igmp_last_member_query_time = DEFAULT_IGMP_LAST_MEMBER_QUERY_TIME;
+    options.allow_peer_talk_without_ssl = 0;
     
     int have_fragmentation_latency = 0;
     
@@ -1089,6 +1097,9 @@ int parse_arguments (int argc, char *argv[])
             }
             i++;
         }
+        else if (!strcmp(arg, "--allow-peer-talk-without-ssl")) {
+            options.allow_peer_talk_without_ssl = 1;
+        }
         else {
             fprintf(stderr, "unknown option: %s\n", arg);
             return 0;
@@ -1270,39 +1281,6 @@ void peer_add (peerid_t id, int flags, const uint8_t *cert, int cert_len)
     // set no common name
     peer->common_name = NULL;
     
-    // init jobs
-    BPending_Init(&peer->job_send_seed_after_binding, BReactor_PendingGroup(&ss), (BPending_handler)peer_job_send_seed_after_binding, peer);
-    BPending_Init(&peer->job_init, BReactor_PendingGroup(&ss), (BPending_handler)peer_job_init, peer);
-    
-    // set init job (must be before initing server flow so we can send)
-    BPending_Set(&peer->job_init);
-    
-    // init chat
-    if (!PeerChat_Init(&peer->chat, peer->id, BReactor_PendingGroup(&ss), peer, NULL,
-                                                                                (PeerChat_handler_message)peer_chat_handler_message
-    )) {
-        peer_log(peer, BLOG_ERROR, "PeerChat_Init failed");
-        goto fail1;
-    }
-    
-    // init server flow
-    if (!(peer->chat_send_flow = server_flow_init(peer->id, PeerChat_GetSendOutput(&peer->chat)))) {
-        peer_log(peer, BLOG_ERROR, "server_flow_init failed");
-        goto fail1a;
-    }
-    
-    // init chat send writer
-    BufferWriter_Init(&peer->chat_send_writer, SC_MAX_MSGLEN, BReactor_PendingGroup(&ss));
-    
-    // init chat send buffer
-    if (!PacketBuffer_Init(&peer->chat_send_buffer, BufferWriter_GetOutput(&peer->chat_send_writer), PeerChat_GetSendInput(&peer->chat), SERVER_BUFFER_MIN_PACKETS, BReactor_PendingGroup(&ss))) {
-        peer_log(peer, BLOG_ERROR, "PacketBuffer_Init failed");
-        goto fail1b;
-    }
-    
-    // set no message
-    peer->chat_send_msg_len = -1;
-    
     if (options.ssl) {
         // remember certificate
         memcpy(peer->cert, cert, cert_len);
@@ -1312,7 +1290,7 @@ void peer_add (peerid_t id, int flags, const uint8_t *cert, int cert_len)
         // in which case following workaroud wouldn't help
         if (!(cert_len > 0 && (cert[0] & 0x1f) == 0x10)) {
             peer_log(peer, BLOG_ERROR, "certificate does not look like DER");
-            goto fail1c;
+            goto fail1;
         }
         
         // copy the certificate and append it a good load of zero bytes,
@@ -1321,7 +1299,7 @@ void peer_add (peerid_t id, int flags, const uint8_t *cert, int cert_len)
         uint8_t *certbuf = malloc(cert_len + 100);
         if (!certbuf) {
             peer_log(peer, BLOG_ERROR, "malloc failed");
-            goto fail1c;
+            goto fail1;
         }
         memcpy(certbuf, cert, cert_len);
         memset(certbuf + cert_len, 0, 100);
@@ -1331,7 +1309,7 @@ void peer_add (peerid_t id, int flags, const uint8_t *cert, int cert_len)
         if (!nsscert) {
             peer_log(peer, BLOG_ERROR, "CERT_DecodeCertFromPackage failed (%d)", PORT_GetError());
             free(certbuf);
-            goto fail1c;
+            goto fail1;
         }
         
         free(certbuf);
@@ -1340,22 +1318,93 @@ void peer_add (peerid_t id, int flags, const uint8_t *cert, int cert_len)
         if (!(peer->common_name = CERT_GetCommonName(&nsscert->subject))) {
             peer_log(peer, BLOG_ERROR, "CERT_GetCommonName failed");
             CERT_DestroyCertificate(nsscert);
-            goto fail1c;
+            goto fail1;
         }
         
         CERT_DestroyCertificate(nsscert);
     }
     
+    // init jobs
+    BPending_Init(&peer->job_send_seed_after_binding, BReactor_PendingGroup(&ss), (BPending_handler)peer_job_send_seed_after_binding, peer);
+    BPending_Init(&peer->job_init, BReactor_PendingGroup(&ss), (BPending_handler)peer_job_init, peer);
+    
+    // set init job (must be before initing server flow so we can send)
+    BPending_Set(&peer->job_init);
+    
+    // init server flow
+    if (!(peer->server_flow = server_flow_init(peer->id))) {
+        peer_log(peer, BLOG_ERROR, "server_flow_init failed");
+        goto fail2;
+    }
+    
+    if ((peer->flags & SCID_NEWCLIENT_FLAG_SSL) && !options.ssl) {
+        peer_log(peer, BLOG_ERROR, "peer requires talking with SSL, but we're not using SSL!?");
+        goto fail3;
+    }
+    
+    if (!(peer->flags & SCID_NEWCLIENT_FLAG_SSL) && !options.allow_peer_talk_without_ssl) {
+        peer_log(peer, BLOG_ERROR, "peer requires talking without SSL, but we don't allow that");
+        goto fail3;
+    }
+    
+    // choose chat SSL mode
+    int chat_ssl_mode = PEERCHAT_SSL_NONE;
+    if ((peer->flags & SCID_NEWCLIENT_FLAG_SSL)) {
+        chat_ssl_mode = (peer_am_master(peer) ? PEERCHAT_SSL_SERVER : PEERCHAT_SSL_CLIENT);
+    }
+    
+    switch (chat_ssl_mode) {
+        case PEERCHAT_SSL_NONE:
+            peer_log(peer, BLOG_INFO, "talking to peer in plaintext mode");
+            break;
+        case PEERCHAT_SSL_CLIENT:
+            peer_log(peer, BLOG_INFO, "talking to peer in SSL client mode");
+            break;
+        case PEERCHAT_SSL_SERVER:
+            peer_log(peer, BLOG_INFO, "talking to peer in SSL server mode");
+            break;
+    }
+    
+    // init chat
+    if (!PeerChat_Init(&peer->chat, peer->id, chat_ssl_mode, client_cert, client_key, peer->cert, peer->cert_len, BReactor_PendingGroup(&ss), peer,
+        (PeerChat_handler_error)peer_chat_handler_error,
+        (PeerChat_handler_message)peer_chat_handler_message
+    )) {
+        peer_log(peer, BLOG_ERROR, "PeerChat_Init failed");
+        goto fail3;
+    }
+    
+    // init chat send writer
+    BufferWriter_Init(&peer->chat_send_writer, SC_MAX_MSGLEN, BReactor_PendingGroup(&ss));
+    
+    // init chat send buffer
+    if (!PacketBuffer_Init(&peer->chat_send_buffer, BufferWriter_GetOutput(&peer->chat_send_writer), PeerChat_GetSendInput(&peer->chat), SERVER_BUFFER_MIN_PACKETS, BReactor_PendingGroup(&ss))) {
+        peer_log(peer, BLOG_ERROR, "PacketBuffer_Init failed");
+        goto fail4;
+    }
+    
+    // set no message
+    peer->chat_send_msg_len = -1;
+    
+    // connect server flow to chat
+    server_flow_connect(peer->server_flow, PeerChat_GetSendOutput(&peer->chat));
+    
+    // set have chat
+    peer->have_chat = 1;
+    
+    // set have no resetpeer
+    peer->have_resetpeer = 0;
+    
     // init local flow
     if (!DataProtoFlow_Init(&peer->local_dpflow, &device_input_dpd, my_id, peer->id, options.send_buffer_size, -1, NULL, NULL)) {
         peer_log(peer, BLOG_ERROR, "DataProtoFlow_Init failed");
-        goto fail2;
+        goto fail5;
     }
     
     // init frame decider peer
     if (!FrameDeciderPeer_Init(&peer->decider_peer, &frame_decider)) {
         peer_log(peer, BLOG_ERROR, "FrameDeciderPeer_Init failed");
-        goto fail3;
+        goto fail6;
     }
     
     // init receive peer
@@ -1387,22 +1436,23 @@ void peer_add (peerid_t id, int flags, const uint8_t *cert, int cert_len)
     
     return;
     
-fail3:
+fail6:
     DataProtoFlow_Free(&peer->local_dpflow);
-fail2:
-    if (peer->common_name) {
-        PORT_Free(peer->common_name);
-    }
-fail1c:
+fail5:
+    server_flow_disconnect(peer->server_flow);
     PacketBuffer_Free(&peer->chat_send_buffer);
-fail1b:
+fail4:
     BufferWriter_Free(&peer->chat_send_writer);
-    server_flow_free(peer->chat_send_flow);
-fail1a:
     PeerChat_Free(&peer->chat);
-fail1:
+fail3:
+    server_flow_free(peer->server_flow);
+fail2:
     BPending_Free(&peer->job_init);
     BPending_Free(&peer->job_send_seed_after_binding);
+    if (peer->common_name) {
+        PORT_Free(peer->common_name);
+    }
+fail1:
     free(peer);
 fail0:;
 }
@@ -1446,22 +1496,27 @@ void peer_remove (struct peer_data *peer, int exiting)
     // free local flow
     DataProtoFlow_Free(&peer->local_dpflow);
     
-    // free chat send buffer
-    PacketBuffer_Free(&peer->chat_send_buffer);
+    // free chat
+    if (peer->have_chat) {
+        peer_free_chat(peer);
+    }
     
-    // free chat send writer
-    BufferWriter_Free(&peer->chat_send_writer);
+    // free resetpeer
+    if (peer->have_resetpeer) {
+        // disconnect resetpeer source from server flow
+        server_flow_disconnect(peer->server_flow);
+        
+        // free resetpeer source
+        SinglePacketSource_Free(&peer->resetpeer_source);
+    }
     
     // free/die server flow
-    if (exiting || !PacketPassFairQueueFlow_IsBusy(&peer->chat_send_flow->qflow)) {
-        server_flow_free(peer->chat_send_flow);
+    if (exiting || !PacketPassFairQueueFlow_IsBusy(&peer->server_flow->qflow)) {
+        server_flow_free(peer->server_flow);
     } else {
-        server_flow_die(peer->chat_send_flow);
+        server_flow_die(peer->server_flow);
     }
     
-    // free chat
-    PeerChat_Free(&peer->chat);
-    
     // free jobs
     BPending_Free(&peer->job_init);
     BPending_Free(&peer->job_send_seed_after_binding);
@@ -1493,6 +1548,26 @@ int peer_am_master (struct peer_data *peer)
     return (my_id > peer->id);
 }
 
+void peer_free_chat (struct peer_data *peer)
+{
+    ASSERT(peer->have_chat)
+    
+    // disconnect chat from server flow
+    server_flow_disconnect(peer->server_flow);
+    
+    // free chat send buffer
+    PacketBuffer_Free(&peer->chat_send_buffer);
+    
+    // free chat send writer
+    BufferWriter_Free(&peer->chat_send_writer);
+    
+    // free chat
+    PeerChat_Free(&peer->chat);
+    
+    // set have no chat
+    peer->have_chat = 0;
+}
+
 int peer_init_link (struct peer_data *peer)
 {
     ASSERT(!peer->have_link)
@@ -1794,8 +1869,37 @@ void peer_reset (struct peer_data *peer)
     }
 }
 
+void peer_chat_handler_error (struct peer_data *peer)
+{
+    ASSERT(peer->have_chat)
+    ASSERT(!peer->have_resetpeer)
+    
+    peer_log(peer, BLOG_ERROR, "chat error, resetting peer");
+    
+    // free chat
+    peer_free_chat(peer);
+    
+    // build resetpeer packet
+    struct packetproto_header *pp_header = (struct packetproto_header *)peer->resetpeer_packet;
+    pp_header->len = htol16(sizeof(struct sc_header) + sizeof(struct sc_client_resetpeer));
+    struct sc_header *sc_header = (struct sc_header *)(pp_header + 1);
+    sc_header->type = htol8(SCID_RESETPEER);
+    struct sc_client_resetpeer *sc_resetpeer = (struct sc_client_resetpeer *)(sc_header + 1);
+    sc_resetpeer->clientid = htol16(peer->id);
+    
+    // init resetpeer sourse
+    SinglePacketSource_Init(&peer->resetpeer_source, peer->resetpeer_packet, sizeof(peer->resetpeer_packet), BReactor_PendingGroup(&ss));
+    
+    // connect server flow to resetpeer source
+    server_flow_connect(peer->server_flow, SinglePacketSource_GetOutput(&peer->resetpeer_source));
+    
+    // set have resetpeer
+    peer->have_resetpeer = 1;
+}
+
 void peer_chat_handler_message (struct peer_data *peer, uint8_t *data, int data_len)
 {
+    ASSERT(peer->have_chat)
     ASSERT(data_len >= 0)
     ASSERT(data_len <= SC_MAX_MSGLEN)
     
@@ -2291,12 +2395,17 @@ static int peer_start_msg (struct peer_data *peer, void **data, int type, int le
     ASSERT(!(len > 0) || data)
     ASSERT(peer->chat_send_msg_len == -1)
     
+    // make sure we have chat
+    if (!peer->have_chat) {
+        peer_log(peer, BLOG_ERROR, "cannot send message, chat is down");
+        return 0;
+    }
+    
     // obtain buffer location
     uint8_t *packet;
     if (!BufferWriter_StartPacket(&peer->chat_send_writer, &packet)) {
-        BLog(BLOG_ERROR, "out of peer server buffer, exiting");
-        terminate();
-        return -1;
+        peer_log(peer, BLOG_ERROR, "cannot send message, out of buffer");
+        return 0;
     }
     
     // write fields
@@ -2312,12 +2421,13 @@ static int peer_start_msg (struct peer_data *peer, void **data, int type, int le
     if (data) {
         *data = payload_dst;
     }
-    return 0;
+    return 1;
 }
 
 static void peer_end_msg (struct peer_data *peer)
 {
     ASSERT(peer->chat_send_msg_len >= 0)
+    ASSERT(peer->have_chat)
     
     // submit packet to buffer
     BufferWriter_EndPacket(&peer->chat_send_writer, msg_SIZEtype + msg_SIZEpayload(peer->chat_send_msg_len));
@@ -2328,7 +2438,7 @@ static void peer_end_msg (struct peer_data *peer)
 
 void peer_send_simple (struct peer_data *peer, int msgid)
 {
-    if (peer_start_msg(peer, NULL, msgid, 0) < 0) {
+    if (!peer_start_msg(peer, NULL, msgid, 0)) {
         return;
     }
     peer_end_msg(peer);
@@ -2378,7 +2488,7 @@ void peer_send_conectinfo (struct peer_data *peer, int addr_index, int port_adju
         
     // start message
     uint8_t *msg;
-    if (peer_start_msg(peer, (void **)&msg, MSGID_YOUCONNECT, msg_len) < 0) {
+    if (!peer_start_msg(peer, (void **)&msg, MSGID_YOUCONNECT, msg_len)) {
         return;
     }
         
@@ -2457,7 +2567,7 @@ void peer_generate_and_send_seed (struct peer_data *peer)
     // send seed to the peer
     int msg_len = msg_seed_SIZEseed_id + msg_seed_SIZEkey(key_len) + msg_seed_SIZEiv(iv_len);
     uint8_t *msg;
-    if (peer_start_msg(peer, (void **)&msg, MSGID_SEED, msg_len) < 0) {
+    if (!peer_start_msg(peer, (void **)&msg, MSGID_SEED, msg_len)) {
         return;
     }
     msg_seedWriter writer;
@@ -2479,7 +2589,7 @@ void peer_send_confirmseed (struct peer_data *peer, uint16_t seed_id)
     // send confirmation
     int msg_len = msg_confirmseed_SIZEseed_id;
     uint8_t *msg;
-    if (peer_start_msg(peer, (void **)&msg, MSGID_CONFIRMSEED, msg_len) < 0) {
+    if (!peer_start_msg(peer, (void **)&msg, MSGID_CONFIRMSEED, msg_len)) {
         return;
     }
     msg_confirmseedWriter writer;
@@ -2695,6 +2805,12 @@ void server_handler_message (void *user, peerid_t peer_id, uint8_t *data, int da
         return;
     }
     
+    // make sure we have chat
+    if (!peer->have_chat) {
+        peer_log(peer, BLOG_ERROR, "cannot process message, chat is down");
+        return;
+    }
+    
     // pass message to chat
     PeerChat_InputReceived(&peer->chat, data, data_len);
 }
@@ -2717,7 +2833,7 @@ void peer_job_init (struct peer_data *peer)
     }
 }
 
-struct server_flow * server_flow_init (peerid_t peer_id, PacketRecvInterface *input)
+struct server_flow * server_flow_init (peerid_t peer_id)
 {
     ASSERT(server_ready)
     
@@ -2740,8 +2856,8 @@ struct server_flow * server_flow_init (peerid_t peer_id, PacketRecvInterface *in
         goto fail1;
     }
     
-    // connect input
-    PacketRecvConnector_ConnectInput(&flow->connector, input);
+    // set not connected
+    flow->connected = 0;
     
     return flow;
     
@@ -2756,6 +2872,7 @@ fail0:
 void server_flow_free (struct server_flow *flow)
 {
     PacketPassFairQueueFlow_AssertFree(&flow->qflow);
+    ASSERT(!flow->connected)
     
     // remove dying flow reference
     if (flow == dying_server_flow) {
@@ -2778,11 +2895,9 @@ void server_flow_free (struct server_flow *flow)
 void server_flow_die (struct server_flow *flow)
 {
     ASSERT(PacketPassFairQueueFlow_IsBusy(&flow->qflow))
+    ASSERT(!flow->connected)
     ASSERT(!dying_server_flow)
     
-    // disconnect input
-    PacketRecvConnector_DisconnectInput(&flow->connector);
-    
     // request notification when flow is done
     PacketPassFairQueueFlow_SetBusyHandler(&flow->qflow, (PacketPassFairQueue_handler_busy)server_flow_qflow_handler_busy, flow);
     
@@ -2793,8 +2908,33 @@ void server_flow_die (struct server_flow *flow)
 void server_flow_qflow_handler_busy (struct server_flow *flow)
 {
     ASSERT(flow == dying_server_flow)
+    ASSERT(!flow->connected)
     PacketPassFairQueueFlow_AssertFree(&flow->qflow);
     
     // finally free flow
     server_flow_free(flow);
 }
+
+void server_flow_connect (struct server_flow *flow, PacketRecvInterface *input)
+{
+    ASSERT(!flow->connected)
+    ASSERT(flow != dying_server_flow)
+    
+    // connect input
+    PacketRecvConnector_ConnectInput(&flow->connector, input);
+    
+    // set connected
+    flow->connected = 1;
+}
+
+void server_flow_disconnect (struct server_flow *flow)
+{
+    ASSERT(flow->connected)
+    ASSERT(flow != dying_server_flow)
+    
+    // disconnect input
+    PacketRecvConnector_DisconnectInput(&flow->connector);
+    
+    // set not connected
+    flow->connected = 0;
+}

client/client.h

@@ -36,6 +36,7 @@
 #include <client/DPReceive.h>
 #include <client/FrameDecider.h>
 #include <client/PeerChat.h>
+#include <client/SinglePacketSource.h>
 
 // NOTE: all time values are in milliseconds
 
@@ -91,6 +92,7 @@ struct server_flow {
     PacketPassFairQueueFlow qflow;
     SinglePacketBuffer encoder_buffer;
     PacketRecvConnector connector;
+    int connected;
 };
 
 struct peer_data {
@@ -109,15 +111,21 @@ struct peer_data {
     BPending job_send_seed_after_binding;
     BPending job_init;
     
+    // server flow
+    struct server_flow *server_flow;
+    
     // chat
+    int have_chat;
     PeerChat chat;
-    
-    // chat sending
-    struct server_flow *chat_send_flow;
     PacketBuffer chat_send_buffer;
     BufferWriter chat_send_writer;
     int chat_send_msg_len;
     
+    // resetpeer source (when chat fails)
+    int have_resetpeer;
+    uint8_t resetpeer_packet[sizeof(struct packetproto_header) + sizeof(struct sc_header) + sizeof(struct sc_client_resetpeer)];
+    SinglePacketSource resetpeer_source;
+    
     // local flow
     DataProtoFlow local_dpflow;
     

generated/blog_channel_PeerChat.h

@@ -0,0 +1,4 @@
+#ifdef BLOG_CURRENT_CHANNEL
+#undef BLOG_CURRENT_CHANNEL
+#endif
+#define BLOG_CURRENT_CHANNEL BLOG_CHANNEL_PeerChat

generated/blog_channels_defines.h

@@ -83,4 +83,5 @@
 #define BLOG_CHANNEL_BConnection 82
 #define BLOG_CHANNEL_BSSLConnection 83
 #define BLOG_CHANNEL_BDatagram 84
-#define BLOG_NUM_CHANNELS 85
+#define BLOG_CHANNEL_PeerChat 85
+#define BLOG_NUM_CHANNELS 86

generated/blog_channels_list.h

@@ -83,3 +83,4 @@
 {.name = "BConnection", .loglevel = 4},
 {.name = "BSSLConnection", .loglevel = 4},
 {.name = "BDatagram", .loglevel = 4},
+{.name = "PeerChat", .loglevel = 4},

protocol/scproto.h

@@ -54,8 +54,9 @@
 
 #include <stdint.h>
 
-#define SC_VERSION 27
-#define SC_OLDVERSION 26
+#define SC_VERSION 28
+#define SC_OLDVERSION_NOSSL 27
+#define SC_OLDVERSION_BROKENCERT 26
 
 #define SC_KEEPALIVE_INTERVAL 10000
 
@@ -135,12 +136,15 @@ struct sc_server_newclient {
      *     You can relay frames to other peers through this peer.
      *   - SCID_NEWCLIENT_FLAG_RELAY_CLIENT
      *     You must allow this peer to relay frames to other peers through you.
+     *   - SCID_NEWCLIENT_FLAG_SSL
+     *     SSL must be used to talk to this peer through messages.
      */
     uint16_t flags;
 } __attribute__((packed));
 
 #define SCID_NEWCLIENT_FLAG_RELAY_SERVER 1
 #define SCID_NEWCLIENT_FLAG_RELAY_CLIENT 2
+#define SCID_NEWCLIENT_FLAG_SSL 4
 
 #define SCID_NEWCLIENT_MAX_CERT_LEN (SC_MAX_PAYLOAD - sizeof(struct sc_server_newclient))
 

server/server.c

@@ -1313,12 +1313,15 @@ int client_send_newclient (struct client_data *client, struct client_data *nc, i
     if (relay_client) {
         flags |= SCID_NEWCLIENT_FLAG_RELAY_CLIENT;
     }
+    if (options.ssl && client->version > SC_OLDVERSION_NOSSL && nc->version > SC_OLDVERSION_NOSSL) {
+        flags |= SCID_NEWCLIENT_FLAG_SSL;
+    }
     
     uint8_t *cert_data = NULL;
     int cert_len = 0;
     if (options.ssl) {
-        cert_data = (client->version == SC_OLDVERSION ?  nc->cert_old : nc->cert);
-        cert_len = (client->version == SC_OLDVERSION ?  nc->cert_old_len : nc->cert_len);
+        cert_data = (client->version == SC_OLDVERSION_BROKENCERT ?  nc->cert_old : nc->cert);
+        cert_len = (client->version == SC_OLDVERSION_BROKENCERT ?  nc->cert_old_len : nc->cert_len);
     }
     
     struct sc_server_newclient *pack;
@@ -1413,10 +1416,15 @@ void process_packet_hello (struct client_data *client, uint8_t *data, int data_l
     struct sc_client_hello *msg = (struct sc_client_hello *)data;
     client->version = ltoh16(msg->version);
     
-    if (client->version != SC_VERSION && client->version != SC_OLDVERSION) {
-        client_log(client, BLOG_NOTICE, "hello: unknown version");
-        client_remove(client);
-        return;
+    switch (client->version) {
+        case SC_VERSION:
+        case SC_OLDVERSION_NOSSL:
+        case SC_OLDVERSION_BROKENCERT:
+            break;
+        default:
+            client_log(client, BLOG_ERROR, "hello: unknown version (%d)", client->version);
+            client_remove(client);
+            return;
     }
     
     client_log(client, BLOG_INFO, "received hello");