Introduce a brand new design based on a LIFO list of jobs pending execution:

 
- Change the BPending system to work in LIFO mode (rather than FIFO).
- Modify I/O interfaces ({Packet,Stream}{Pass,Recv}Interface) to push I/O calls (Send/Recv, Done) to the queue, instead of calling the handlers directly.
- Port everything to work with the new design.
- Minor beautifications all around as part of the porting.

This new design fits very well into the data flow system. It greatly simplifies writing I/O modules by making I/O programming more declarative.
For instance:
- No more explicit checking what happened as a result of the I/O operation (e.g. was the object freed?) and during it (can it call this function from inside?).
- There is no longer a clear distinction between an I/O operation completing immediately or having to wait for its completion (read on for explanation).

Each time control returns to the reactor, the reactor pops a job off the top of the job list and executes it. A job may push further jobs while it's executing.
The popping of some job from the list and pushing of new jobs from this job's handler can be seen as a transformation of a job into other jobs. Let's
call these jobs the "transformation" of a job. And let's the job, its transformation, transformations of the transformation, and so on recursively, the
"transformation closure"
of a job.

Now suppose a Send operation is pushed to the job queue.
The operation can be considered completed immediately if its transformation closure contains the corresponding Done job.
Similarly, an operation can be considered blocked on external I/O if the job list empties before the Done job is pushed.
(note this leaves a third possibility...)

On the other hand, this design requires changes in the client and server programs, because it becomes impossible to send more than one packet
to a send buffer directly in response to a single input. To, for instance, send two packets into into the same send buffer in response to an input,
you need to first schedule sending the second one, then send the first one.

client/DataProto.c

@@ -39,8 +39,7 @@
 struct dp_relay_flow {
     DataProtoRelaySource *rs;
     DataProtoDest *dp;
-    BestEffortPacketWriteInterface *ainput_if;
-    PacketBufferAsyncInput ainput;
+    BufferWriter ainput;
     PacketBuffer buffer;
     PacketPassInactivityMonitor monitor;
     PacketPassFairQueueFlow qflow;
@@ -52,10 +51,10 @@ struct dp_relay_flow {
 static int peerid_comparator (void *user, peerid_t *val1, peerid_t *val2);
 static struct dp_relay_flow * create_relay_flow (DataProtoRelaySource *rs, DataProtoDest *dp, int num_packets);
 static void dealloc_relay_flow (struct dp_relay_flow *flow);
-static int release_relay_flow (struct dp_relay_flow *flow);
+static void release_relay_flow (struct dp_relay_flow *flow);
 static void flow_monitor_handler (struct dp_relay_flow *flow);
 static void monitor_handler (DataProtoDest *o);
-static int send_keepalive (DataProtoDest *o);
+static void send_keepalive (DataProtoDest *o);
 static void receive_timer_handler (DataProtoDest *o);
 static void notifier_handler (DataProtoDest *o, uint8_t *data, int data_len);
 static int pointer_comparator (void *user, void **val1, void **val2);
@@ -77,7 +76,6 @@ struct dp_relay_flow * create_relay_flow (DataProtoRelaySource *rs, DataProtoDes
     ASSERT(!BAVL_LookupExact(&rs->relay_flows_tree, &dp))
     ASSERT(num_packets > 0)
     ASSERT(!dp->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(dp->d_output))
     
     // allocate flow structure
     struct dp_relay_flow *flow = malloc(sizeof(struct dp_relay_flow));
@@ -97,11 +95,10 @@ struct dp_relay_flow * create_relay_flow (DataProtoRelaySource *rs, DataProtoDes
     PacketPassInactivityMonitor_Init(&flow->monitor, PacketPassFairQueueFlow_GetInput(&flow->qflow), dp->reactor, DATAPROTO_TIMEOUT, (PacketPassInactivityMonitor_handler)flow_monitor_handler, flow);
     
     // init async input
-    PacketBufferAsyncInput_Init(&flow->ainput, dp->mtu);
-    flow->ainput_if = PacketBufferAsyncInput_GetInput(&flow->ainput);
+    BufferWriter_Init(&flow->ainput, dp->mtu, BReactor_PendingGroup(dp->reactor));
     
     // init buffer
-    if (!PacketBuffer_Init(&flow->buffer, PacketBufferAsyncInput_GetOutput(&flow->ainput), PacketPassInactivityMonitor_GetInput(&flow->monitor), num_packets, BReactor_PendingGroup(dp->reactor))) {
+    if (!PacketBuffer_Init(&flow->buffer, BufferWriter_GetOutput(&flow->ainput), PacketPassInactivityMonitor_GetInput(&flow->monitor), num_packets, BReactor_PendingGroup(dp->reactor))) {
         BLog(BLOG_ERROR, "PacketBuffer_Init failed for relay flow from peer %d to %d", (int)rs->source_id, (int)dp->dest_id);
         goto fail1;
     }
@@ -120,7 +117,7 @@ struct dp_relay_flow * create_relay_flow (DataProtoRelaySource *rs, DataProtoDes
     return flow;
     
 fail1:
-    PacketBufferAsyncInput_Free(&flow->ainput);
+    BufferWriter_Free(&flow->ainput);
     PacketPassInactivityMonitor_Free(&flow->monitor);
     PacketPassFairQueueFlow_Free(&flow->qflow);
     free(flow);
@@ -133,12 +130,9 @@ void dealloc_relay_flow (struct dp_relay_flow *flow)
     #ifndef NDEBUG
     if (!flow->dp->d_freeing) {
         ASSERT(!PacketPassFairQueueFlow_IsBusy(&flow->qflow))
-        ASSERT(!PacketPassInterface_InClient(flow->dp->d_output))
     }
     #endif
     
-    DataProtoDest *o = flow->dp;
-    
     // remove from dp list
     LinkedList2_Remove(&flow->dp->relay_flows_list, &flow->dp_list_node);
     
@@ -152,7 +146,7 @@ void dealloc_relay_flow (struct dp_relay_flow *flow)
     PacketBuffer_Free(&flow->buffer);
     
     // free async input
-    PacketBufferAsyncInput_Free(&flow->ainput);
+    BufferWriter_Free(&flow->ainput);
     
     // free inacitvity monitor
     PacketPassInactivityMonitor_Free(&flow->monitor);
@@ -164,63 +158,43 @@ void dealloc_relay_flow (struct dp_relay_flow *flow)
     free(flow);
 }
 
-int release_relay_flow (struct dp_relay_flow *flow)
+void release_relay_flow (struct dp_relay_flow *flow)
 {
     ASSERT(!flow->dp->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(flow->dp->d_output))
-    
-    DataProtoDest *o = flow->dp;
     
+    // release it if it's busy
     if (PacketPassFairQueueFlow_IsBusy(&flow->qflow)) {
-        // release it
-        DEAD_ENTER(o->dead)
         PacketPassFairQueueFlow_Release(&flow->qflow);
-        if (DEAD_LEAVE(o->dead)) {
-            return -1;
-        }
     }
     
     // remove flow
     dealloc_relay_flow(flow);
-    
-    return 0;
 }
 
 void flow_monitor_handler (struct dp_relay_flow *flow)
 {
     ASSERT(!flow->dp->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(flow->dp->d_output))
     
     BLog(BLOG_NOTICE, "relay flow from peer %d to %d timed out", (int)flow->rs->source_id, (int)flow->dp->dest_id);
     
     release_relay_flow(flow);
-    return;
 }
 
 void monitor_handler (DataProtoDest *o)
 {
     ASSERT(!o->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(o->d_output))
     DebugObject_Access(&o->d_obj);
     
     send_keepalive(o);
-    return;
 }
 
-int send_keepalive (DataProtoDest *o)
+void send_keepalive (DataProtoDest *o)
 {
     ASSERT(!o->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(o->d_output))
     
     BLog(BLOG_DEBUG, "sending keepalive to peer %d", (int)o->dest_id);
     
-    DEAD_ENTER(o->dead)
     PacketRecvBlocker_AllowBlockedPacket(&o->ka_blocker);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
-    
-    return 0;
 }
 
 void receive_timer_handler (DataProtoDest *o)
@@ -269,12 +243,9 @@ int pointer_comparator (void *user, void **val1, void **val2)
 
 void keepalive_job_handler (DataProtoDest *o)
 {
-    ASSERT(!o->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(o->d_output))
     DebugObject_Access(&o->d_obj);
     
-    send_keepalive(o);
-    return;
+    PacketRecvBlocker_AllowBlockedPacket(&o->ka_blocker);
 }
 
 int DataProtoDest_Init (DataProtoDest *o, BReactor *reactor, peerid_t dest_id, PacketPassInterface *output, btime_t keepalive_time, btime_t tolerance_time, DataProtoDest_handler handler, void *user)
@@ -297,8 +268,12 @@ int DataProtoDest_Init (DataProtoDest *o, BReactor *reactor, peerid_t dest_id, P
     // set frame MTU
     o->frame_mtu = o->mtu - (sizeof(struct dataproto_header) + sizeof(struct dataproto_peer_id));
     
+    // schedule keep-alive (needs to be before the buffer)
+    BPending_Init(&o->keepalive_job, BReactor_PendingGroup(o->reactor), (BPending_handler)keepalive_job_handler, o);
+    BPending_Set(&o->keepalive_job);
+    
     // init notifier
-    PacketPassNotifier_Init(&o->notifier, output);
+    PacketPassNotifier_Init(&o->notifier, output, BReactor_PendingGroup(o->reactor));
     PacketPassNotifier_SetHandler(&o->notifier, (PacketPassNotifier_handler_notify)notifier_handler, o);
     
     // init monitor
@@ -312,10 +287,10 @@ int DataProtoDest_Init (DataProtoDest *o, BReactor *reactor, peerid_t dest_id, P
     PacketPassFairQueueFlow_Init(&o->ka_qflow, &o->queue);
     
     // init keepalive source
-    DataProtoKeepaliveSource_Init(&o->ka_source);
+    DataProtoKeepaliveSource_Init(&o->ka_source, BReactor_PendingGroup(o->reactor));
     
     // init keepalive blocker
-    PacketRecvBlocker_Init(&o->ka_blocker, DataProtoKeepaliveSource_GetOutput(&o->ka_source));
+    PacketRecvBlocker_Init(&o->ka_blocker, DataProtoKeepaliveSource_GetOutput(&o->ka_source), BReactor_PendingGroup(o->reactor));
     
     // init keepalive buffer
     if (!SinglePacketBuffer_Init(&o->ka_buffer, PacketRecvBlocker_GetOutput(&o->ka_blocker), PacketPassFairQueueFlow_GetInput(&o->ka_qflow), BReactor_PendingGroup(o->reactor))) {
@@ -332,14 +307,7 @@ int DataProtoDest_Init (DataProtoDest *o, BReactor *reactor, peerid_t dest_id, P
     // init relay flows list
     LinkedList2_Init(&o->relay_flows_list);
     
-    // init keepalive job
-    BPending_Init(&o->keepalive_job, BReactor_PendingGroup(o->reactor), (BPending_handler)keepalive_job_handler, o);
-    BPending_Set(&o->keepalive_job);
-    
-    // init flows counter
     DebugCounter_Init(&o->flows_counter);
-    
-    // init debug object
     DebugObject_Init(&o->d_obj);
     
     #ifndef NDEBUG
@@ -356,6 +324,7 @@ fail0:
     PacketPassFairQueue_Free(&o->queue);
     PacketPassInactivityMonitor_Free(&o->monitor);
     PacketPassNotifier_Free(&o->notifier);
+    BPending_Free(&o->keepalive_job);
     return 0;
 }
 
@@ -364,9 +333,6 @@ void DataProtoDest_Free (DataProtoDest *o)
     DebugCounter_Free(&o->flows_counter);
     DebugObject_Free(&o->d_obj);
     
-    // free keepalive job
-    BPending_Free(&o->keepalive_job);
-    
     // allow freeing queue flows
     PacketPassFairQueue_PrepareFree(&o->queue);
     
@@ -401,6 +367,9 @@ void DataProtoDest_Free (DataProtoDest *o)
     // free notifier
     PacketPassNotifier_Free(&o->notifier);
     
+    // free keepalive job
+    BPending_Free(&o->keepalive_job);
+    
     // free dead var
     DEAD_KILL(o->dead);
 }
@@ -423,7 +392,6 @@ void DataProtoDest_SubmitRelayFrame (DataProtoDest *o, DataProtoRelaySource *rs,
     ASSERT(data_len <= o->frame_mtu)
     ASSERT(buffer_num_packets > 0)
     ASSERT(!o->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(o->d_output))
     DebugObject_Access(&rs->d_obj);
     DebugObject_Access(&o->d_obj);
     
@@ -442,7 +410,7 @@ void DataProtoDest_SubmitRelayFrame (DataProtoDest *o, DataProtoRelaySource *rs,
     // get a buffer
     uint8_t *out;
     // safe because of PacketBufferAsyncInput
-    if (!BestEffortPacketWriteInterface_Sender_StartPacket(flow->ainput_if, &out)) {
+    if (!BufferWriter_StartPacket(&flow->ainput, &out)) {
         BLog(BLOG_NOTICE, "out of buffer for relayed frame from peer %d to %d", (int)rs->source_id, (int)o->dest_id);
         return;
     }
@@ -459,15 +427,13 @@ void DataProtoDest_SubmitRelayFrame (DataProtoDest *o, DataProtoRelaySource *rs,
     memcpy(out + sizeof(struct dataproto_header) + sizeof(struct dataproto_peer_id), data, data_len);
     
     // submit it
-    BestEffortPacketWriteInterface_Sender_EndPacket(flow->ainput_if, sizeof(struct dataproto_header) + sizeof(struct dataproto_peer_id) + data_len);
-    return;
+    BufferWriter_EndPacket(&flow->ainput, sizeof(struct dataproto_header) + sizeof(struct dataproto_peer_id) + data_len);
 }
 
 void DataProtoDest_Received (DataProtoDest *o, int peer_receiving)
 {
     ASSERT(peer_receiving == 0 || peer_receiving == 1)
     ASSERT(!o->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(o->d_output))
     DebugObject_Access(&o->d_obj);
     
     int prev_up = o->up;
@@ -479,9 +445,7 @@ void DataProtoDest_Received (DataProtoDest *o, int peer_receiving)
         // peer reports not receiving, consider down
         o->up = 0;
         // send keep-alive to converge faster
-        if (send_keepalive(o) < 0) {
-            return;
-        }
+        send_keepalive(o);
     } else {
         // consider up
         o->up = 1;
@@ -515,11 +479,10 @@ int DataProtoLocalSource_Init (DataProtoLocalSource *o, int frame_mtu, peerid_t
     PacketPassConnector_Init(&o->connector, packet_mtu, BReactor_PendingGroup(reactor));
     
     // init async input
-    PacketBufferAsyncInput_Init(&o->ainput, packet_mtu);
-    o->ainput_if = PacketBufferAsyncInput_GetInput(&o->ainput);
+    BufferWriter_Init(&o->ainput, packet_mtu, BReactor_PendingGroup(reactor));
     
     // init buffer
-    if (!PacketBuffer_Init(&o->buffer, PacketBufferAsyncInput_GetOutput(&o->ainput), PacketPassConnector_GetInput(&o->connector), num_packets, BReactor_PendingGroup(reactor))) {
+    if (!PacketBuffer_Init(&o->buffer, BufferWriter_GetOutput(&o->ainput), PacketPassConnector_GetInput(&o->connector), num_packets, BReactor_PendingGroup(reactor))) {
         BLog(BLOG_ERROR, "PacketBuffer_Init failed");
         goto fail1;
     }
@@ -527,13 +490,12 @@ int DataProtoLocalSource_Init (DataProtoLocalSource *o, int frame_mtu, peerid_t
     // set no DataProto
     o->dp = NULL;
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
     
     return 1;
     
 fail1:
-    PacketBufferAsyncInput_Free(&o->ainput);
+    BufferWriter_Free(&o->ainput);
     PacketPassConnector_Free(&o->connector);
 fail0:
     return 0;
@@ -548,7 +510,7 @@ void DataProtoLocalSource_Free (DataProtoLocalSource *o)
     PacketBuffer_Free(&o->buffer);
     
     // free async input
-    PacketBufferAsyncInput_Free(&o->ainput);
+    BufferWriter_Free(&o->ainput);
     
     // free connector
     PacketPassConnector_Free(&o->connector);
@@ -564,14 +526,13 @@ void DataProtoLocalSource_SubmitFrame (DataProtoLocalSource *o, uint8_t *data, i
     if (o->dp) {
         ASSERT(!o->d_dp_released)
         ASSERT(!o->dp->d_freeing)
-        ASSERT(!PacketPassInterface_InClient(o->dp->d_output))
     }
     DebugObject_Access(&o->d_obj);
     
     // get a buffer
     uint8_t *out;
     // safe because of PacketBufferAsyncInput
-    if (!BestEffortPacketWriteInterface_Sender_StartPacket(o->ainput_if, &out)) {
+    if (!BufferWriter_StartPacket(&o->ainput, &out)) {
         BLog(BLOG_NOTICE, "out of buffer for frame from peer %d to %d", (int)o->source_id, (int)o->dest_id);
         return;
     }
@@ -588,8 +549,7 @@ void DataProtoLocalSource_SubmitFrame (DataProtoLocalSource *o, uint8_t *data, i
     memcpy(out + sizeof(struct dataproto_header) + sizeof(struct dataproto_peer_id), data, data_len);
     
     // submit it
-    BestEffortPacketWriteInterface_Sender_EndPacket(o->ainput_if, sizeof(struct dataproto_header) + sizeof(struct dataproto_peer_id) + data_len);
-    return;
+    BufferWriter_EndPacket(&o->ainput, sizeof(struct dataproto_header) + sizeof(struct dataproto_peer_id) + data_len);
 }
 
 void DataProtoLocalSource_Attach (DataProtoLocalSource *o, DataProtoDest *dp)
@@ -598,7 +558,6 @@ void DataProtoLocalSource_Attach (DataProtoLocalSource *o, DataProtoDest *dp)
     ASSERT(!o->dp)
     ASSERT(o->frame_mtu <= dp->frame_mtu)
     ASSERT(!dp->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(dp->d_output))
     DebugObject_Access(&o->d_obj);
     DebugObject_Access(&dp->d_obj);
     
@@ -624,15 +583,10 @@ void DataProtoLocalSource_Release (DataProtoLocalSource *o)
     ASSERT(o->dp)
     ASSERT(!o->d_dp_released)
     ASSERT(!o->dp->d_freeing)
-    ASSERT(!PacketPassInterface_InClient(o->dp->d_output))
     DebugObject_Access(&o->d_obj);
     
     if (PacketPassFairQueueFlow_IsBusy(&o->dp_qflow)) {
-        DEAD_ENTER(o->dead)
         PacketPassFairQueueFlow_Release(&o->dp_qflow);
-        if (DEAD_LEAVE(o->dead)) {
-            return;
-        }
     }
     
     #ifndef NDEBUG
@@ -645,9 +599,6 @@ void DataProtoLocalSource_Detach (DataProtoLocalSource *o)
     #ifndef NDEBUG
     ASSERT(o->dp)
     ASSERT(o->d_dp_released || o->dp->d_freeing)
-    if (!o->dp->d_freeing) {
-        ASSERT(!PacketPassInterface_InClient(o->dp->d_output))
-    }
     #endif
     DebugObject_Access(&o->d_obj);
     
@@ -680,7 +631,6 @@ void DataProtoRelaySource_Init (DataProtoRelaySource *o, peerid_t source_id)
     // init relay flows tree
     BAVL_Init(&o->relay_flows_tree, OFFSET_DIFF(struct dp_relay_flow, dp, source_tree_node), (BAVL_comparator)pointer_comparator, NULL);
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
@@ -709,11 +659,7 @@ void DataProtoRelaySource_Release (DataProtoRelaySource *o)
     while (node = LinkedList2_GetFirst(&o->relay_flows_list)) {
         struct dp_relay_flow *flow = UPPER_OBJECT(node, struct dp_relay_flow, source_list_node);
         
-        DEAD_ENTER(o->dead)
         release_relay_flow(flow);
-        if (DEAD_LEAVE(o->dead)) {
-            return;
-        }
     }
 }
 

client/DataProto.h

@@ -37,13 +37,14 @@
 #include <structure/BAVL.h>
 #include <system/DebugObject.h>
 #include <system/BReactor.h>
+#include <system/BPending.h>
 #include <flow/PacketPassFairQueue.h>
 #include <flow/PacketPassInactivityMonitor.h>
 #include <flow/PacketPassNotifier.h>
 #include <flow/DataProtoKeepaliveSource.h>
 #include <flow/PacketRecvBlocker.h>
 #include <flow/SinglePacketBuffer.h>
-#include <flow/PacketBufferAsyncInput.h>
+#include <flow/BufferWriter.h>
 #include <flow/PacketBuffer.h>
 #include <flow/PacketPassConnector.h>
 
@@ -89,8 +90,7 @@ typedef struct {
     int frame_mtu;
     peerid_t source_id;
     peerid_t dest_id;
-    BestEffortPacketWriteInterface *ainput_if;
-    PacketBufferAsyncInput ainput;
+    BufferWriter ainput;
     PacketBuffer buffer;
     PacketPassConnector connector;
     DataProtoDest *dp;

client/DatagramPeerIO.c

@@ -52,24 +52,19 @@ int init_persistent_io (DatagramPeerIO *o, btime_t latency, PacketPassInterface
     // init error domain
     FlowErrorDomain_Init(&o->domain, (FlowErrorDomain_handler)error_handler, o);
     
-    // init encoder group
-    if (!SPProtoEncoderGroup_Init(&o->encoder_group, o->sp_params)) {
-        goto fail0;
-    }
-    
     // init sending base
     
     // init disassembler
     FragmentProtoDisassembler_Init(&o->send_disassembler, o->reactor, o->payload_mtu, o->spproto_payload_mtu, -1, latency);
     
     // init encoder
-    if (!SPProtoEncoder_Init(&o->send_encoder, &o->encoder_group, FragmentProtoDisassembler_GetOutput(&o->send_disassembler), BReactor_PendingGroup(o->reactor))) {
+    if (!SPProtoEncoder_Init(&o->send_encoder, o->sp_params, FragmentProtoDisassembler_GetOutput(&o->send_disassembler), BReactor_PendingGroup(o->reactor))) {
         BLog(BLOG_ERROR, "SPProtoEncoder_Init failed");
         goto fail1;
     }
     
     // init notifier
-    PacketRecvNotifier_Init(&o->send_notifier, SPProtoEncoder_GetOutput(&o->send_encoder));
+    PacketRecvNotifier_Init(&o->send_notifier, SPProtoEncoder_GetOutput(&o->send_encoder), BReactor_PendingGroup(o->reactor));
     if (SPPROTO_HAVE_OTP(o->sp_params)) {
         PacketRecvNotifier_SetHandler(&o->send_notifier, (PacketRecvNotifier_handler_notify)send_encoder_notifier_handler, o);
     }
@@ -86,15 +81,15 @@ int init_persistent_io (DatagramPeerIO *o, btime_t latency, PacketPassInterface
     // init receiving
     
     // init assembler
-    if (!FragmentProtoAssembler_Init(&o->recv_assembler, o->spproto_payload_mtu, recv_userif, 1, fragmentproto_max_chunks_for_frame(o->spproto_payload_mtu, o->payload_mtu))) {
+    if (!FragmentProtoAssembler_Init(&o->recv_assembler, o->spproto_payload_mtu, recv_userif, 1, fragmentproto_max_chunks_for_frame(o->spproto_payload_mtu, o->payload_mtu), BReactor_PendingGroup(o->reactor))) {
         goto fail3;
     }
     
     // init notifier
-    PacketPassNotifier_Init(&o->recv_notifier, FragmentProtoAssembler_GetInput(&o->recv_assembler));
+    PacketPassNotifier_Init(&o->recv_notifier, FragmentProtoAssembler_GetInput(&o->recv_assembler), BReactor_PendingGroup(o->reactor));
     
     // init decoder
-    if (!SPProtoDecoder_Init(&o->recv_decoder, PacketPassNotifier_GetInput(&o->recv_notifier), o->sp_params, 2)) {
+    if (!SPProtoDecoder_Init(&o->recv_decoder, PacketPassNotifier_GetInput(&o->recv_notifier), o->sp_params, 2, BReactor_PendingGroup(o->reactor))) {
         goto fail4;
     }
     
@@ -122,9 +117,6 @@ fail2:
     SPProtoEncoder_Free(&o->send_encoder);
 fail1:
     FragmentProtoDisassembler_Free(&o->send_disassembler);
-fail0a:
-    SPProtoEncoderGroup_Free(&o->encoder_group);
-fail0:
     return 0;
 }
 
@@ -143,15 +135,12 @@ void free_persistent_io (DatagramPeerIO *o)
     PacketRecvNotifier_Free(&o->send_notifier);
     SPProtoEncoder_Free(&o->send_encoder);
     FragmentProtoDisassembler_Free(&o->send_disassembler);
-    
-    // free encoder group
-    SPProtoEncoderGroup_Free(&o->encoder_group);
 }
 
 void init_sending (DatagramPeerIO *o, BAddr addr, BIPAddr local_addr)
 {
     // init sink
-    DatagramSocketSink_Init(&o->send_sink, FlowErrorReporter_Create(&o->domain, DATAGRAMPEERIO_COMPONENT_SINK), &o->sock, o->effective_socket_mtu, addr, local_addr);
+    DatagramSocketSink_Init(&o->send_sink, FlowErrorReporter_Create(&o->domain, DATAGRAMPEERIO_COMPONENT_SINK), &o->sock, o->effective_socket_mtu, addr, local_addr, BReactor_PendingGroup(o->reactor));
     
     // connect sink
     PacketPassConnector_ConnectOutput(&o->send_connector, DatagramSocketSink_GetInput(&o->send_sink));
@@ -169,7 +158,7 @@ void free_sending (DatagramPeerIO *o)
 void init_receiving (DatagramPeerIO *o)
 {
     // init source
-    DatagramSocketSource_Init(&o->recv_source, FlowErrorReporter_Create(&o->domain, DATAGRAMPEERIO_COMPONENT_SOURCE), &o->sock, o->effective_socket_mtu);
+    DatagramSocketSource_Init(&o->recv_source, FlowErrorReporter_Create(&o->domain, DATAGRAMPEERIO_COMPONENT_SOURCE), &o->sock, o->effective_socket_mtu, BReactor_PendingGroup(o->reactor));
     
     // connect source
     PacketRecvConnector_ConnectInput(&o->recv_connector, DatagramSocketSource_GetOutput(&o->recv_source));
@@ -280,7 +269,7 @@ void send_encoder_notifier_handler (DatagramPeerIO *o, uint8_t *data, int data_l
 {
     ASSERT(SPPROTO_HAVE_OTP(o->sp_params))
     
-    if (o->handler_otp_warning && SPProtoEncoderGroup_GetOTPPosition(&o->encoder_group) == o->handler_otp_warning_num_used) { 
+    if (o->handler_otp_warning && SPProtoEncoder_GetOTPPosition(&o->send_encoder) == o->handler_otp_warning_num_used) { 
         o->handler_otp_warning(o->handler_otp_warning_user);
         return;
     }
@@ -451,7 +440,7 @@ void DatagramPeerIO_SetEncryptionKey (DatagramPeerIO *o, uint8_t *encryption_key
     ASSERT(o->sp_params.encryption_mode != SPPROTO_ENCRYPTION_MODE_NONE)
     
     // set sending key
-    SPProtoEncoderGroup_SetEncryptionKey(&o->encoder_group, encryption_key);
+    SPProtoEncoder_SetEncryptionKey(&o->send_encoder, encryption_key);
     
     // set receiving key
     SPProtoDecoder_SetEncryptionKey(&o->recv_decoder, encryption_key);
@@ -462,7 +451,7 @@ void DatagramPeerIO_RemoveEncryptionKey (DatagramPeerIO *o)
     ASSERT(o->sp_params.encryption_mode != SPPROTO_ENCRYPTION_MODE_NONE)
     
     // remove sending key
-    SPProtoEncoderGroup_RemoveEncryptionKey(&o->encoder_group);
+    SPProtoEncoder_RemoveEncryptionKey(&o->send_encoder);
     
     // remove receiving key
     SPProtoDecoder_RemoveEncryptionKey(&o->recv_decoder);
@@ -473,7 +462,7 @@ void DatagramPeerIO_SetOTPSendSeed (DatagramPeerIO *o, uint16_t seed_id, uint8_t
     ASSERT(SPPROTO_HAVE_OTP(o->sp_params))
     
     // set sending seed
-    SPProtoEncoderGroup_SetOTPSeed(&o->encoder_group, seed_id, key, iv);
+    SPProtoEncoder_SetOTPSeed(&o->send_encoder, seed_id, key, iv);
 }
 
 void DatagramPeerIO_RemoveOTPSendSeed (DatagramPeerIO *o)
@@ -481,7 +470,7 @@ void DatagramPeerIO_RemoveOTPSendSeed (DatagramPeerIO *o)
     ASSERT(SPPROTO_HAVE_OTP(o->sp_params))
     
     // remove sending seed
-    SPProtoEncoderGroup_RemoveOTPSeed(&o->encoder_group);
+    SPProtoEncoder_RemoveOTPSeed(&o->send_encoder);
 }
 
 void DatagramPeerIO_AddOTPRecvSeed (DatagramPeerIO *o, uint16_t seed_id, uint8_t *key, uint8_t *iv)

client/DatagramPeerIO.h

@@ -88,9 +88,6 @@ typedef struct {
     // flow error domain
     FlowErrorDomain domain;
     
-    // encoder group
-    SPProtoEncoderGroup encoder_group;
-    
     // persistent I/O objects
     
     // sending base

client/PasswordSender.c

@@ -81,15 +81,15 @@ void PasswordSender_Init (PasswordSender *o, uint64_t password, int ssl, BSocket
     // init sink
     StreamPassInterface *sink_if;
     if (o->ssl) {
-        PRStreamSink_Init(&o->sink.ssl, FlowErrorReporter_Create(&o->domain, COMPONENT_SINK), o->ssl_bprfd);
+        PRStreamSink_Init(&o->sink.ssl, FlowErrorReporter_Create(&o->domain, COMPONENT_SINK), o->ssl_bprfd,  BReactor_PendingGroup(reactor));
         sink_if = PRStreamSink_GetInput(&o->sink.ssl);
     } else {
-        StreamSocketSink_Init(&o->sink.plain, FlowErrorReporter_Create(&o->domain, COMPONENT_SINK), o->plain_sock);
+        StreamSocketSink_Init(&o->sink.plain, FlowErrorReporter_Create(&o->domain, COMPONENT_SINK), o->plain_sock,  BReactor_PendingGroup(reactor));
         sink_if = StreamSocketSink_GetInput(&o->sink.plain);
     }
     
     // init PacketStreamSender
-    PacketStreamSender_Init(&o->pss, sink_if, sizeof(o->password));
+    PacketStreamSender_Init(&o->pss, sink_if, sizeof(o->password),  BReactor_PendingGroup(reactor));
     
     // init SinglePacketSender
     SinglePacketSender_Init(&o->sps, (uint8_t *)&o->password, sizeof(o->password), PacketStreamSender_GetInput(&o->pss), (SinglePacketSender_handler)sent_handler, o, BReactor_PendingGroup(reactor));

client/StreamPeerIO.c

@@ -374,8 +374,8 @@ int init_persistent_io (StreamPeerIO *pio, PacketPassInterface *user_recv_if)
     FlowErrorDomain_Init(&pio->ioerrdomain, (FlowErrorDomain_handler)error_handler, pio);
     
     // init sending objects
-    PacketCopier_Init(&pio->output_user_copier, pio->payload_mtu);
-    PacketProtoEncoder_Init(&pio->output_user_ppe, PacketCopier_GetOutput(&pio->output_user_copier));
+    PacketCopier_Init(&pio->output_user_copier, pio->payload_mtu, BReactor_PendingGroup(pio->reactor));
+    PacketProtoEncoder_Init(&pio->output_user_ppe, PacketCopier_GetOutput(&pio->output_user_copier), BReactor_PendingGroup(pio->reactor));
     PacketPassConnector_Init(&pio->output_connector, PACKETPROTO_ENCLEN(pio->payload_mtu), BReactor_PendingGroup(pio->reactor));
     if (!SinglePacketBuffer_Init(&pio->output_user_spb, PacketProtoEncoder_GetOutput(&pio->output_user_ppe), PacketPassConnector_GetInput(&pio->output_connector), BReactor_PendingGroup(pio->reactor))) {
         goto fail1;
@@ -429,36 +429,32 @@ int init_io (StreamPeerIO *pio, sslsocket *sock)
     StreamPassInterface *sink_interface;
     if (pio->ssl) {
         PRStreamSink_Init(
-            &pio->output_sink.ssl,
-            FlowErrorReporter_Create(&pio->ioerrdomain, COMPONENT_SINK),
-            &sock->ssl_bprfd
+            &pio->output_sink.ssl, FlowErrorReporter_Create(&pio->ioerrdomain, COMPONENT_SINK),
+            &sock->ssl_bprfd, BReactor_PendingGroup(pio->reactor)
         );
         sink_interface = PRStreamSink_GetInput(&pio->output_sink.ssl);
     } else {
         StreamSocketSink_Init(
-            &pio->output_sink.plain,
-            FlowErrorReporter_Create(&pio->ioerrdomain, COMPONENT_SINK),
-            &sock->sock
+            &pio->output_sink.plain, FlowErrorReporter_Create(&pio->ioerrdomain, COMPONENT_SINK),
+            &sock->sock, BReactor_PendingGroup(pio->reactor)
         );
         sink_interface = StreamSocketSink_GetInput(&pio->output_sink.plain);
     }
-    PacketStreamSender_Init(&pio->output_pss, sink_interface, PACKETPROTO_ENCLEN(pio->payload_mtu));
+    PacketStreamSender_Init(&pio->output_pss, sink_interface, PACKETPROTO_ENCLEN(pio->payload_mtu), BReactor_PendingGroup(pio->reactor));
     PacketPassConnector_ConnectOutput(&pio->output_connector, PacketStreamSender_GetInput(&pio->output_pss));
     
     // init receiving
     StreamRecvInterface *source_interface;
     if (pio->ssl) {
         PRStreamSource_Init(
-            &pio->input_source.ssl,
-            FlowErrorReporter_Create(&pio->ioerrdomain, COMPONENT_SOURCE),
-            &sock->ssl_bprfd
+            &pio->input_source.ssl, FlowErrorReporter_Create(&pio->ioerrdomain, COMPONENT_SOURCE),
+            &sock->ssl_bprfd, BReactor_PendingGroup(pio->reactor)
         );
         source_interface = PRStreamSource_GetOutput(&pio->input_source.ssl);
     } else {
         StreamSocketSource_Init(
-            &pio->input_source.plain,
-            FlowErrorReporter_Create(&pio->ioerrdomain, COMPONENT_SOURCE),
-            &sock->sock
+            &pio->input_source.plain, FlowErrorReporter_Create(&pio->ioerrdomain, COMPONENT_SOURCE),
+            &sock->sock, BReactor_PendingGroup(pio->reactor)
         );
         source_interface = StreamSocketSource_GetOutput(&pio->input_source.plain);
     }

client/client.h

@@ -26,8 +26,8 @@
 #include <protocol/scproto.h>
 #include <structure/LinkedList2.h>
 #include <structure/HashTable.h>
-#include <structure/BAVL.h>
 #include <flow/SinglePacketBuffer.h>
+#include <flow/PacketPassFairQueue.h>
 #include <tuntap/BTap.h>
 #include <client/DatagramPeerIO.h>
 #include <client/StreamPeerIO.h>
@@ -85,11 +85,13 @@
 struct device_data {
     BTap btap;
     int mtu;
+    
+    // input
     SinglePacketBuffer input_buffer;
     PacketPassInterface input_interface;
-    PacketPassInterface *output_interface;
-    uint8_t *framebuf;
-    int framelen;
+    
+    // output
+    PacketPassFairQueue output_queue;
 };
 
 struct peer_data;
@@ -142,6 +144,10 @@ struct peer_data {
     // local flow
     DataProtoLocalSource local_dpflow;
     
+    // local receive flow
+    PacketPassInterface *local_recv_if;
+    PacketPassFairQueueFlow local_recv_qflow;
+    
     // relay source
     DataProtoRelaySource relay_source;
     
@@ -196,11 +202,6 @@ struct peer_data {
     LinkedList2 groups_free;
     HashTable groups_hashtable;
     
-    // peers linked list node
-    LinkedList2Node list_node;
-    // peers-by-ID hash table node
-    HashTableNode table_node;
-    
     // relay server specific
     int is_relay;
     LinkedList2Node relay_list_node;
@@ -209,4 +210,12 @@ struct peer_data {
     // binding state
     int binding;
     int binding_addrpos;
+    
+    // jobs
+    BPending job_send_seed_after_binding;
+    
+    // peers linked list node
+    LinkedList2Node list_node;
+    // peers-by-ID hash table node
+    HashTableNode table_node;
 };

examples/FastPacketSource.h

@@ -26,45 +26,23 @@
 #include <stdint.h>
 #include <string.h>
 
-#include <misc/dead.h>
 #include <misc/debug.h>
-#include <system/BPending.h>
+#include <system/DebugObject.h>
 #include <flow/PacketPassInterface.h>
 
 typedef struct {
-    dead_t dead;
     PacketPassInterface *output;
     int psize;
     uint8_t *data;
     int data_len;
-    BPending start_job;
+    DebugObject d_obj;
 } FastPacketSource;
 
-static void _FastPacketSource_send (FastPacketSource *s)
-{
-    while (1) {
-        DEAD_ENTER(s->dead)
-        int res = PacketPassInterface_Sender_Send(s->output, s->data, s->data_len);
-        if (DEAD_LEAVE(s->dead)) {
-            return;
-        }
-        ASSERT(res == 0 || res == 1)
-        if (res == 0) {
-            return;
-        }
-    }
-}
-
 static void _FastPacketSource_output_handler_done (FastPacketSource *s)
 {
-    _FastPacketSource_send(s);
-    return;
-}
-
-static void _FastPacketSource_job_handler (FastPacketSource *s)
-{
-    _FastPacketSource_send(s);
-    return;
+    DebugObject_Access(&s->d_obj);
+    
+    PacketPassInterface_Sender_Send(s->output, s->data, s->data_len);
 }
 
 static void FastPacketSource_Init (FastPacketSource *s, PacketPassInterface *output, uint8_t *data, int data_len, BPendingGroup *pg)
@@ -77,24 +55,18 @@ static void FastPacketSource_Init (FastPacketSource *s, PacketPassInterface *out
     s->data = data;
     s->data_len = data_len;
     
-    // init dead var
-    DEAD_INIT(s->dead);
-    
     // init output
     PacketPassInterface_Sender_Init(s->output, (PacketPassInterface_handler_done)_FastPacketSource_output_handler_done, s);
     
-    // init start job
-    BPending_Init(&s->start_job, pg, (BPending_handler)_FastPacketSource_job_handler, s);
-    BPending_Set(&s->start_job);
+    // schedule send
+    PacketPassInterface_Sender_Send(s->output, s->data, s->data_len);
+    
+    DebugObject_Init(&s->d_obj);
 }
 
 static void FastPacketSource_Free (FastPacketSource *s)
 {
-    // free start job
-    BPending_Free(&s->start_job);
-    
-    // free dead var
-    DEAD_KILL(s->dead);
+    DebugObject_Free(&s->d_obj);
 }
 
 #endif

examples/RandomPacketSink.h

@@ -27,16 +27,20 @@
 
 #include <security/BRandom.h>
 #include <system/BReactor.h>
+#include <system/DebugObject.h>
 #include <flow/PacketPassInterface.h>
 
 typedef struct {
     BReactor *reactor;
     PacketPassInterface input;
     BTimer timer;
+    DebugObject d_obj;
 } RandomPacketSink;
 
-static int _RandomPacketSink_input_handler_send (RandomPacketSink *s, uint8_t *data, int data_len)
+static void _RandomPacketSink_input_handler_send (RandomPacketSink *s, uint8_t *data, int data_len)
 {
+    DebugObject_Access(&s->d_obj);
+    
     printf("sink: send '");
     fwrite(data, data_len, 1, stdout);
     
@@ -44,41 +48,58 @@ static int _RandomPacketSink_input_handler_send (RandomPacketSink *s, uint8_t *d
     BRandom_randomize(&r, sizeof(r));
     if (r&(uint8_t)1) {
         printf("' accepting\n");
-        return 1;
+        PacketPassInterface_Done(&s->input);
+    } else {
+        printf("' delaying\n");
+        BReactor_SetTimer(s->reactor, &s->timer);
     }
-    
-    printf("' delaying\n");
-    BReactor_SetTimer(s->reactor, &s->timer);
-    return 0;
 }
 
 static void _RandomPacketSink_input_handler_cancel (RandomPacketSink *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     printf("sink: cancelled\n");
     BReactor_RemoveTimer(s->reactor, &s->timer);
 }
 
 static void _RandomPacketSink_timer_handler (RandomPacketSink *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     PacketPassInterface_Done(&s->input);
 }
 
 static void RandomPacketSink_Init (RandomPacketSink *s, BReactor *reactor, int mtu, int ms)
 {
+    // init arguments
     s->reactor = reactor;
-    PacketPassInterface_Init(&s->input, mtu, (PacketPassInterface_handler_send)_RandomPacketSink_input_handler_send, s);
+    
+    // init input
+    PacketPassInterface_Init(&s->input, mtu, (PacketPassInterface_handler_send)_RandomPacketSink_input_handler_send, s, BReactor_PendingGroup(reactor));
     PacketPassInterface_EnableCancel(&s->input, (PacketPassInterface_handler_cancel)_RandomPacketSink_input_handler_cancel);
+    
+    // init timer
     BTimer_Init(&s->timer, ms, (BTimer_handler)_RandomPacketSink_timer_handler, s);
+    
+    DebugObject_Init(&s->d_obj);
 }
 
 static void RandomPacketSink_Free (RandomPacketSink *s)
 {
+    DebugObject_Free(&s->d_obj);
+    
+    // free timer
     BReactor_RemoveTimer(s->reactor, &s->timer);
+    
+    // free input
     PacketPassInterface_Free(&s->input);
 }
 
 static PacketPassInterface * RandomPacketSink_GetInput (RandomPacketSink *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     return &s->input;
 }
 

examples/TimerPacketSink.h

@@ -34,14 +34,13 @@ typedef struct {
     BTimer timer;
 } TimerPacketSink;
 
-static int _TimerPacketSink_input_handler_send (TimerPacketSink *s, uint8_t *data, int data_len)
+static void _TimerPacketSink_input_handler_send (TimerPacketSink *s, uint8_t *data, int data_len)
 {
     printf("sink: send '");
     fwrite(data, data_len, 1, stdout);
     printf("'\n");
     
     BReactor_SetTimer(s->reactor, &s->timer);
-    return 0;
 }
 
 static void _TimerPacketSink_input_handler_cancel (TimerPacketSink *s)
@@ -56,7 +55,6 @@ static void _TimerPacketSink_timer_handler (TimerPacketSink *s)
     printf("sink: done\n");
     
     PacketPassInterface_Done(&s->input);
-    return;
 }
 
 static void TimerPacketSink_Init (TimerPacketSink *s, BReactor *reactor, int mtu, int ms)
@@ -65,7 +63,7 @@ static void TimerPacketSink_Init (TimerPacketSink *s, BReactor *reactor, int mtu
     s->reactor = reactor;
     
     // init input
-    PacketPassInterface_Init(&s->input, mtu, (PacketPassInterface_handler_send)_TimerPacketSink_input_handler_send, s);
+    PacketPassInterface_Init(&s->input, mtu, (PacketPassInterface_handler_send)_TimerPacketSink_input_handler_send, s, BReactor_PendingGroup(s->reactor));
     PacketPassInterface_EnableCancel(&s->input, (PacketPassInterface_handler_cancel)_TimerPacketSink_input_handler_cancel);
     
     // init timer

examples/fairqueue_test2.c

@@ -30,6 +30,8 @@
 #include <examples/FastPacketSource.h>
 #include <examples/RandomPacketSink.h>
 
+#define SINK_TIMER 0
+
 int main ()
 {
     // initialize logging
@@ -47,7 +49,7 @@ int main ()
     
     // initialize sink
     RandomPacketSink sink;
-    RandomPacketSink_Init(&sink, &reactor, 500, 0);
+    RandomPacketSink_Init(&sink, &reactor, 500, SINK_TIMER);
     
     // initialize queue
     PacketPassFairQueue fq;

examples/ipc_client.c

@@ -26,7 +26,6 @@
 #include <system/DebugObject.h>
 #include <system/BLog.h>
 #include <system/BSignal.h>
-#include <system/BPending.h>
 #include <ipc/BIPC.h>
 
 #define SEND_MTU 100
@@ -41,15 +40,13 @@ BReactor reactor;
 BIPC ipc;
 PacketPassInterface recv_if;
 PacketPassInterface *send_if;
-BPending start_job;
 
 static void terminate (int ret);
 static void signal_handler (void *user);
 static void ipc_handler (void *user);
-static void job_handler (void *user);
 static void send_packets (void);
 static void ipc_send_handler_done (void *user);
-static int ipc_recv_handler_send (void *user, uint8_t *data, int data_len);
+static void ipc_recv_handler_send (void *user, uint8_t *data, int data_len);
 
 int main (int argc, char **argv)
 {
@@ -99,7 +96,7 @@ int main (int argc, char **argv)
         goto fail2;
     }
     
-    PacketPassInterface_Init(&recv_if, 0, ipc_recv_handler_send, NULL);
+    PacketPassInterface_Init(&recv_if, 0, ipc_recv_handler_send, NULL, BReactor_PendingGroup(&reactor));
     
     if (!BIPC_InitConnect(&ipc, path, SEND_MTU, &recv_if, ipc_handler, NULL, &reactor)) {
         DEBUG("BIPC_InitConnect failed");
@@ -109,8 +106,7 @@ int main (int argc, char **argv)
     send_if = BIPC_GetSendInterface(&ipc);
     PacketPassInterface_Sender_Init(send_if, ipc_send_handler_done, NULL);
     
-    BPending_Init(&start_job, BReactor_PendingGroup(&reactor), job_handler, NULL);
-    BPending_Set(&start_job);
+    send_packets();
     
     int ret = BReactor_Exec(&reactor);
     
@@ -136,8 +132,6 @@ fail0:
 
 void terminate (int ret)
 {
-    BPending_Free(&start_job);
-    
     BIPC_Free(&ipc);
     
     PacketPassInterface_Free(&recv_if);
@@ -163,12 +157,6 @@ void ipc_handler (void *user)
     terminate(1);
 }
 
-void job_handler (void *user)
-{
-    send_packets();
-    return;
-}
-
 void send_packets (void)
 {
     ASSERT(current_packet >= 0)
@@ -176,25 +164,10 @@ void send_packets (void)
     ASSERT(!waiting)
     
     if (current_packet < num_packets) {
-        DEAD_ENTER(dead)
-        int res = PacketPassInterface_Sender_Send(send_if, packets[current_packet], strlen(packets[current_packet]));
-        if (DEAD_LEAVE(dead)) {
-            return;
-        }
-        
-        ASSERT(res == 0 || res == 1)
-        
-        if (!res) {
-            return;
-        }
-        
-        // wait for confirmation
-        waiting = 1;
-        
-        return;
+        PacketPassInterface_Sender_Send(send_if, (uint8_t *)packets[current_packet], strlen(packets[current_packet]));
+    } else {
+        terminate(0);
     }
-    
-    terminate(0);
 }
 
 void ipc_send_handler_done (void *user)
@@ -205,11 +178,9 @@ void ipc_send_handler_done (void *user)
     
     // wait for confirmation
     waiting = 1;
-    
-    return;
 }
 
-int ipc_recv_handler_send (void *user, uint8_t *data, int data_len)
+void ipc_recv_handler_send (void *user, uint8_t *data, int data_len)
 {
     ASSERT(current_packet >= 0)
     ASSERT(current_packet <= num_packets)
@@ -218,18 +189,21 @@ int ipc_recv_handler_send (void *user, uint8_t *data, int data_len)
     if (!waiting) {
         DEBUG("not waiting!");
         terminate(1);
-        return -1;
+        return;
     }
     
     if (data_len != 0) {
         DEBUG("reply not empty!");
         terminate(1);
-        return -1;
+        return;
     }
     
     current_packet++;
     waiting = 0;
     
+    // accept received packet
+    PacketPassInterface_Done(&recv_if);
+    
+    // send more packets
     send_packets();
-    return 1;
 }

examples/ipc_server.c

@@ -54,7 +54,7 @@ static void signal_handler (void *user);
 static void server_handler (void *user);
 static void remove_client (struct client *client);
 static void client_ipc_handler (struct client *client);
-static int client_recv_handler_send (struct client *client, uint8_t *data, int data_len);
+static void client_recv_handler_send (struct client *client, uint8_t *data, int data_len);
 static void client_send_handler_done (struct client *client);
 
 int main (int argc, char **argv)
@@ -143,7 +143,7 @@ void server_handler (void *user)
     
     DEAD_INIT(client->dead);
     
-    PacketPassInterface_Init(&client->recv_if, RECV_MTU, (PacketPassInterface_handler_send)client_recv_handler_send, client);
+    PacketPassInterface_Init(&client->recv_if, RECV_MTU, (PacketPassInterface_handler_send)client_recv_handler_send, client, BReactor_PendingGroup(&reactor));
     
     if (!BIPC_InitAccept(&client->ipc, &server, 0, &client->recv_if, (BIPC_handler)client_ipc_handler, client, &reactor)) {
         DEBUG("BIPC_InitAccept failed");
@@ -186,7 +186,7 @@ void client_ipc_handler (struct client *client)
     remove_client(client);
 }
 
-int client_recv_handler_send (struct client *client, uint8_t *data, int data_len)
+void client_recv_handler_send (struct client *client, uint8_t *data, int data_len)
 {
     // print message
     uint8_t buf[data_len + 1];
@@ -195,23 +195,11 @@ int client_recv_handler_send (struct client *client, uint8_t *data, int data_len
     printf("received: '%s'\n", buf);
     
     // send reply
-    DEAD_ENTER(client->dead)
-    int res = PacketPassInterface_Sender_Send(client->send_if, NULL, 0);
-    if (DEAD_LEAVE(client->dead)) {
-        return -1;
-    }
-    
-    if (!res) {
-        // wait for reply to be sent, then allow next message
-        return 0;
-    }
-    
-    return 1;
+    PacketPassInterface_Sender_Send(client->send_if, NULL, 0);
 }
 
 void client_send_handler_done (struct client *client)
 {
     // allow next message
     PacketPassInterface_Done(&client->recv_if);
-    return;
 }

flooder/flooder.c

@@ -151,7 +151,7 @@ static void server_handler_newclient (void *user, peerid_t peer_id, int flags, c
 static void server_handler_endclient (void *user, peerid_t peer_id);
 static void server_handler_message (void *user, peerid_t peer_id, uint8_t *data, int data_len);
 
-static int flood_source_handler_recv (void *user, uint8_t *data, int *data_len);
+static void flood_source_handler_recv (void *user, uint8_t *data);
 
 int main (int argc, char *argv[])
 {
@@ -405,7 +405,7 @@ void print_help (const char *name)
         "        [--ssl --nssdb <string> --client-cert-name <string>]\n"
         "        [--server-name <string>]\n"
         "        --server-addr <addr>\n"
-        "        [--flood-id <id> ...]\n"
+        "        [--flood-id <id>] ...\n"
         "Address format is a.b.c.d:port (IPv4) or [addr]:port (IPv6).\n",
         name
     );
@@ -637,10 +637,10 @@ void server_handler_ready (void *user, peerid_t param_my_id, uint32_t ext_ip)
     // init flooding
     
     // init source
-    PacketRecvInterface_Init(&flood_source, SC_MAX_ENC, flood_source_handler_recv, NULL);
+    PacketRecvInterface_Init(&flood_source, SC_MAX_ENC, flood_source_handler_recv, NULL, BReactor_PendingGroup(&ss));
     
     // init encoder
-    PacketProtoEncoder_Init(&flood_encoder, &flood_source);
+    PacketProtoEncoder_Init(&flood_encoder, &flood_source, BReactor_PendingGroup(&ss));
     
     // init buffer
     if (!SinglePacketBuffer_Init(&flood_buffer, PacketProtoEncoder_GetOutput(&flood_encoder), ServerConnection_GetSendInterface(&server), BReactor_PendingGroup(&ss))) {
@@ -687,7 +687,7 @@ void server_handler_message (void *user, peerid_t peer_id, uint8_t *data, int da
     BLog(BLOG_INFO, "message from %d", (int)peer_id);
 }
 
-int flood_source_handler_recv (void *user, uint8_t *data, int *data_len)
+void flood_source_handler_recv (void *user, uint8_t *data)
 {
     ASSERT(server_ready)
     ASSERT(!flood_blocking)
@@ -698,7 +698,7 @@ int flood_source_handler_recv (void *user, uint8_t *data, int *data_len)
     
     if (options.num_floods == 0) {
         flood_blocking = 1;
-        return 0;
+        return;
     }
     
     peerid_t peer_id = options.floods[flood_next];
@@ -714,6 +714,5 @@ int flood_source_handler_recv (void *user, uint8_t *data, int *data_len)
     
     memset(data + sizeof(struct sc_header) + sizeof(struct sc_client_outmsg), 0, SC_MAX_MSGLEN);
     
-    *data_len = sizeof(struct sc_header) + sizeof(struct sc_client_outmsg) + SC_MAX_MSGLEN;
-    return 1;
+    PacketRecvInterface_Done(&flood_source, sizeof(struct sc_header) + sizeof(struct sc_client_outmsg) + SC_MAX_MSGLEN);
 }

flow/BestEffortPacketWriteInterface.h

@@ -1,214 +0,0 @@
-/**
- * @file BestEffortPacketWriteInterface.h
- * @author Ambroz Bizjak <[email protected]>
- * 
- * @section LICENSE
- * 
- * This file is part of BadVPN.
- * 
- * BadVPN is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2
- * as published by the Free Software Foundation.
- * 
- * BadVPN is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- * 
- * @section DESCRIPTION
- * 
- * Interface which allows a sender to write packets to a buffer provided by the receiver
- * in a best-effort fashion.
- */
-
-#ifndef BADVPN_FLOW_BESTEFFORTPACKETWRITEINTERFACE
-#define BADVPN_FLOW_BESTEFFORTPACKETWRITEINTERFACE
-
-#include <stdint.h>
-#include <stdlib.h>
-
-#include <misc/debug.h>
-#include <misc/dead.h>
-#include <system/DebugObject.h>
-
-/**
- * Callback function invoked at the receiver when the sender requests a memory location
- * for writing a packet.
- * The interface was not in writing state.
- *
- * @param user value given to {@link BestEffortPacketWriteInterface_Init}
- * @param data if a memory location was provided, it must be returned here. It must have
- *             MTU bytes available. It may be set to NULL if MTU is 0.
- * @param return 1 - a memory location was provided. The interface will enter writing state.
- *               0 - a memory location was not provided
- */
-typedef int (*BestEffortPacketWriteInterface_handler_startpacket) (void *user, uint8_t **data);
-
-/**
- * Callback function invoked at the receiver when the sender has finished writing a packet.
- * The interface was in writing state before.
- * The interface enters not writing state after this function returns.
- *
- * @param user value given to {@link BestEffortPacketWriteInterface_Init}
- * @param len length of the packet written. Will be >=0 and <=MTU.
- */
-typedef void (*BestEffortPacketWriteInterface_handler_endpacket) (void *user, int len);
-
-/**
- * Interface which allows a sender to write packets to a buffer provided by the receiver
- * in a best-effort fashion.
- */
-typedef struct {
-    DebugObject d_obj;
-    int mtu;
-    BestEffortPacketWriteInterface_handler_startpacket handler_startpacket;
-    BestEffortPacketWriteInterface_handler_endpacket handler_endpacket;
-    void *user;
-    #ifndef NDEBUG
-    int sending;
-    int in_call;
-    dead_t dead;
-    #endif
-} BestEffortPacketWriteInterface;
-
-/**
- * Initializes the interface.
- * The interface is initialized in not writing state.
- *
- * @param i the object
- * @param mtu maximum packet size. Must be >=0.
- * @param handler_startpacket callback function invoked at the receiver when
- *                            the sender wants a memory location for writing a packet
- * @param handler_endpacket callback function invoked at the receiver when the sender
- *                          has finished writing a packet
- * @param user value passed to receiver callback functions
- */
-static void BestEffortPacketWriteInterface_Init (
-    BestEffortPacketWriteInterface *i,
-    int mtu,
-    BestEffortPacketWriteInterface_handler_startpacket handler_startpacket,
-    BestEffortPacketWriteInterface_handler_endpacket handler_endpacket,
-    void *user
-);
-
-/**
- * Frees the interface.
- *
- * @param i the object
- */
-static void BestEffortPacketWriteInterface_Free (BestEffortPacketWriteInterface *i);
-
-/**
- * Requests a memory location for writing a packet to the receiver.
- * The interface must be in not writing state.
- *
- * @param i the object
- * @param data if the function returns 1, will be set to the memory location where the
- * *           packet should be written. May be set to NULL if MTU is 0.
- * @param return - 1 a memory location was provided. The interface enters writing state.
- *               - 0 a memory location was not provided
- */
-static int BestEffortPacketWriteInterface_Sender_StartPacket (BestEffortPacketWriteInterface *i, uint8_t **data);
-
-/**
- * Sumbits a packet written to the memory location provided by the receiver.
- * The interface must be in writing state.
- * The interface enters not writing state.
- *
- * @param i the object
- * @param len length of the packet written. Must be >=0 and <=MTU.
- */
-static void BestEffortPacketWriteInterface_Sender_EndPacket (BestEffortPacketWriteInterface *i, int len);
-
-void BestEffortPacketWriteInterface_Init (
-    BestEffortPacketWriteInterface *i,
-    int mtu,
-    BestEffortPacketWriteInterface_handler_startpacket handler_startpacket,
-    BestEffortPacketWriteInterface_handler_endpacket handler_endpacket,
-    void *user
-)
-{
-    ASSERT(mtu >= 0)
-    
-    // init arguments
-    i->mtu = mtu;
-    i->handler_startpacket = handler_startpacket;
-    i->handler_endpacket = handler_endpacket;
-    i->user = user;
-    
-    // init debug object
-    DebugObject_Init(&i->d_obj);
-    
-    // init debugging
-    #ifndef NDEBUG
-    i->sending = 0;
-    i->in_call = 0;
-    DEAD_INIT(i->dead);
-    #endif
-}
-
-void BestEffortPacketWriteInterface_Free (BestEffortPacketWriteInterface *i)
-{
-    // free debug object
-    DebugObject_Free(&i->d_obj);
-    
-    // free debugging
-    #ifndef NDEBUG
-    DEAD_KILL(i->dead);
-    #endif
-}
-
-int BestEffortPacketWriteInterface_Sender_StartPacket (BestEffortPacketWriteInterface *i, uint8_t **data)
-{
-    ASSERT(!i->sending)
-    ASSERT(!i->in_call)
-    
-    #ifndef NDEBUG
-    i->in_call = 1;
-    DEAD_ENTER(i->dead)
-    #endif
-    
-    int res = i->handler_startpacket(i->user, data);
-    
-    #ifndef NDEBUG
-    if (DEAD_LEAVE(i->dead)) {
-        return -1;
-    }
-    ASSERT(res == 0 || res == 1)
-    i->in_call = 0;
-    if (res) {
-        i->sending = 1;
-    }
-    #endif
-    
-    return res;
-}
-
-void BestEffortPacketWriteInterface_Sender_EndPacket (BestEffortPacketWriteInterface *i, int len)
-{
-    ASSERT(len >= 0)
-    ASSERT(len <= i->mtu)
-    ASSERT(i->sending)
-    ASSERT(!i->in_call)
-    
-    #ifndef NDEBUG
-    i->in_call = 1;
-    DEAD_ENTER(i->dead)
-    #endif
-    
-    i->handler_endpacket(i->user, len);
-    
-    #ifndef NDEBUG
-    if (DEAD_LEAVE(i->dead)) {
-        return;
-    }
-    i->in_call = 0;
-    i->sending = 0;
-    #endif
-}
-
-#endif

flow/BufferWriter.c

@@ -0,0 +1,103 @@
+/**
+ * @file BufferWriter.c
+ * @author Ambroz Bizjak <[email protected]>
+ * 
+ * @section LICENSE
+ * 
+ * This file is part of BadVPN.
+ * 
+ * BadVPN is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ * 
+ * BadVPN is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <misc/debug.h>
+
+#include <flow/BufferWriter.h>
+
+static void output_handler_recv (BufferWriter *o, uint8_t *data)
+{
+    ASSERT(!o->out_have)
+    
+    // set output packet
+    o->out_have = 1;
+    o->out = data;
+}
+
+void BufferWriter_Init (BufferWriter *o, int mtu, BPendingGroup *pg)
+{
+    ASSERT(mtu >= 0)
+    
+    // init output
+    PacketRecvInterface_Init(&o->recv_interface, mtu, (PacketRecvInterface_handler_recv)output_handler_recv, o, pg);
+    
+    // set no output packet
+    o->out_have = 0;
+    
+    DebugObject_Init(&o->d_obj);
+    #ifndef NDEBUG
+    o->d_mtu = mtu;
+    o->d_writing = 0;
+    #endif
+}
+
+void BufferWriter_Free (BufferWriter *o)
+{
+    DebugObject_Free(&o->d_obj);
+    
+    // free output
+    PacketRecvInterface_Free(&o->recv_interface);
+}
+
+PacketRecvInterface * BufferWriter_GetOutput (BufferWriter *o)
+{
+    DebugObject_Access(&o->d_obj);
+    
+    return &o->recv_interface;
+}
+
+int BufferWriter_StartPacket (BufferWriter *o, uint8_t **buf)
+{
+    ASSERT(!o->d_writing)
+    DebugObject_Access(&o->d_obj);
+    
+    if (!o->out_have) {
+        return 0;
+    }
+    
+    if (buf) {
+        *buf = o->out;
+    }
+    
+    #ifndef NDEBUG
+    o->d_writing = 1;
+    #endif
+    
+    return 1;
+}
+
+void BufferWriter_EndPacket (BufferWriter *o, int len)
+{
+    ASSERT(o->out_have)
+    ASSERT(o->d_writing)
+    DebugObject_Access(&o->d_obj);
+    
+    // set no output packet
+    o->out_have = 0;
+    
+    // finish packet
+    PacketRecvInterface_Done(&o->recv_interface, len);
+    
+    #ifndef NDEBUG
+    o->d_writing = 0;
+    #endif
+}

flow/BufferWriter.h

@@ -0,0 +1,99 @@
+/**
+ * @file BufferWriter.h
+ * @author Ambroz Bizjak <[email protected]>
+ * 
+ * @section LICENSE
+ * 
+ * This file is part of BadVPN.
+ * 
+ * BadVPN is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ * 
+ * BadVPN is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ * 
+ * @section DESCRIPTION
+ * 
+ * Object for writing packets to a {@link PacketRecvInterface} client
+ * in a best-effort fashion.
+ */
+
+#ifndef BADVPN_FLOW_BUFFERWRITER_H
+#define BADVPN_FLOW_BUFFERWRITER_H
+
+#include <stdint.h>
+
+#include <system/DebugObject.h>
+#include <flow/PacketRecvInterface.h>
+
+/**
+ * Object for writing packets to a {@link PacketRecvInterface} client
+ * in a best-effort fashion.
+ */
+typedef struct {
+    PacketRecvInterface recv_interface;
+    int out_have;
+    uint8_t *out;
+    DebugObject d_obj;
+    #ifndef NDEBUG
+    int d_mtu;
+    int d_writing;
+    #endif
+} BufferWriter;
+
+/**
+ * Initializes the object.
+ * The object is initialized in not writing state.
+ *
+ * @param o the object
+ * @param mtu maximum input packet length
+ * @param pg pending group
+ */
+void BufferWriter_Init (BufferWriter *o, int mtu, BPendingGroup *pg);
+
+/**
+ * Frees the object.
+ *
+ * @param o the object
+ */
+void BufferWriter_Free (BufferWriter *o);
+
+/**
+ * Returns the output interface.
+ *
+ * @param o the object
+ * @return output interface
+ */
+PacketRecvInterface * BufferWriter_GetOutput (BufferWriter *o);
+
+/**
+ * Attempts to provide a memory location for writing a packet.
+ * The object must be in not writing state.
+ * On success, the object enters writing state.
+ * 
+ * @param o the object
+ * @param buf if not NULL, on success, the memory location will be stored here.
+ *            It will have space for MTU bytes.
+ * @return 1 on success, 0 on failure
+ */
+int BufferWriter_StartPacket (BufferWriter *o, uint8_t **buf);
+
+/**
+ * Submits a packet written to the buffer.
+ * The object must be in writing state.
+ * Yhe object enters not writing state.
+ * 
+ * @param o the object
+ * @param len length of the packet that was written. Must be >=0 and
+ *            <=MTU.
+ */
+void BufferWriter_EndPacket (BufferWriter *o, int len);
+
+#endif

flow/CMakeLists.txt

@@ -27,5 +27,10 @@ add_library(flow
     DataProtoKeepaliveSource.c
     PacketProtoFlow.c
     SinglePacketSender.c
+    BufferWriter.c
+    PacketPassInterface.c
+    PacketRecvInterface.c
+    StreamPassInterface.c
+    StreamRecvInterface.c
 )
 target_link_libraries(flow system security)

flow/DataProtoKeepaliveSource.c

@@ -25,29 +25,29 @@
 
 #include <flow/DataProtoKeepaliveSource.h>
 
-static int output_handler_recv (DataProtoKeepaliveSource *o, uint8_t *data, int *data_len)
+static void output_handler_recv (DataProtoKeepaliveSource *o, uint8_t *data)
 {
+    DebugObject_Access(&o->d_obj);
+    
     struct dataproto_header *header = (struct dataproto_header *)data;
     header->flags = 0;
     header->from_id = htol16(0);
     header->num_peer_ids = htol16(0);
     
-    *data_len = sizeof(struct dataproto_header);
-    return 1;
+    // finish packet
+    PacketRecvInterface_Done(&o->output, sizeof(struct dataproto_header));
 }
 
-void DataProtoKeepaliveSource_Init (DataProtoKeepaliveSource *o)
+void DataProtoKeepaliveSource_Init (DataProtoKeepaliveSource *o, BPendingGroup *pg)
 {
     // init output
-    PacketRecvInterface_Init(&o->output, sizeof(struct dataproto_header), (PacketRecvInterface_handler_recv)output_handler_recv, o);
+    PacketRecvInterface_Init(&o->output, sizeof(struct dataproto_header), (PacketRecvInterface_handler_recv)output_handler_recv, o, pg);
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
 void DataProtoKeepaliveSource_Free (DataProtoKeepaliveSource *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // free output
@@ -56,5 +56,7 @@ void DataProtoKeepaliveSource_Free (DataProtoKeepaliveSource *o)
 
 PacketRecvInterface * DataProtoKeepaliveSource_GetOutput (DataProtoKeepaliveSource *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->output;
 }

flow/DataProtoKeepaliveSource.h

@@ -43,8 +43,9 @@ typedef struct {
  * Initializes the object.
  *
  * @param o the object
+ * @param pg pending group
  */
-void DataProtoKeepaliveSource_Init (DataProtoKeepaliveSource *o);
+void DataProtoKeepaliveSource_Init (DataProtoKeepaliveSource *o, BPendingGroup *pg);
 
 /**
  * Frees the object.

flow/DatagramSocketSink.c

@@ -24,85 +24,73 @@
 
 #include <flow/DatagramSocketSink.h>
 
-static int report_error (DatagramSocketSink *s, int error)
+static void report_error (DatagramSocketSink *s, int error)
 {
-    #ifndef NDEBUG
-    s->in_error = 1;
-    #endif
-    
+    DebugIn_GoIn(&s->d_in_error);
     DEAD_ENTER(s->dead)
     FlowErrorReporter_ReportError(&s->rep, &error);
     if (DEAD_LEAVE(s->dead)) {
-        return -1;
+        return;
     }
+    DebugIn_GoOut(&s->d_in_error);
+}
+
+static void try_send (DatagramSocketSink *s)
+{
+    ASSERT(s->in_len >= 0)
     
-    #ifndef NDEBUG
-    s->in_error = 0;
-    #endif
+    int res = BSocket_SendToFrom(s->bsock, s->in, s->in_len, &s->addr, &s->local_addr);
+    if (res < 0 && BSocket_GetError(s->bsock) == BSOCKET_ERROR_LATER) {
+        // wait for socket in socket_handler
+        BSocket_EnableEvent(s->bsock, BSOCKET_WRITE);
+        return;
+    }
     
-    return 0;
+    int old_len = s->in_len;
+    
+    // finish packet
+    s->in_len = -1;
+    PacketPassInterface_Done(&s->input);
+    
+    if (res < 0) {
+        report_error(s, DATAGRAMSOCKETSINK_ERROR_BSOCKET);
+        return;
+    }
+    else if (res != old_len) {
+        report_error(s, DATAGRAMSOCKETSINK_ERROR_WRONGSIZE);
+        return;
+    }
 }
 
-static int input_handler_send (DatagramSocketSink *s, uint8_t *data, int data_len)
+static void input_handler_send (DatagramSocketSink *s, uint8_t *data, int data_len)
 {
     ASSERT(s->in_len == -1)
     ASSERT(data_len >= 0)
-    ASSERT(!s->in_error)
+    DebugIn_AmOut(&s->d_in_error);
+    DebugObject_Access(&s->d_obj);
     
-    int res = BSocket_SendToFrom(s->bsock, data, data_len, &s->addr, &s->local_addr);
-    if (res < 0) {
-        int error = BSocket_GetError(s->bsock);
-        if (error == BSOCKET_ERROR_LATER) {
-            s->in_len = data_len;
-            s->in = data;
-            BSocket_EnableEvent(s->bsock, BSOCKET_WRITE);
-            return 0;
-        }
-        if (report_error(s, DATAGRAMSOCKETSINK_ERROR_BSOCKET) < 0) {
-            return -1;
-        }
-    } else {
-        if (res != data_len) {
-            if (report_error(s, DATAGRAMSOCKETSINK_ERROR_WRONGSIZE) < 0) {
-                return -1;
-            }
-        }
-    }
+    // set packet
+    s->in_len = data_len;
+    s->in = data;
     
-    return 1;
+    try_send(s);
+    return;
 }
 
 static void socket_handler (DatagramSocketSink *s, int event)
 {
     ASSERT(s->in_len >= 0)
     ASSERT(event == BSOCKET_WRITE)
-    ASSERT(!s->in_error)
-    
-    int res = BSocket_SendToFrom(s->bsock, s->in, s->in_len, &s->addr, &s->local_addr);
-    if (res < 0) {
-        int error = BSocket_GetError(s->bsock);
-        if (error == BSOCKET_ERROR_LATER) {
-            return;
-        }
-        if (report_error(s, DATAGRAMSOCKETSINK_ERROR_BSOCKET) < 0) {
-            return;
-        }
-    } else {
-        if (res != s->in_len) {
-            if (report_error(s, DATAGRAMSOCKETSINK_ERROR_WRONGSIZE) < 0) {
-                return;
-            }
-        }
-    }
+    DebugIn_AmOut(&s->d_in_error);
+    DebugObject_Access(&s->d_obj);
     
     BSocket_DisableEvent(s->bsock, BSOCKET_WRITE);
-    s->in_len = -1;
     
-    PacketPassInterface_Done(&s->input);
+    try_send(s);
     return;
 }
 
-void DatagramSocketSink_Init (DatagramSocketSink *s, FlowErrorReporter rep, BSocket *bsock, int mtu, BAddr addr, BIPAddr local_addr)
+void DatagramSocketSink_Init (DatagramSocketSink *s, FlowErrorReporter rep, BSocket *bsock, int mtu, BAddr addr, BIPAddr local_addr, BPendingGroup *pg)
 {
     ASSERT(mtu >= 0)
     ASSERT(BAddr_IsRecognized(&addr) && !BAddr_IsInvalid(&addr))
@@ -121,23 +109,17 @@ void DatagramSocketSink_Init (DatagramSocketSink *s, FlowErrorReporter rep, BSoc
     BSocket_AddEventHandler(s->bsock, BSOCKET_WRITE, (BSocket_handler)socket_handler, s);
     
     // init input
-    PacketPassInterface_Init(&s->input, mtu, (PacketPassInterface_handler_send)input_handler_send, s);
+    PacketPassInterface_Init(&s->input, mtu, (PacketPassInterface_handler_send)input_handler_send, s, pg);
     
     // have no input packet
     s->in_len = -1;
     
-    // init debugging
-    #ifndef NDEBUG
-    s->in_error = 0;
-    #endif
-    
-    // init debug object
+    DebugIn_Init(&s->d_in_error);
     DebugObject_Init(&s->d_obj);
 }
 
 void DatagramSocketSink_Free (DatagramSocketSink *s)
 {
-    // free debug object
     DebugObject_Free(&s->d_obj);
 
     // free input
@@ -152,6 +134,8 @@ void DatagramSocketSink_Free (DatagramSocketSink *s)
 
 PacketPassInterface * DatagramSocketSink_GetInput (DatagramSocketSink *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     return &s->input;
 }
 
@@ -159,6 +143,7 @@ void DatagramSocketSink_SetAddresses (DatagramSocketSink *s, BAddr addr, BIPAddr
 {
     ASSERT(BAddr_IsRecognized(&addr) && !BAddr_IsInvalid(&addr))
     ASSERT(BIPAddr_IsRecognized(&local_addr))
+    DebugObject_Access(&s->d_obj);
     
     s->addr = addr;
     s->local_addr = local_addr;

flow/DatagramSocketSink.h

@@ -30,6 +30,7 @@
 #include <stdint.h>
 
 #include <misc/dead.h>
+#include <misc/debugin.h>
 #include <system/DebugObject.h>
 #include <system/BSocket.h>
 #include <flow/PacketPassInterface.h>
@@ -42,7 +43,6 @@
  * A {@link PacketPassInterface} sink which sends packets to a datagram socket.
  */
 typedef struct {
-    DebugObject d_obj;
     dead_t dead;
     FlowErrorReporter rep;
     BSocket *bsock;
@@ -51,9 +51,8 @@ typedef struct {
     PacketPassInterface input;
     int in_len;
     uint8_t *in;
-    #ifndef NDEBUG
-    int in_error;
-    #endif
+    DebugIn d_in_error;
+    DebugObject d_obj;
 } DatagramSocketSink;
 
 /**
@@ -73,8 +72,9 @@ typedef struct {
  * @param addr remote address. Must be recognized and valid. Passed to {@link BSocket_SendToFrom}.
  * @param local_addr source address. Must be recognized.
  *                   Passed to {@link BSocket_SendToFrom}.
+ * @param pg pending group
  */
-void DatagramSocketSink_Init (DatagramSocketSink *s, FlowErrorReporter rep, BSocket *bsock, int mtu, BAddr addr, BIPAddr local_addr);
+void DatagramSocketSink_Init (DatagramSocketSink *s, FlowErrorReporter rep, BSocket *bsock, int mtu, BAddr addr, BIPAddr local_addr, BPendingGroup *pg);
 
 /**
  * Frees the sink.

flow/DatagramSocketSource.c

@@ -20,100 +20,88 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
 
-#include <stdlib.h>
-
 #include <misc/debug.h>
 
 #include <flow/DatagramSocketSource.h>
 
-static int report_error (DatagramSocketSource *s, int error)
+static void report_error (DatagramSocketSource *s, int error)
 {
-    #ifndef NDEBUG
-    s->in_error = 1;
-    #endif
-    
+    DebugIn_GoIn(&s->d_in_error);
     DEAD_ENTER(s->dead)
     FlowErrorReporter_ReportError(&s->rep, &error);
     if (DEAD_LEAVE(s->dead)) {
-        return -1;
+        return;
+    }
+    DebugIn_GoOut(&s->d_in_error);
+}
+
+static void try_recv (DatagramSocketSource *s)
+{
+    ASSERT(s->out_have)
+    
+    int res = BSocket_RecvFromTo(s->bsock, s->out, s->mtu, &s->last_addr, &s->last_local_addr);
+    if (res < 0 && BSocket_GetError(s->bsock) == BSOCKET_ERROR_LATER) {
+        // wait for socket in socket_handler
+        BSocket_EnableEvent(s->bsock, BSOCKET_READ);
+        return;
+    }
+    
+    if (res < 0) {
+        // schedule retry
+        BPending_Set(&s->retry_job);
+        
+        // report error
+        report_error(s, DATAGRAMSOCKETSOURCE_ERROR_BSOCKET);
+        return;
     }
     
     #ifndef NDEBUG
-    s->in_error = 0;
+    s->have_last_addr = 1;
     #endif
     
-    return 0;
+    // finish packet
+    s->out_have = 0;
+    PacketRecvInterface_Done(&s->output, res);
 }
 
-static int output_handler_recv (DatagramSocketSource *s, uint8_t *data, int *data_len)
+static void output_handler_recv (DatagramSocketSource *s, uint8_t *data)
 {
     ASSERT(!s->out_have)
-    ASSERT(!s->in_error)
-    
-    int res;
-    
-    while (1) {
-        res = BSocket_RecvFromTo(s->bsock, data, s->mtu, &s->last_addr, &s->last_local_addr);
-        if (res < 0) {
-            int error = BSocket_GetError(s->bsock);
-            if (error == BSOCKET_ERROR_LATER) {
-                s->out_have = 1;
-                s->out = data;
-                BSocket_EnableEvent(s->bsock, BSOCKET_READ);
-                return 0;
-            }
-            if (report_error(s, DATAGRAMSOCKETSOURCE_ERROR_BSOCKET) < 0) {
-                return -1;
-            }
-            continue;
-        }
-        break;
-    }
+    DebugIn_AmOut(&s->d_in_error);
+    DebugObject_Access(&s->d_obj);
     
-    #ifndef NDEBUG
-    s->have_last_addr = 1;
-    #endif
+    // set packet
+    s->out_have = 1;
+    s->out = data;
     
-    *data_len = res;
-    return 1;
+    try_recv(s);
+    return;
 }
 
 static void socket_handler (DatagramSocketSource *s, int event)
 {
     ASSERT(s->out_have)
     ASSERT(event == BSOCKET_READ)
-    ASSERT(!s->in_error)
-    
-    int res;
-    
-    while (1) {
-        res = BSocket_RecvFromTo(s->bsock, s->out, s->mtu, &s->last_addr, &s->last_local_addr);
-        if (res < 0) {
-            int error = BSocket_GetError(s->bsock);
-            if (error == BSOCKET_ERROR_LATER) {
-                // nothing to receive, continue in socket_handler
-                return;
-            }
-            if (report_error(s, DATAGRAMSOCKETSOURCE_ERROR_BSOCKET) < 0) {
-                return;
-            }
-            continue;
-        }
-        break;
-    }
+    DebugIn_AmOut(&s->d_in_error);
+    DebugObject_Access(&s->d_obj);
     
     BSocket_DisableEvent(s->bsock, BSOCKET_READ);
-    s->out_have = 0;
     
-    #ifndef NDEBUG
-    s->have_last_addr = 1;
-    #endif
+    try_recv(s);
+    return;
+}
+
+static void retry_job_handler (DatagramSocketSource *s)
+{
+    ASSERT(s->out_have)
+    DebugIn_AmOut(&s->d_in_error);
+    DebugObject_Access(&s->d_obj);
     
-    PacketRecvInterface_Done(&s->output, res);
+    try_recv(s);
     return;
 }
 
-void DatagramSocketSource_Init (DatagramSocketSource *s, FlowErrorReporter rep, BSocket *bsock, int mtu)
+void DatagramSocketSource_Init (DatagramSocketSource *s, FlowErrorReporter rep, BSocket *bsock, int mtu, BPendingGroup *pg)
 {
     ASSERT(mtu >= 0)
     
@@ -129,26 +117,28 @@ void DatagramSocketSource_Init (DatagramSocketSource *s, FlowErrorReporter rep,
     BSocket_AddEventHandler(s->bsock, BSOCKET_READ, (BSocket_handler)socket_handler, s);
     
     // init output
-    PacketRecvInterface_Init(&s->output, mtu, (PacketRecvInterface_handler_recv)output_handler_recv, s);
+    PacketRecvInterface_Init(&s->output, mtu, (PacketRecvInterface_handler_recv)output_handler_recv, s, pg);
     
     // have no output packet
     s->out_have = 0;
     
-    // init debugging
+    // init retry job
+    BPending_Init(&s->retry_job, pg, (BPending_handler)retry_job_handler, s);
+    
+    DebugIn_Init(&s->d_in_error);
+    DebugObject_Init(&s->d_obj);
     #ifndef NDEBUG
     s->have_last_addr = 0;
-    s->in_error = 0;
     #endif
-    
-    // init debug object
-    DebugObject_Init(&s->d_obj);
 }
 
 void DatagramSocketSource_Free (DatagramSocketSource *s)
 {
-    // free debug object
     DebugObject_Free(&s->d_obj);
-
+    
+    // free retry job
+    BPending_Free(&s->retry_job);
+    
     // free output
     PacketRecvInterface_Free(&s->output);
     
@@ -161,12 +151,15 @@ void DatagramSocketSource_Free (DatagramSocketSource *s)
 
 PacketRecvInterface * DatagramSocketSource_GetOutput (DatagramSocketSource *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     return &s->output;
 }
 
 void DatagramSocketSource_GetLastAddresses (DatagramSocketSource *s, BAddr *addr, BIPAddr *local_addr)
 {
     ASSERT(s->have_last_addr)
+    DebugObject_Access(&s->d_obj);
     
     if (addr) {
         *addr = s->last_addr;

flow/DatagramSocketSource.h

@@ -27,11 +27,15 @@
 #ifndef BADVPN_FLOW_DATAGRAMSOCKETSOURCE_H
 #define BADVPN_FLOW_DATAGRAMSOCKETSOURCE_H
 
+#include <stdint.h>
+
 #include <misc/dead.h>
+#include <misc/debugin.h>
 #include <system/DebugObject.h>
 #include <system/BSocket.h>
-#include <flow/error.h>
+#include <system/BPending.h>
 #include <flow/PacketRecvInterface.h>
+#include <flow/error.h>
 
 #define DATAGRAMSOCKETSOURCE_ERROR_BSOCKET 1
 
@@ -39,7 +43,6 @@
  * A {@link PacketRecvInterface} source which receives packets from a datagram socket.
  */
 typedef struct {
-    DebugObject d_obj;
     dead_t dead;
     FlowErrorReporter rep;
     BSocket *bsock;
@@ -49,9 +52,11 @@ typedef struct {
     uint8_t *out;
     BAddr last_addr;
     BIPAddr last_local_addr;
+    BPending retry_job;
+    DebugIn d_in_error;
+    DebugObject d_obj;
     #ifndef NDEBUG
     int have_last_addr;
-    int in_error;
     #endif
 } DatagramSocketSource;
 
@@ -67,8 +72,9 @@ typedef struct {
  * @param bsock datagram socket to read data from. The BSOCKET_READ event must be disabled.
  * *            Takes over reading on the socket.
  * @param mtu maximum packet size. Must be >=0.
+ * @param pg pending group
  */
-void DatagramSocketSource_Init (DatagramSocketSource *s, FlowErrorReporter rep, BSocket *bsock, int mtu);
+void DatagramSocketSource_Init (DatagramSocketSource *s, FlowErrorReporter rep, BSocket *bsock, int mtu, BPendingGroup *pg);
 
 /**
  * Frees the object.

flow/FragmentProtoAssembler.c

@@ -152,9 +152,9 @@ static void reduce_times (FragmentProtoAssembler *o)
     o->time -= min_time;
 }
 
-static int process_chunk (FragmentProtoAssembler *o, fragmentproto_frameid frame_id, int chunk_start, int chunk_len, int is_last, uint8_t *payload)
+static void process_chunk (FragmentProtoAssembler *o, fragmentproto_frameid frame_id, int chunk_start, int chunk_len, int is_last, uint8_t *payload)
 {
-    ASSERT(!o->output_blocking)
+    ASSERT(!o->output_ready)
     ASSERT(chunk_start >= 0)
     ASSERT(chunk_len >= 0)
     ASSERT(is_last == 0 || is_last == 1)
@@ -164,13 +164,13 @@ static int process_chunk (FragmentProtoAssembler *o, fragmentproto_frameid frame
     // check start
     if (chunk_start > o->output_mtu) {
         BLog(BLOG_INFO, "chunk starts outside");
-        return 0;
+        return;
     }
     
     // check frame size bound
     if (chunk_len > o->output_mtu - chunk_start) {
         BLog(BLOG_INFO, "chunk ends outside");
-        return 0;
+        return;
     }
     
     // calculate end
@@ -262,7 +262,7 @@ static int process_chunk (FragmentProtoAssembler *o, fragmentproto_frameid frame
             BLog(BLOG_INFO, "all chunks used, but frame not complete");
             goto fail_frame;
         }
-        return 0;
+        return;
     }
     
     ASSERT(frame->sum == frame->length)
@@ -272,33 +272,20 @@ static int process_chunk (FragmentProtoAssembler *o, fragmentproto_frameid frame
     // free frame entry
     free_frame(o, frame);
     
-    // submit frame to output
-    // this is fine even though the frame entry was freed
-    DEAD_ENTER(o->dead)
-    int res = PacketPassInterface_Sender_Send(o->output, frame->buffer, frame->length);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    // if output blocked, don't accept any new input until it's done
-    if (!res) {
-        o->output_blocking = 1;
-        return 0;
-    }
-    
-    return 0;
+    // remember frame
+    o->output_ready = 1;
+    o->output_packet_data = frame->buffer;
+    o->output_packet_len = frame->length;
+    return;
     
 fail_frame:
     free_frame(o, frame);
-    return 0;
 }
 
-static int process_input (FragmentProtoAssembler *o)
+static void process_input (FragmentProtoAssembler *o)
 {
     ASSERT(o->in_len >= 0)
-    ASSERT(!o->output_blocking)
+    ASSERT(!o->output_ready)
     
     // read chunks
     while (o->in_pos < o->in_len) {
@@ -326,14 +313,12 @@ static int process_input (FragmentProtoAssembler *o)
         }
         
         // process chunk
-        if (process_chunk(o, frame_id, chunk_start, chunk_len, header->is_last, o->in + o->in_pos) < 0) {
-            return -1;
-        }
+        process_chunk(o, frame_id, chunk_start, chunk_len, header->is_last, o->in + o->in_pos);
         o->in_pos += chunk_len;
         
         // if output is blocking, stop processing input
-        if (o->output_blocking) {
-            return 0;
+        if (o->output_ready) {
+            return;
         }
     }
     
@@ -354,64 +339,54 @@ static int process_input (FragmentProtoAssembler *o)
     } else {
         o->time++;
     }
+}
+
+static void do_io (FragmentProtoAssembler *o)
+{
+    ASSERT(o->in_len >= 0)
+    ASSERT(!o->output_ready)
     
-    return 0;
+    // process input
+    process_input(o);
+    
+    ASSERT((o->in_len >= 0) == o->output_ready)
+    
+    if (o->output_ready) {
+        PacketPassInterface_Sender_Send(o->output, o->output_packet_data, o->output_packet_len);
+    } else {
+        PacketPassInterface_Done(&o->input);
+    }
 }
 
-static int input_handler_send (FragmentProtoAssembler *o, uint8_t *data, int data_len)
+static void input_handler_send (FragmentProtoAssembler *o, uint8_t *data, int data_len)
 {
     ASSERT(o->in_len == -1)
-    ASSERT(!o->output_blocking)
+    ASSERT(!o->output_ready)
     ASSERT(data_len >= 0)
     ASSERT(data_len <= o->input_mtu)
+    DebugObject_Access(&o->d_obj);
     
     // save input packet
     o->in_len = data_len;
     o->in = data;
     o->in_pos = 0;
     
-    // process input
-    if (process_input(o) < 0) {
-        return -1;
-    }
-    
-    ASSERT((o->in_len >= 0) == o->output_blocking)
-    
-    // if not all input was processed (output is blocking), block input
-    if (o->in_len >= 0) {
-        return 0;
-    }
-    
-    // all input was processed
-    return 1;
+    do_io(o);
 }
 
 static void output_handler_done (FragmentProtoAssembler *o)
 {
     ASSERT(o->in_len >= 0)
-    ASSERT(o->output_blocking)
+    ASSERT(o->output_ready)
+    DebugObject_Access(&o->d_obj);
     
     // output no longer blocking
-    o->output_blocking = 0;
+    o->output_ready = 0;
     
-    // process any further input
-    if (process_input(o) < 0) {
-        return;
-    }
-    
-    ASSERT((o->in_len >= 0) == o->output_blocking)
-    
-    // if output is again blocking, keep input blocked
-    if (o->in_len >= 0) {
-        return;
-    }
-    
-    // tell input we're done
-    PacketPassInterface_Done(&o->input);
-    return;
+    do_io(o);
 }
 
-int FragmentProtoAssembler_Init (FragmentProtoAssembler *o, int input_mtu, PacketPassInterface *output, int num_frames, int num_chunks)
+int FragmentProtoAssembler_Init (FragmentProtoAssembler *o, int input_mtu, PacketPassInterface *output, int num_frames, int num_chunks, BPendingGroup *pg)
 {
     ASSERT(input_mtu >= 0)
     ASSERT(num_frames > 0)
@@ -424,11 +399,8 @@ int FragmentProtoAssembler_Init (FragmentProtoAssembler *o, int input_mtu, Packe
     o->num_frames = num_frames;
     o->num_chunks = num_chunks;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init input
-    PacketPassInterface_Init(&o->input, o->input_mtu, (PacketPassInterface_handler_send)input_handler_send, o);
+    PacketPassInterface_Init(&o->input, o->input_mtu, (PacketPassInterface_handler_send)input_handler_send, o, pg);
     
     // init output
     PacketPassInterface_Sender_Init(o->output, (PacketPassInterface_handler_done)output_handler_done, o);
@@ -479,9 +451,8 @@ int FragmentProtoAssembler_Init (FragmentProtoAssembler *o, int input_mtu, Packe
     o->in_len = -1;
     
     // output not blocking
-    o->output_blocking = 0;
+    o->output_ready = 0;
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
     
     return 1;
@@ -497,7 +468,6 @@ fail1:
 
 void FragmentProtoAssembler_Free (FragmentProtoAssembler *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // free buffers
@@ -511,12 +481,11 @@ void FragmentProtoAssembler_Free (FragmentProtoAssembler *o)
     
     // free input
     PacketPassInterface_Free(&o->input);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }
 
 PacketPassInterface * FragmentProtoAssembler_GetInput (FragmentProtoAssembler *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->input;
 }

flow/FragmentProtoAssembler.h

@@ -30,7 +30,6 @@
 #include <stdint.h>
 
 #include <protocol/fragmentproto.h>
-#include <misc/dead.h>
 #include <misc/debug.h>
 #include <system/DebugObject.h>
 #include <structure/LinkedList2.h>
@@ -64,7 +63,6 @@ struct FragmentProtoAssembler_frame {
  */
 typedef struct {
     DebugObject d_obj;
-    dead_t dead;
     PacketPassInterface input;
     int input_mtu;
     PacketPassInterface *output;
@@ -82,7 +80,9 @@ typedef struct {
     int in_len;
     uint8_t *in;
     int in_pos;
-    int output_blocking;
+    int output_ready;
+    uint8_t *output_packet_data;
+    int output_packet_len;
 } FragmentProtoAssembler;
 
 /**
@@ -94,9 +94,10 @@ typedef struct {
  * @param output output interface
  * @param num_frames number of frames we can hold. Must be >0 and < UINT32_MAX.
  * @param num_chunks maximum number of chunks a frame can come in. Must be >0.
+ * @param pg pending group
  * @return 1 on success, 0 on failure
  */
-int FragmentProtoAssembler_Init (FragmentProtoAssembler *o, int input_mtu, PacketPassInterface *output, int num_frames, int num_chunks) WARN_UNUSED;
+int FragmentProtoAssembler_Init (FragmentProtoAssembler *o, int input_mtu, PacketPassInterface *output, int num_frames, int num_chunks, BPendingGroup *pg) WARN_UNUSED;
 
 /**
  * Frees the object.

flow/FragmentProtoDisassembler.c

@@ -107,10 +107,9 @@ static void write_chunks (FragmentProtoDisassembler *o)
     ASSERT(o->in_len < 0 || !o->out)
 }
 
-static int input_handler_send (FragmentProtoDisassembler *o, uint8_t *data, int data_len)
+static void input_handler_send (FragmentProtoDisassembler *o, uint8_t *data, int data_len)
 {
     ASSERT(o->in_len == -1)
-    ASSERT(!o->doing_send)
     ASSERT(data_len >= 0)
     ASSERT(data_len <= o->input_mtu)
     
@@ -119,80 +118,59 @@ static int input_handler_send (FragmentProtoDisassembler *o, uint8_t *data, int
     o->in = data;
     o->in_used = 0;
     
-    // if there is no output, block input
+    // if there is no output, wait for it
     if (!o->out) {
-        return 0;
+        return;
     }
     
     // write input to output
     write_chunks(o);
     
-    // if we finished the output packet and are not in recv, notify output
-    if (!o->out && !o->doing_recv) {
-        o->doing_send = 1;
-        DEAD_ENTER(o->dead)
-        PacketRecvInterface_Done(&o->output, o->out_used);
-        if (DEAD_LEAVE(o->dead)) {
-            return -1;
-        }
-        o->doing_send = 0;
+    // finish input packet if needed
+    if (o->in_len == -1) {
+        PacketPassInterface_Done(&o->input);
     }
     
-    // if we still have some input, block input
-    if (o->in_len >= 0) {
-        return 0;
+    // finish output packet if needed
+    if (!o->out) {
+        PacketRecvInterface_Done(&o->output, o->out_used);
     }
-    
-    // all input was processed, accept packet
-    return 1;
 }
 
 static void input_handler_cancel (FragmentProtoDisassembler *o)
 {
     ASSERT(o->in_len >= 0)
     ASSERT(!o->out)
-    ASSERT(!o->doing_send)
     
     o->in_len = -1;
 }
 
-static int output_handler_recv (FragmentProtoDisassembler *o, uint8_t *data, int *data_len)
+static void output_handler_recv (FragmentProtoDisassembler *o, uint8_t *data)
 {
     ASSERT(!o->out)
-    ASSERT(!o->doing_recv)
     ASSERT(data)
     
     // set output packet
     o->out = data;
     o->out_used = 0;
     
-    // if there is no input, block output
+    // if there is no input, wait for it
     if (o->in_len < 0) {
-        return 0;
+        return;
     }
     
     // write input to output
     write_chunks(o);
     
-    // if we finished the input packet and are not in send, notify input
-    if (o->in_len < 0 && !o->doing_send) {
-        o->doing_recv = 1;
-        DEAD_ENTER(o->dead)
-        PacketPassInterface_Done(&o->input);
-        if (DEAD_LEAVE(o->dead)) {
-            return -1;
-        }
-        o->doing_recv = 0;
+    // finish output packet if needed
+    if (!o->out) {
+        PacketRecvInterface_Done(&o->output, o->out_used);
     }
     
-    // if we are not going to finish the output packet now, block output
-    if (o->out) {
-        return 0;
+    // finish input packet if needed
+    if (o->in_len == -1) {
+        PacketPassInterface_Done(&o->input);
     }
-    
-    // return packet now
-    *data_len = o->out_used;
-    return 1;
 }
 
 static void timer_handler (FragmentProtoDisassembler *o)
@@ -200,15 +178,10 @@ static void timer_handler (FragmentProtoDisassembler *o)
     ASSERT(o->latency >= 0)
     ASSERT(o->out)
     ASSERT(o->in_len = -1)
-    ASSERT(!o->doing_send)
-    ASSERT(!o->doing_recv)
     
     // finish output packet
     o->out = NULL;
-    
-    // inform output
     PacketRecvInterface_Done(&o->output, o->out_used);
-    return;
 }
 
 void FragmentProtoDisassembler_Init (FragmentProtoDisassembler *o, BReactor *reactor, int input_mtu, int output_mtu, int chunk_mtu, btime_t latency)
@@ -225,15 +198,12 @@ void FragmentProtoDisassembler_Init (FragmentProtoDisassembler *o, BReactor *rea
     o->chunk_mtu = chunk_mtu;
     o->latency = latency;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init input
-    PacketPassInterface_Init(&o->input, o->input_mtu, (PacketPassInterface_handler_send)input_handler_send, o);
+    PacketPassInterface_Init(&o->input, o->input_mtu, (PacketPassInterface_handler_send)input_handler_send, o, BReactor_PendingGroup(reactor));
     PacketPassInterface_EnableCancel(&o->input, (PacketPassInterface_handler_cancel)input_handler_cancel);
     
     // init output
-    PacketRecvInterface_Init(&o->output, o->output_mtu, (PacketRecvInterface_handler_recv)output_handler_recv, o);
+    PacketRecvInterface_Init(&o->output, o->output_mtu, (PacketRecvInterface_handler_recv)output_handler_recv, o, BReactor_PendingGroup(reactor));
     
     // init timer
     if (o->latency >= 0) {
@@ -249,19 +219,11 @@ void FragmentProtoDisassembler_Init (FragmentProtoDisassembler *o, BReactor *rea
     // start with zero frame ID
     o->frame_id = 0;
     
-    // not callback from send
-    o->doing_send = 0;
-    
-    // not callback from recv
-    o->doing_recv = 0;
-    
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
 void FragmentProtoDisassembler_Free (FragmentProtoDisassembler *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // free timer
@@ -274,17 +236,18 @@ void FragmentProtoDisassembler_Free (FragmentProtoDisassembler *o)
     
     // free input
     PacketPassInterface_Free(&o->input);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }
 
 PacketPassInterface * FragmentProtoDisassembler_GetInput (FragmentProtoDisassembler *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->input;
 }
 
 PacketRecvInterface * FragmentProtoDisassembler_GetOutput (FragmentProtoDisassembler *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->output;
 }

flow/FragmentProtoDisassembler.h

@@ -31,7 +31,6 @@
 #include <stdint.h>
 
 #include <protocol/fragmentproto.h>
-#include <misc/dead.h>
 #include <system/DebugObject.h>
 #include <system/BReactor.h>
 #include <system/BTime.h>
@@ -46,8 +45,6 @@
  * Output is with {@link PacketRecvInterface}.
  */
 typedef struct {
-    DebugObject d_obj;
-    dead_t dead;
     BReactor *reactor;
     int input_mtu;
     int output_mtu;
@@ -62,8 +59,7 @@ typedef struct {
     uint8_t *out;
     int out_used;
     fragmentproto_frameid frame_id;
-    int doing_send;
-    int doing_recv;
+    DebugObject d_obj;
 } FragmentProtoDisassembler;
 
 /**

flow/KeepaliveIO.c

@@ -26,8 +26,9 @@
 
 static void keepalive_handler (KeepaliveIO *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     PacketRecvBlocker_AllowBlockedPacket(&o->ka_blocker);
-    return;
 }
 
 int KeepaliveIO_Init (KeepaliveIO *o, BReactor *reactor, PacketPassInterface *output, PacketRecvInterface *keepalive_input, btime_t keepalive_interval_ms)
@@ -38,9 +39,6 @@ int KeepaliveIO_Init (KeepaliveIO *o, BReactor *reactor, PacketPassInterface *ou
     // set arguments
     o->reactor = reactor;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init keep-alive sender
     PacketPassInactivityMonitor_Init(&o->kasender, output, o->reactor, keepalive_interval_ms, (PacketPassInactivityMonitor_handler)keepalive_handler, o);
     
@@ -51,7 +49,7 @@ int KeepaliveIO_Init (KeepaliveIO *o, BReactor *reactor, PacketPassInterface *ou
     PacketPassPriorityQueueFlow_Init(&o->ka_qflow, &o->queue, -1);
     
     // init keepalive blocker
-    PacketRecvBlocker_Init(&o->ka_blocker, keepalive_input);
+    PacketRecvBlocker_Init(&o->ka_blocker, keepalive_input, BReactor_PendingGroup(reactor));
     
     // init keepalive buffer
     if (!SinglePacketBuffer_Init(&o->ka_buffer, PacketRecvBlocker_GetOutput(&o->ka_blocker), PacketPassPriorityQueueFlow_GetInput(&o->ka_qflow), BReactor_PendingGroup(o->reactor))) {
@@ -61,7 +59,6 @@ int KeepaliveIO_Init (KeepaliveIO *o, BReactor *reactor, PacketPassInterface *ou
     // init user flow
     PacketPassPriorityQueueFlow_Init(&o->user_qflow, &o->queue, 0);
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
     
     return 1;
@@ -76,7 +73,6 @@ fail1:
 
 void KeepaliveIO_Free (KeepaliveIO *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // allow freeing queue flows
@@ -99,12 +95,11 @@ void KeepaliveIO_Free (KeepaliveIO *o)
     
     // free keep-alive sender
     PacketPassInactivityMonitor_Free(&o->kasender);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }
 
 PacketPassInterface * KeepaliveIO_GetInput (KeepaliveIO *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return PacketPassPriorityQueueFlow_GetInput(&o->user_qflow);
 }

flow/KeepaliveIO.h

@@ -27,7 +27,6 @@
 #ifndef BADVPN_FLOW_KEEPALIVEIO
 #define BADVPN_FLOW_KEEPALIVEIO
 
-#include <misc/dead.h>
 #include <misc/debug.h>
 #include <system/DebugObject.h>
 #include <system/BReactor.h>
@@ -42,8 +41,6 @@
  * A {@link PacketPassInterface} layer for sending keep-alive packets.
  */
 typedef struct {
-    DebugObject d_obj;
-    dead_t dead;
     BReactor *reactor;
     PacketPassInactivityMonitor kasender;
     PacketPassPriorityQueue queue;
@@ -51,6 +48,7 @@ typedef struct {
     PacketPassPriorityQueueFlow ka_qflow;
     SinglePacketBuffer ka_buffer;
     PacketRecvBlocker ka_blocker;
+    DebugObject d_obj;
 } KeepaliveIO;
 
 /**

flow/PacketBuffer.c

@@ -26,117 +26,14 @@
 
 #include <flow/PacketBuffer.h>
 
-static int call_recv (PacketBuffer *buf, uint8_t *data, int *len);
-static int call_send (PacketBuffer *buf, uint8_t *data, int len);
-static int try_recv (PacketBuffer *buf);
-static int try_send (PacketBuffer *buf);
 static void input_handler_done (PacketBuffer *buf, int in_len);
 static void output_handler_done (PacketBuffer *buf);
-static void job_handler (PacketBuffer *buf);
-
-int call_recv (PacketBuffer *buf, uint8_t *data, int *len)
-{
-    ASSERT(!PacketRecvInterface_InClient(buf->input))
-    
-    DEAD_ENTER(buf->dead)
-    int res = PacketRecvInterface_Receiver_Recv(buf->input, data, len);
-    if (DEAD_LEAVE(buf->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    if (res) {
-        ASSERT(*len >= 0)
-        ASSERT(*len <= buf->input_mtu)
-    }
-    
-    return res;
-}
-
-int call_send (PacketBuffer *buf, uint8_t *data, int len)
-{
-    ASSERT(len >= 0)
-    ASSERT(len <= buf->input_mtu)
-    ASSERT(!PacketPassInterface_InClient(buf->output))
-    
-    DEAD_ENTER(buf->dead)
-    int res = PacketPassInterface_Sender_Send(buf->output, data, len);
-    if (DEAD_LEAVE(buf->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    return res;
-}
-
-int try_recv (PacketBuffer *buf)
-{
-    ASSERT(buf->buf.input_avail >= buf->input_mtu)
-    ASSERT(!PacketRecvInterface_InClient(buf->input))
-    ASSERT(!PacketPassInterface_InClient(buf->output))
-    
-    do {
-        // receive packet
-        int in_len;
-        int res;
-        if ((res = call_recv(buf, buf->buf.input_dest, &in_len)) < 0) {
-            return -1;
-        }
-        
-        if (!res) {
-            // input busy, continue in input_handler_done
-            return 0;
-        }
-        
-        // remember if buffer is empty
-        int was_empty = (buf->buf.output_avail < 0);
-        
-        // submit packet to buffer
-        ChunkBuffer2_SubmitPacket(&buf->buf, in_len);
-        
-        // if buffer was empty, start sending
-        if (was_empty) {
-            if (try_send(buf) < 0) {
-                return -1;
-            }
-        }
-    } while (buf->buf.input_avail >= buf->input_mtu);
-    
-    return 0;
-}
-
-int try_send (PacketBuffer *buf)
-{
-    ASSERT(buf->buf.output_avail >= 0)
-    ASSERT(!PacketRecvInterface_InClient(buf->input))
-    ASSERT(!PacketPassInterface_InClient(buf->output))
-    
-    do {
-        // send packet
-        int res;
-        if ((res = call_send(buf, buf->buf.output_dest, buf->buf.output_avail)) < 0) {
-            return -1;
-        }
-        
-        if (!res) {
-            // output busy, continue in output_handler_done
-            return 0;
-        }
-        
-        // remove packet from buffer
-        ChunkBuffer2_ConsumePacket(&buf->buf);
-    } while (buf->buf.output_avail >= 0);
-    
-    return 0;
-}
 
 void input_handler_done (PacketBuffer *buf, int in_len)
 {
     ASSERT(in_len >= 0)
     ASSERT(in_len <= buf->input_mtu)
-    ASSERT(!PacketRecvInterface_InClient(buf->input))
-    ASSERT(!PacketPassInterface_InClient(buf->output))
+    DebugObject_Access(&buf->d_obj);
     
     // remember if buffer is empty
     int was_empty = (buf->buf.output_avail < 0);
@@ -144,24 +41,20 @@ void input_handler_done (PacketBuffer *buf, int in_len)
     // submit packet to buffer
     ChunkBuffer2_SubmitPacket(&buf->buf, in_len);
     
-    // if buffer was empty, try sending
+    // if buffer was empty, schedule send
     if (was_empty) {
-        if (try_send(buf) < 0) {
-            return;
-        }
+        PacketPassInterface_Sender_Send(buf->output, buf->buf.output_dest, buf->buf.output_avail);
     }
     
-    // try receiving more
+    // if there is space, schedule receive
     if (buf->buf.input_avail >= buf->input_mtu) {
-        try_recv(buf);
-        return;
+        PacketRecvInterface_Receiver_Recv(buf->input, buf->buf.input_dest);
     }
 }
 
 void output_handler_done (PacketBuffer *buf)
 {
-    ASSERT(!PacketRecvInterface_InClient(buf->input))
-    ASSERT(!PacketPassInterface_InClient(buf->output))
+    DebugObject_Access(&buf->d_obj);
     
     // remember if buffer is full
     int was_full = (buf->buf.input_avail < buf->input_mtu);
@@ -169,26 +62,17 @@ void output_handler_done (PacketBuffer *buf)
     // remove packet from buffer
     ChunkBuffer2_ConsumePacket(&buf->buf);
     
-    // try sending more
+    // if there is more data, schedule send
     if (buf->buf.output_avail >= 0) {
-        if (try_send(buf) < 0) {
-            return;
-        }
+        PacketPassInterface_Sender_Send(buf->output, buf->buf.output_dest, buf->buf.output_avail);
     }
     
-    // try receiving
+    // if buffer was full and there is space, schedule receive
     if (was_full && buf->buf.input_avail >= buf->input_mtu) {
-        try_recv(buf);
-        return;
+        PacketRecvInterface_Receiver_Recv(buf->input, buf->buf.input_dest);
     }
 }
 
-void job_handler (PacketBuffer *buf)
-{
-    try_recv(buf);
-    return;
-}
-
 int PacketBuffer_Init (PacketBuffer *buf, PacketRecvInterface *input, PacketPassInterface *output, int num_packets, BPendingGroup *pg)
 {
     ASSERT(PacketPassInterface_GetMTU(output) >= PacketRecvInterface_GetMTU(input))
@@ -198,9 +82,6 @@ int PacketBuffer_Init (PacketBuffer *buf, PacketRecvInterface *input, PacketPass
     buf->input = input;
     buf->output = output;
     
-    // init dead var
-    DEAD_INIT(buf->dead);
-    
     // init input
     PacketRecvInterface_Receiver_Init(buf->input, (PacketRecvInterface_handler_done)input_handler_done, buf);
     
@@ -219,11 +100,9 @@ int PacketBuffer_Init (PacketBuffer *buf, PacketRecvInterface *input, PacketPass
     // init buffer
     ChunkBuffer2_Init(&buf->buf, buf->buf_data, num_blocks, buf->input_mtu);
     
-    // init start job
-    BPending_Init(&buf->start_job, pg, (BPending_handler)job_handler, buf);
-    BPending_Set(&buf->start_job);
+    // schedule receive
+    PacketRecvInterface_Receiver_Recv(buf->input, buf->buf.input_dest);
     
-    // init debug object
     DebugObject_Init(&buf->d_obj);
     
     return 1;
@@ -234,15 +113,8 @@ fail0:
 
 void PacketBuffer_Free (PacketBuffer *buf)
 {
-    // free debug object
     DebugObject_Free(&buf->d_obj);
     
-    // free start job
-    BPending_Free(&buf->start_job);
-    
     // free buffer
     free(buf->buf_data);
-    
-    // free dead var
-    DEAD_KILL(buf->dead);
 }

flow/PacketBuffer.h

@@ -29,10 +29,8 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <misc/debug.h>
 #include <system/DebugObject.h>
-#include <system/BPending.h>
 #include <structure/ChunkBuffer2.h>
 #include <flow/PacketRecvInterface.h>
 #include <flow/PacketPassInterface.h>
@@ -42,13 +40,11 @@
  */
 typedef struct {
     DebugObject d_obj;
-    dead_t dead;
     PacketRecvInterface *input;
     int input_mtu;
     PacketPassInterface *output;
     struct ChunkBuffer2_block *buf_data;
     ChunkBuffer2 buf;
-    BPending start_job;
 } PacketBuffer;
 
 /**

flow/PacketBufferAsyncInput.h

@@ -1,160 +0,0 @@
-/**
- * @file PacketBufferAsyncInput.h
- * @author Ambroz Bizjak <[email protected]>
- * 
- * @section LICENSE
- * 
- * This file is part of BadVPN.
- * 
- * BadVPN is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2
- * as published by the Free Software Foundation.
- * 
- * BadVPN is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- * 
- * @section DESCRIPTION
- * 
- * Object for writing packets to a {@link PacketRecvInterface} client
- * via {@link BestEffortPacketWriteInterface}.
- */
-
-#ifndef BADVPN_FLOW_PACKETBUFFERASYNCINPUT_H
-#define BADVPN_FLOW_PACKETBUFFERASYNCINPUT_H
-
-#include <stdint.h>
-
-#include <misc/debug.h>
-#include <system/DebugObject.h>
-#include <flow/PacketRecvInterface.h>
-#include <flow/BestEffortPacketWriteInterface.h>
-
-typedef void (*PacketBufferAsyncInput_handler_keepalive) (void *user);
-
-/**
- * Object for writing packets to a {@link PacketRecvInterface} client
- * via {@link BestEffortPacketWriteInterface}.
- */
-typedef struct {
-    DebugObject d_obj;
-    BestEffortPacketWriteInterface input;
-    PacketRecvInterface recv_interface;
-    int have_output_packet;
-    uint8_t *output_packet;
-} PacketBufferAsyncInput;
-
-/**
- * Initializes the object.
- *
- * @param f the object
- */
-static void PacketBufferAsyncInput_Init (PacketBufferAsyncInput *f, int mtu);
-
-/**
- * Frees the object.
- *
- * @param f the object
- */
-static void PacketBufferAsyncInput_Free (PacketBufferAsyncInput *f);
-
-/**
- * Returns the output interface.
- *
- * @param f the object
- * @return output interface
- */
-static PacketRecvInterface * PacketBufferAsyncInput_GetOutput (PacketBufferAsyncInput *f);
-
-/**
- * Returns the input interface.
- *
- * @param f the object
- * @return input interface
- */
-static BestEffortPacketWriteInterface * PacketBufferAsyncInput_GetInput (PacketBufferAsyncInput *f);
-
-static int _PacketBufferAsyncInput_output_handler_recv (PacketBufferAsyncInput *f, uint8_t *data, int *data_len)
-{
-    ASSERT(!f->have_output_packet)
-    
-    // store destination
-    f->have_output_packet = 1;
-    f->output_packet = data;
-    
-    // block
-    return 0;
-}
-
-static int _PacketBufferAsyncInput_handler_startpacket (PacketBufferAsyncInput *f, uint8_t **data)
-{
-    if (!f->have_output_packet) {
-        // buffer full
-        return 0;
-    }
-    
-    if (data) {
-        *data = f->output_packet;
-    }
-    
-    return 1;
-}
-
-static void _PacketBufferAsyncInput_handler_endpacket (PacketBufferAsyncInput *f, int len)
-{
-    f->have_output_packet = 0;
-    
-    PacketRecvInterface_Done(&f->recv_interface, len);
-    return;
-}
-
-void PacketBufferAsyncInput_Init (PacketBufferAsyncInput *f, int mtu)
-{
-    ASSERT(mtu >= 0)
-    
-    PacketRecvInterface_Init(
-        &f->recv_interface,
-        mtu,
-        (PacketRecvInterface_handler_recv)_PacketBufferAsyncInput_output_handler_recv,
-        f
-    );
-    
-    BestEffortPacketWriteInterface_Init(
-        &f->input,
-        mtu,
-        (BestEffortPacketWriteInterface_handler_startpacket)_PacketBufferAsyncInput_handler_startpacket,
-        (BestEffortPacketWriteInterface_handler_endpacket)_PacketBufferAsyncInput_handler_endpacket,
-        f
-    );
-    
-    f->have_output_packet = 0;
-    
-    // init debug object
-    DebugObject_Init(&f->d_obj);
-}
-
-void PacketBufferAsyncInput_Free (PacketBufferAsyncInput *f)
-{
-    // free debug object
-    DebugObject_Free(&f->d_obj);
-
-    BestEffortPacketWriteInterface_Free(&f->input);
-    PacketRecvInterface_Free(&f->recv_interface);
-}
-
-PacketRecvInterface * PacketBufferAsyncInput_GetOutput (PacketBufferAsyncInput *f)
-{
-    return &f->recv_interface;
-}
-
-BestEffortPacketWriteInterface * PacketBufferAsyncInput_GetInput (PacketBufferAsyncInput *f)
-{
-    return &f->input;
-}
-
-#endif

flow/PacketCopier.c

@@ -26,77 +26,70 @@
 
 #include <flow/PacketCopier.h>
 
-static int input_handler_send (PacketCopier *o, uint8_t *data, int data_len)
+static void input_handler_send (PacketCopier *o, uint8_t *data, int data_len)
 {
     ASSERT(o->in_len == -1)
     ASSERT(data_len >= 0)
+    DebugObject_Access(&o->d_obj);
     
     if (!o->out_have) {
         o->in_len = data_len;
         o->in = data;
-        return 0;
+        return;
     }
     
     memcpy(o->out, data, data_len);
     
-    o->out_have = 0;
-    
-    DEAD_ENTER(o->dead)
+    // finish output packet
     PacketRecvInterface_Done(&o->output, data_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
     
-    return 1;
+    // finish input packet
+    PacketPassInterface_Done(&o->input);
+    
+    o->out_have = 0;
 }
 
 static void input_handler_cancel (PacketCopier *o)
 {
     ASSERT(o->in_len >= 0)
     ASSERT(!o->out_have)
+    DebugObject_Access(&o->d_obj);
     
     o->in_len = -1;
 }
 
-static int output_handler_recv (PacketCopier *o, uint8_t *data, int *data_len)
+static void output_handler_recv (PacketCopier *o, uint8_t *data)
 {
     ASSERT(!o->out_have)
+    DebugObject_Access(&o->d_obj);
     
     if (o->in_len < 0) {
         o->out_have = 1;
         o->out = data;
-        return 0;
+        return;
     }
     
-    int len = o->in_len;
-    
-    memcpy(data, o->in, len);
-    
-    o->in_len = -1;
+    memcpy(data, o->in, o->in_len);
     
-    DEAD_ENTER(o->dead)
+    // finish input packet
     PacketPassInterface_Done(&o->input);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
     
-    *data_len = len;
-    return 1;
+    // finish output packet
+    PacketRecvInterface_Done(&o->output, o->in_len);
+    
+    o->in_len = -1;
 }
 
-void PacketCopier_Init (PacketCopier *o, int mtu)
+void PacketCopier_Init (PacketCopier *o, int mtu, BPendingGroup *pg)
 {
     ASSERT(mtu >= 0)
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init input
-    PacketPassInterface_Init(&o->input, mtu, (PacketPassInterface_handler_send)input_handler_send, o);
+    PacketPassInterface_Init(&o->input, mtu, (PacketPassInterface_handler_send)input_handler_send, o, pg);
     PacketPassInterface_EnableCancel(&o->input, (PacketPassInterface_handler_cancel)input_handler_cancel);
     
     // init output
-    PacketRecvInterface_Init(&o->output, mtu, (PacketRecvInterface_handler_recv)output_handler_recv, o);
+    PacketRecvInterface_Init(&o->output, mtu, (PacketRecvInterface_handler_recv)output_handler_recv, o, pg);
     
     // set no input packet
     o->in_len = -1;
@@ -104,13 +97,11 @@ void PacketCopier_Init (PacketCopier *o, int mtu)
     // set no output packet
     o->out_have = 0;
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
 void PacketCopier_Free (PacketCopier *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // free output
@@ -118,17 +109,18 @@ void PacketCopier_Free (PacketCopier *o)
     
     // free input
     PacketPassInterface_Free(&o->input);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }
 
 PacketPassInterface * PacketCopier_GetInput (PacketCopier *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->input;
 }
 
 PacketRecvInterface * PacketCopier_GetOutput (PacketCopier *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->output;
 }

flow/PacketCopier.h

@@ -29,7 +29,6 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <flow/PacketPassInterface.h>
 #include <flow/PacketRecvInterface.h>
 
@@ -40,7 +39,6 @@
  */
 typedef struct {
     DebugObject d_obj;
-    dead_t dead;
     PacketPassInterface input;
     PacketRecvInterface output;
     int in_len;
@@ -54,8 +52,9 @@ typedef struct {
  * 
  * @param o the object
  * @param mtu maximum packet size. Must be >=0.
+ * @param pg pending group
  */
-void PacketCopier_Init (PacketCopier *o, int mtu);
+void PacketCopier_Init (PacketCopier *o, int mtu, BPendingGroup *pg);
 
 /**
  * Frees the object.

flow/PacketPassConnector.c

@@ -26,55 +26,25 @@
 
 #include <flow/PacketPassConnector.h>
 
-static int input_handler_send (PacketPassConnector *o, uint8_t *data, int data_len)
+static void input_handler_send (PacketPassConnector *o, uint8_t *data, int data_len)
 {
-    ASSERT(o->in_len == -1)
-    ASSERT(!(o->output) || !o->out_blocking)
     ASSERT(data_len >= 0)
     ASSERT(data_len <= o->input_mtu)
-    
-    // if we have no output, remember input packet
-    if (!o->output) {
-        o->in_len = data_len;
-        o->in = data;
-        return 0;
-    }
-    
-    // try to send the packet
-    int res;
-    while (1) {
-        DEAD_ENTER_N(obj, o->dead)
-        DEAD_ENTER_N(out, o->output_dead)
-        res = PacketPassInterface_Sender_Send(o->output, data, data_len);
-        DEAD_LEAVE_N(obj, o->dead);
-        DEAD_LEAVE_N(out, o->output_dead);
-        if (DEAD_KILLED_N(obj)) {
-            return -1;
-        }
-        if (DEAD_KILLED_N(out)) {
-            if (!o->output) {
-                // lost output
-                o->in_len = data_len;
-                o->in = data;
-                return 0;
-            }
-            // got a new output, retry
-            continue;
-        }
-        break;
+    ASSERT(o->in_len == -1)
+    if (o->output) {
+        ASSERT(!o->out_blocking)
     }
+    DebugObject_Access(&o->d_obj);
     
-    ASSERT(res == 0 || res == 1)
+    // remember input packet
+    o->in_len = data_len;
+    o->in = data;
     
-    if (!res) {
-        // output blocking
-        o->in_len = data_len;
-        o->in = data;
+    if (o->output) {
+        // schedule send
+        PacketPassInterface_Sender_Send(o->output, o->in, o->in_len);
         o->out_blocking = 1;
-        return 0;
     }
-    
-    return 1;
 }
 
 static void output_handler_done (PacketPassConnector *o)
@@ -82,53 +52,16 @@ static void output_handler_done (PacketPassConnector *o)
     ASSERT(o->in_len >= 0)
     ASSERT(o->output)
     ASSERT(o->out_blocking)
-    
-    // have no input packet
-    o->in_len = -1;
+    DebugObject_Access(&o->d_obj);
     
     // output not blocking any more
     o->out_blocking = 0;
     
-    // allow input to send more packets
-    PacketPassInterface_Done(&o->input);
-    return;
-}
-
-static void job_handler (PacketPassConnector *o)
-{
-    ASSERT(o->output)
-    ASSERT(!o->out_blocking)
-    ASSERT(o->in_len >= 0)
-    
-    // try to send the packet
-    DEAD_ENTER_N(obj, o->dead)
-    DEAD_ENTER_N(out, o->output_dead)
-    int res = PacketPassInterface_Sender_Send(o->output, o->in, o->in_len);
-    DEAD_LEAVE_N(obj, o->dead);
-    DEAD_LEAVE_N(out, o->output_dead);
-    if (DEAD_KILLED_N(obj)) {
-        return;
-    }
-    if (DEAD_KILLED_N(out)) {
-        // lost current output. Do nothing here.
-        // If we gained a new one, its own job is responsible for it.
-        return;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    if (!res) {
-        // output blocking
-        o->out_blocking = 1;
-        return;
-    }
-    
     // have no input packet
     o->in_len = -1;
     
     // allow input to send more packets
     PacketPassInterface_Done(&o->input);
-    return;
 }
 
 void PacketPassConnector_Init (PacketPassConnector *o, int mtu, BPendingGroup *pg)
@@ -138,11 +71,8 @@ void PacketPassConnector_Init (PacketPassConnector *o, int mtu, BPendingGroup *p
     // init arguments
     o->input_mtu = mtu;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init input
-    PacketPassInterface_Init(&o->input, o->input_mtu, (PacketPassInterface_handler_send)input_handler_send, o);
+    PacketPassInterface_Init(&o->input, o->input_mtu, (PacketPassInterface_handler_send)input_handler_send, o, pg);
     
     // have no input packet
     o->in_len = -1;
@@ -150,35 +80,21 @@ void PacketPassConnector_Init (PacketPassConnector *o, int mtu, BPendingGroup *p
     // have no output
     o->output = NULL;
     
-    // init continue job
-    BPending_Init(&o->continue_job, pg, (BPending_handler)job_handler, o);
-    
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
 void PacketPassConnector_Free (PacketPassConnector *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
     
-    // free continue job
-    BPending_Free(&o->continue_job);
-    
-    // free output dead var
-    if (o->output) {
-        DEAD_KILL(o->output_dead);
-    }
-    
     // free input
     PacketPassInterface_Free(&o->input);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }
 
 PacketPassInterface * PacketPassConnector_GetInput (PacketPassConnector *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->input;
 }
 
@@ -186,6 +102,7 @@ void PacketPassConnector_ConnectOutput (PacketPassConnector *o, PacketPassInterf
 {
     ASSERT(!o->output)
     ASSERT(PacketPassInterface_GetMTU(output) >= o->input_mtu)
+    DebugObject_Access(&o->d_obj);
     
     // set output
     o->output = output;
@@ -193,27 +110,20 @@ void PacketPassConnector_ConnectOutput (PacketPassConnector *o, PacketPassInterf
     // init output
     PacketPassInterface_Sender_Init(o->output, (PacketPassInterface_handler_done)output_handler_done, o);
     
-    // init output dead var
-    DEAD_INIT(o->output_dead);
-    
     // set output not blocking
     o->out_blocking = 0;
     
-    // if we have an input packet, set continue job
+    // if we have an input packet, schedule send
     if (o->in_len >= 0) {
-        BPending_Set(&o->continue_job);
+        PacketPassInterface_Sender_Send(o->output, o->in, o->in_len);
+        o->out_blocking = 1;
     }
 }
 
 void PacketPassConnector_DisconnectOutput (PacketPassConnector *o)
 {
     ASSERT(o->output)
-    
-    // unset continue job (in case it wasn't called yet)
-    BPending_Unset(&o->continue_job);
-    
-    // free dead var
-    DEAD_KILL(o->output_dead);
+    DebugObject_Access(&o->d_obj);
     
     // set no output
     o->output = NULL;

flow/PacketPassConnector.h

@@ -30,9 +30,7 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <system/DebugObject.h>
-#include <system/BPending.h>
 #include <flow/PacketPassInterface.h>
 
 /**
@@ -40,16 +38,13 @@
  * connected and disconnected on the fly.
  */
 typedef struct {
-    DebugObject d_obj;
-    dead_t dead;
     PacketPassInterface input;
     int input_mtu;
     int in_len;
     uint8_t *in;
     PacketPassInterface *output;
-    dead_t output_dead;
     int out_blocking;
-    BPending continue_job;
+    DebugObject d_obj;
 } PacketPassConnector;
 
 /**

flow/PacketPassFairQueue.c

@@ -27,58 +27,14 @@
 
 #include <flow/PacketPassFairQueue.h>
 
-static int call_send (PacketPassFairQueue *m, uint8_t *data, int data_len)
-{
-    DebugIn_GoIn(&m->in_output);
-    DEAD_ENTER(m->dead)
-    int res = PacketPassInterface_Sender_Send(m->output, data, data_len);
-    if (DEAD_LEAVE(m->dead)) {
-        return -1;
-    }
-    DebugIn_GoOut(&m->in_output);
-    
-    ASSERT(!m->freeing)
-    ASSERT(res == 0 || res == 1)
-    
-    return res;
-}
-
-static int call_cancel (PacketPassFairQueue *m)
-{
-    DebugIn_GoIn(&m->in_output);
-    DEAD_ENTER(m->dead)
-    PacketPassInterface_Sender_Cancel(m->output);
-    if (DEAD_LEAVE(m->dead)) {
-        return -1;
-    }
-    DebugIn_GoOut(&m->in_output);
-    
-    ASSERT(!m->freeing)
-    
-    return 0;
-}
-
-static int call_done (PacketPassFairQueue *m, PacketPassFairQueueFlow *flow)
+static int time_comparator (void *user, uint64_t *time1, uint64_t *time2)
 {
-    DEAD_ENTER_N(m, m->dead)
-    DEAD_ENTER_N(flow, flow->dead)
-    PacketPassInterface_Done(&flow->input);
-    DEAD_LEAVE_N(m, m->dead);
-    DEAD_LEAVE_N(flow, flow->dead);
-    
-    if (DEAD_KILLED_N(m)) {
+    if (*time1 < *time2) {
         return -1;
     }
-    
-    ASSERT(!m->freeing)
-    
-    if (!DEAD_KILLED_N(flow)) {
-        ASSERT(flow->have_time)
-        if (flow != m->sending_flow && !flow->is_queued) {
-            flow->have_time = 0;
-        }
+    if (*time1 > *time2) {
+        return 1;
     }
-    
     return 0;
 }
 
@@ -141,124 +97,110 @@ static void increment_sent_flow (PacketPassFairQueueFlow *flow, int iamount)
     }
 }
 
-static void process_queue (PacketPassFairQueue *m)
+static void schedule (PacketPassFairQueue *m)
 {
     ASSERT(!m->freeing)
     ASSERT(!m->sending_flow)
+    ASSERT(BHeap_GetFirst(&m->queued_heap))
     
-    do {
-        // get first queued flow
-        BHeapNode *heap_node = BHeap_GetFirst(&m->queued_heap);
-        if (!heap_node) {
-            return;
-        }
-        PacketPassFairQueueFlow *qflow = UPPER_OBJECT(heap_node, PacketPassFairQueueFlow, queued.heap_node);
-        ASSERT(qflow->is_queued)
-        ASSERT(qflow->have_time)
-        
-        // remove flow from queue
-        BHeap_Remove(&m->queued_heap, &qflow->queued.heap_node);
-        LinkedList2_Remove(&m->queued_list, &qflow->queued.list_node);
-        qflow->is_queued = 0;
-        
-        // try to send the packet
-        int res = call_send(m, qflow->queued.data, qflow->queued.data_len);
-        if (res < 0) {
-            return;
-        }
-        
-        if (res == 0) {
-            // sending in progress
-            m->sending_flow = qflow;
-            m->sending_len = qflow->queued.data_len;
-            return;
-        }
-        
-        // increment flow time
-        increment_sent_flow(qflow, qflow->queued.data_len);
-        
-        // notify sender
-        if (call_done(m, qflow) < 0) {
-            return;
-        }
-    } while (!m->sending_flow);
+    // get first queued flow
+    BHeapNode *heap_node = BHeap_GetFirst(&m->queued_heap);
+    PacketPassFairQueueFlow *qflow = UPPER_OBJECT(heap_node, PacketPassFairQueueFlow, queued.heap_node);
+    ASSERT(qflow->is_queued)
+    ASSERT(qflow->have_time)
+    
+    // remove flow from queue
+    BHeap_Remove(&m->queued_heap, &qflow->queued.heap_node);
+    LinkedList2_Remove(&m->queued_list, &qflow->queued.list_node);
+    qflow->is_queued = 0;
+    
+    // schedule send
+    PacketPassInterface_Sender_Send(m->output, qflow->queued.data, qflow->queued.data_len);
+    m->sending_flow = qflow;
+    m->sending_len = qflow->queued.data_len;
 }
 
-static int time_comparator (void *user, uint64_t *time1, uint64_t *time2)
+static void schedule_job_handler (PacketPassFairQueue *m)
 {
-    if (*time1 < *time2) {
-        return -1;
+    ASSERT(!m->freeing)
+    ASSERT(!m->sending_flow)
+    DebugObject_Access(&m->d_obj);
+    
+    // if the previous flow didn't contend for sending, remove its time
+    if (m->previous_flow) {
+        ASSERT(!m->previous_flow->is_queued)
+        ASSERT(m->previous_flow->have_time)
+        
+        // remove time from flow
+        m->previous_flow->have_time = 0;
+        
+        // set no previous flow
+        m->previous_flow = NULL;
     }
-    if (*time1 > *time2) {
-        return 1;
+    
+    if (BHeap_GetFirst(&m->queued_heap)) {
+        schedule(m);
     }
-    return 0;
 }
 
-static int input_handler_send (PacketPassFairQueueFlow *flow, uint8_t *data, int data_len)
+static void input_handler_send (PacketPassFairQueueFlow *flow, uint8_t *data, int data_len)
 {
     ASSERT(!flow->m->freeing)
     ASSERT(flow != flow->m->sending_flow)
     ASSERT(!flow->is_queued)
-    DebugIn_AmOut(&flow->m->in_output);
+    DebugObject_Access(&flow->d_obj);
     
     PacketPassFairQueue *m = flow->m;
     
     // assign time if needed
-    int had_time = flow->have_time;
     if (!flow->have_time) {
         flow->time = get_current_time(m);
         flow->have_time = 1;
     }
     
-    // if nothing is being sent and queue is empty, send immediately without queueing
-    if (!m->sending_flow && !BHeap_GetFirst(&m->queued_heap)) {
-        int res = call_send(m, data, data_len);
-        if (res < 0) {
-            return -1;
-        }
-        
-        if (res == 0) {
-            // output busy, continue in output_handler_done
-            m->sending_flow = flow;
-            m->sending_len = data_len;
-            return 0;
-        }
-        
-        // if flow had no time before it shouldn't have after
-        if (!had_time) {
-            flow->have_time = 0;
-        }
-        
-        return 1;
+    // remove from previous flow
+    if (flow == m->previous_flow) {
+        m->previous_flow = NULL;
     }
     
-    // add flow to queue
+    // queue flow
     flow->queued.data = data;
     flow->queued.data_len = data_len;
     BHeap_Insert(&m->queued_heap, &flow->queued.heap_node);
     LinkedList2_Append(&m->queued_list, &flow->queued.list_node);
     flow->is_queued = 1;
     
-    return 0;
+    if (!m->sending_flow && !BPending_IsSet(&m->schedule_job)) {
+        schedule(m);
+    }
 }
 
 static void output_handler_done (PacketPassFairQueue *m)
 {
     ASSERT(!m->freeing)
     ASSERT(m->sending_flow)
+    ASSERT(!m->previous_flow)
     ASSERT(!m->sending_flow->is_queued)
     ASSERT(m->sending_flow->have_time)
-    DebugIn_AmOut(&m->in_output);
+    ASSERT(!BPending_IsSet(&m->schedule_job))
     
     PacketPassFairQueueFlow *flow = m->sending_flow;
     
     // sending finished
     m->sending_flow = NULL;
     
+    // remember this flow so the schedule job can remove its time if it didn's send
+    m->previous_flow = flow;
+    
     // update flow time by packet size
     increment_sent_flow(flow, m->sending_len);
     
+    // schedule schedule
+    BPending_Set(&m->schedule_job);
+    
+    // finish flow packet
+    PacketPassInterface_Done(&flow->input);
+    
     // call busy handler if set
     if (flow->handler_busy) {
         // handler is one-shot, unset it before calling
@@ -266,42 +208,14 @@ static void output_handler_done (PacketPassFairQueue *m)
         flow->handler_busy = NULL;
         
         // call handler
-        DEAD_ENTER_N(m, m->dead)
-        DEAD_ENTER_N(flow, flow->dead)
+        DEAD_ENTER(m->dead)
         handler(flow->user);
-        DEAD_LEAVE_N(m, m->dead);
-        DEAD_LEAVE_N(flow, flow->dead);
-        if (DEAD_KILLED_N(m)) {
+        if (DEAD_LEAVE(m->dead)) {
             return;
         }
-        if (DEAD_KILLED_N(flow)) {
-            flow = NULL;
-        }
         
         ASSERT(!m->freeing)
-    }
-    
-    // report completion to sender
-    if (flow) {
-        if (call_done(m, flow) < 0) {
-            return;
-        }
-    }
-    
-    // process queued flows
-    if (!m->sending_flow) {
-        process_queue(m);
-        return;
-    }
-}
-
-static void job_handler (PacketPassFairQueue *m)
-{
-    ASSERT(!m->freeing)
-    
-    if (!m->sending_flow) {
-        process_queue(m);
-        return;
+        ASSERT(!m->sending_flow)
     }
 }
 
@@ -309,6 +223,7 @@ void PacketPassFairQueue_Init (PacketPassFairQueue *m, PacketPassInterface *outp
 {
     // init arguments
     m->output = output;
+    m->pg = pg;
     
     // init dead var
     DEAD_INIT(m->dead);
@@ -319,6 +234,9 @@ void PacketPassFairQueue_Init (PacketPassFairQueue *m, PacketPassInterface *outp
     // not sending
     m->sending_flow = NULL;
     
+    // no previous flow
+    m->previous_flow = NULL;
+    
     // init queued heap
     BHeap_Init(&m->queued_heap, OFFSET_DIFF(PacketPassFairQueueFlow, time, queued.heap_node), (BHeap_comparator)time_comparator, NULL);
     
@@ -331,16 +249,10 @@ void PacketPassFairQueue_Init (PacketPassFairQueue *m, PacketPassInterface *outp
     // not using cancel
     m->use_cancel = 0;
     
-    // init continue job
-    BPending_Init(&m->continue_job, pg, (BPending_handler)job_handler, m);
+    // init schedule job
+    BPending_Init(&m->schedule_job, m->pg, (BPending_handler)schedule_job_handler, m);
     
-    // init debug counter
     DebugCounter_Init(&m->d_ctr);
-    
-    // init debug in output
-    DebugIn_Init(&m->in_output);
-    
-    // init debug object
     DebugObject_Init(&m->d_obj);
 }
 
@@ -352,8 +264,8 @@ void PacketPassFairQueue_Free (PacketPassFairQueue *m)
     DebugCounter_Free(&m->d_ctr);
     DebugObject_Free(&m->d_obj);
     
-    // free continue job
-    BPending_Free(&m->continue_job);
+    // free schedule job
+    BPending_Free(&m->schedule_job);
     
     // free dead var
     DEAD_KILL(m->dead);
@@ -363,6 +275,7 @@ void PacketPassFairQueue_EnableCancel (PacketPassFairQueue *m)
 {
     ASSERT(!m->use_cancel)
     ASSERT(PacketPassInterface_HasCancel(m->output))
+    DebugObject_Access(&m->d_obj);
     
     // using cancel
     m->use_cancel = 1;
@@ -370,25 +283,24 @@ void PacketPassFairQueue_EnableCancel (PacketPassFairQueue *m)
 
 void PacketPassFairQueue_PrepareFree (PacketPassFairQueue *m)
 {
+    DebugObject_Access(&m->d_obj);
+    
     m->freeing = 1;
 }
 
 void PacketPassFairQueueFlow_Init (PacketPassFairQueueFlow *flow, PacketPassFairQueue *m)
 {
     ASSERT(!m->freeing)
-    DebugIn_AmOut(&m->in_output);
+    DebugObject_Access(&m->d_obj);
     
     // init arguments
     flow->m = m;
     
-    // init dead var
-    DEAD_INIT(flow->dead);
-    
     // have no canfree handler
     flow->handler_busy = NULL;
     
     // init input
-    PacketPassInterface_Init(&flow->input, PacketPassInterface_GetMTU(flow->m->output), (PacketPassInterface_handler_send)input_handler_send, flow);
+    PacketPassInterface_Init(&flow->input, PacketPassInterface_GetMTU(flow->m->output), (PacketPassInterface_handler_send)input_handler_send, flow, m->pg);
     
     // doesn't have time
     flow->have_time = 0;
@@ -396,10 +308,7 @@ void PacketPassFairQueueFlow_Init (PacketPassFairQueueFlow *flow, PacketPassFair
     // is not queued
     flow->is_queued = 0;
     
-    // increment debug counter
     DebugCounter_Increment(&m->d_ctr);
-    
-    // init debug object
     DebugObject_Init(&flow->d_obj);
 }
 
@@ -407,16 +316,20 @@ void PacketPassFairQueueFlow_Free (PacketPassFairQueueFlow *flow)
 {
     if (!flow->m->freeing) {
         ASSERT(flow != flow->m->sending_flow)
-        DebugIn_AmOut(&flow->m->in_output);
     }
     DebugCounter_Decrement(&flow->m->d_ctr);
     DebugObject_Free(&flow->d_obj);
     
     PacketPassFairQueue *m = flow->m;
     
-    // remove current flow
-    if (flow == flow->m->sending_flow) {
-        flow->m->sending_flow = NULL;
+    // remove from current flow
+    if (flow == m->sending_flow) {
+        m->sending_flow = NULL;
+    }
+    
+    // remove from previous flow
+    if (flow == m->previous_flow) {
+        m->previous_flow = NULL;
     }
     
     // remove from queue
@@ -427,15 +340,20 @@ void PacketPassFairQueueFlow_Free (PacketPassFairQueueFlow *flow)
     
     // free input
     PacketPassInterface_Free(&flow->input);
-    
-    // free dead var
-    DEAD_KILL(flow->dead);
+}
+
+void PacketPassFairQueueFlow_AssertFree (PacketPassFairQueueFlow *flow)
+{
+    if (!flow->m->freeing) {
+        ASSERT(flow != flow->m->sending_flow)
+    }
+    DebugObject_Access(&flow->d_obj);
 }
 
 int PacketPassFairQueueFlow_IsBusy (PacketPassFairQueueFlow *flow)
 {
     ASSERT(!flow->m->freeing)
-    DebugIn_AmOut(&flow->m->in_output);
+    DebugObject_Access(&flow->d_obj);
     
     return (flow == flow->m->sending_flow);
 }
@@ -445,27 +363,26 @@ void PacketPassFairQueueFlow_Release (PacketPassFairQueueFlow *flow)
     ASSERT(flow->m->use_cancel)
     ASSERT(flow == flow->m->sending_flow)
     ASSERT(!flow->m->freeing)
-    DebugIn_AmOut(&flow->m->in_output);
+    ASSERT(!BPending_IsSet(&flow->m->schedule_job))
+    DebugObject_Access(&flow->d_obj);
     
     PacketPassFairQueue *m = flow->m;
     
     // cancel current packet
-    if (call_cancel(m) < 0) {
-        return;
-    }
+    PacketPassInterface_Sender_Cancel(m->output);
     
     // set no sending flow
     m->sending_flow = NULL;
     
-    // set continue job
-    BPending_Set(&m->continue_job);
+    // schedule schedule
+    BPending_Set(&m->schedule_job);
 }
 
 void PacketPassFairQueueFlow_SetBusyHandler (PacketPassFairQueueFlow *flow, PacketPassFairQueue_handler_busy handler, void *user)
 {
     ASSERT(flow == flow->m->sending_flow)
     ASSERT(!flow->m->freeing)
-    DebugIn_AmOut(&flow->m->in_output);
+    DebugObject_Access(&flow->d_obj);
     
     flow->handler_busy = handler;
     flow->user = user;
@@ -473,5 +390,7 @@ void PacketPassFairQueueFlow_SetBusyHandler (PacketPassFairQueueFlow *flow, Pack
 
 PacketPassInterface * PacketPassFairQueueFlow_GetInput (PacketPassFairQueueFlow *flow)
 {
+    DebugObject_Access(&flow->d_obj);
+    
     return &flow->input;
 }

flow/PacketPassFairQueue.h

@@ -30,12 +30,11 @@
 #include <stdint.h>
 
 #include <misc/dead.h>
-#include <misc/debugin.h>
 #include <system/DebugObject.h>
+#include <system/BPending.h>
 #include <misc/debugcounter.h>
 #include <structure/BHeap.h>
 #include <structure/LinkedList2.h>
-#include <system/BPending.h>
 #include <flow/PacketPassInterface.h>
 
 typedef void (*PacketPassFairQueue_handler_busy) (void *user);
@@ -49,19 +48,19 @@ typedef struct {
     dead_t dead;
     PacketPassInterface *output;
     struct PacketPassFairQueueFlow_s *sending_flow;
+    struct PacketPassFairQueueFlow_s *previous_flow;
     int sending_len;
     BHeap queued_heap;
     LinkedList2 queued_list;
     int freeing;
     int use_cancel;
-    BPending continue_job;
+    BPending schedule_job;
+    BPendingGroup *pg;
     DebugCounter d_ctr;
-    DebugIn in_output;
     DebugObject d_obj;
 } PacketPassFairQueue;
 
 typedef struct PacketPassFairQueueFlow_s {
-    dead_t dead;
     PacketPassFairQueue *m;
     PacketPassFairQueue_handler_busy handler_busy;
     void *user;
@@ -135,6 +134,14 @@ void PacketPassFairQueueFlow_Init (PacketPassFairQueueFlow *flow, PacketPassFair
  */
 void PacketPassFairQueueFlow_Free (PacketPassFairQueueFlow *flow);
 
+/**
+ * Does nothing.
+ * It must be possible to free the flow (see {@link PacketPassFairQueueFlow_Free}).
+ * 
+ * @param flow the object
+ */
+void PacketPassFairQueueFlow_AssertFree (PacketPassFairQueueFlow *flow);
+
 /**
  * Determines if the flow is busy. If the flow is considered busy, it must not
  * be freed.

flow/PacketPassInactivityMonitor.c

@@ -22,53 +22,43 @@
 
 #include <flow/PacketPassInactivityMonitor.h>
 
-static int input_handler_send (PacketPassInactivityMonitor *o, uint8_t *data, int data_len)
+static void input_handler_send (PacketPassInactivityMonitor *o, uint8_t *data, int data_len)
 {
-    // set send called
-    o->send_called = 1;
+    DebugObject_Access(&o->d_obj);
     
-    // call send
-    DEAD_ENTER(o->dead)
-    int res = PacketPassInterface_Sender_Send(o->output, data, data_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    if (res == 0) {
-        // output busy, stop timer
-        BReactor_RemoveTimer(o->reactor, &o->timer);
-    } else {
-        // output accepted packet, restart timer
-        BReactor_SetTimer(o->reactor, &o->timer);
-    }
+    // schedule send
+    PacketPassInterface_Sender_Send(o->output, data, data_len);
     
-    return res;
+    // stop timer
+    BReactor_RemoveTimer(o->reactor, &o->timer);
 }
 
 static void input_handler_cancel (PacketPassInactivityMonitor *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     // output no longer busy, restart timer
     BReactor_SetTimer(o->reactor, &o->timer);
     
     // call cancel
     PacketPassInterface_Sender_Cancel(o->output);
-    return;
 }
 
 static void output_handler_done (PacketPassInactivityMonitor *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     // output no longer busy, restart timer
     BReactor_SetTimer(o->reactor, &o->timer);
     
     // call done
     PacketPassInterface_Done(&o->input);
-    return;
 }
 
 static void timer_handler (PacketPassInactivityMonitor *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     // restart timer
     BReactor_SetTimer(o->reactor, &o->timer);
     
@@ -79,19 +69,14 @@ static void timer_handler (PacketPassInactivityMonitor *o)
 
 void PacketPassInactivityMonitor_Init (PacketPassInactivityMonitor *o, PacketPassInterface *output, BReactor *reactor, btime_t interval, PacketPassInactivityMonitor_handler handler, void *user)
 {
-    ASSERT(interval > 0)
-    
     // init arguments
     o->output = output;
     o->reactor = reactor;
     o->handler = handler;
     o->user = user;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init input
-    PacketPassInterface_Init(&o->input, PacketPassInterface_GetMTU(o->output), (PacketPassInterface_handler_send)input_handler_send, o);
+    PacketPassInterface_Init(&o->input, PacketPassInterface_GetMTU(o->output), (PacketPassInterface_handler_send)input_handler_send, o, BReactor_PendingGroup(o->reactor));
     if (PacketPassInterface_HasCancel(o->output)) {
         PacketPassInterface_EnableCancel(&o->input, (PacketPassInterface_handler_cancel)input_handler_cancel);
     }
@@ -99,20 +84,15 @@ void PacketPassInactivityMonitor_Init (PacketPassInactivityMonitor *o, PacketPas
     // init output
     PacketPassInterface_Sender_Init(o->output, (PacketPassInterface_handler_done)output_handler_done, o);
     
-    // init timer and start it
+    // init timer
     BTimer_Init(&o->timer, interval, (BTimer_handler)timer_handler, o);
     BReactor_SetTimer(o->reactor, &o->timer);
     
-    // set send not called
-    o->send_called = 0;
-    
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
 void PacketPassInactivityMonitor_Free (PacketPassInactivityMonitor *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // free timer
@@ -120,12 +100,11 @@ void PacketPassInactivityMonitor_Free (PacketPassInactivityMonitor *o)
     
     // free input
     PacketPassInterface_Free(&o->input);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }
 
 PacketPassInterface * PacketPassInactivityMonitor_GetInput (PacketPassInactivityMonitor *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->input;
 }

flow/PacketPassInactivityMonitor.h

@@ -27,7 +27,6 @@
 #ifndef BADVPN_FLOW_PACKETPASSINACTIVITYMONITOR_H
 #define BADVPN_FLOW_PACKETPASSINACTIVITYMONITOR_H
 
-#include <misc/dead.h>
 #include <system/DebugObject.h>
 #include <system/BReactor.h>
 #include <flow/PacketPassInterface.h>
@@ -61,14 +60,12 @@ typedef void (*PacketPassInactivityMonitor_handler) (void *user);
  */
 typedef struct {
     DebugObject d_obj;
-    dead_t dead;
     PacketPassInterface *output;
     BReactor *reactor;
     PacketPassInactivityMonitor_handler handler;
     void *user;
     PacketPassInterface input;
     BTimer timer;
-    int send_called;
 } PacketPassInactivityMonitor;
 
 /**
@@ -78,7 +75,7 @@ typedef struct {
  * @param o the object
  * @param output output interface
  * @param reactor reactor we live in
- * @param interval timer interval is milliseconds. Must be >0.
+ * @param interval timer value in milliseconds
  * @param handler handler function for reporting inactivity
  * @param user value passed to handler functions
  */

flow/PacketPassInterface.c

@@ -0,0 +1,81 @@
+/**
+ * @file PacketPassInterface.c
+ * @author Ambroz Bizjak <[email protected]>
+ * 
+ * @section LICENSE
+ * 
+ * This file is part of BadVPN.
+ * 
+ * BadVPN is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ * 
+ * BadVPN is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <flow/PacketPassInterface.h>
+
+void _PacketPassInterface_job_operation (PacketPassInterface *i)
+{
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len >= 0)
+    ASSERT(i->buf_len <= i->mtu)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_cancel);
+    DebugIn_AmOut(&i->d_in_done);
+    #endif
+    
+    // call operation handler
+    #ifndef NDEBUG
+    DebugIn_GoIn(&i->d_in_operation);
+    DEAD_ENTER(i->d_dead)
+    #endif
+    i->handler_operation(i->user_provider, i->buf, i->buf_len);
+    #ifndef NDEBUG
+    if (DEAD_LEAVE(i->d_dead)) {
+        return;
+    }
+    DebugIn_GoOut(&i->d_in_operation);
+    #endif
+}
+
+void _PacketPassInterface_job_done (PacketPassInterface *i)
+{
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len >= 0)
+    ASSERT(i->buf_len <= i->mtu)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_cancel);
+    DebugIn_AmOut(&i->d_in_done);
+    #endif
+    
+    #ifndef NDEBUG
+    i->d_user_busy = 0;
+    #endif
+    
+    // call done handler
+    #ifndef NDEBUG
+    DebugIn_GoIn(&i->d_in_done);
+    DEAD_ENTER(i->d_dead)
+    #endif
+    i->handler_done(i->user_user);
+    #ifndef NDEBUG
+    if (DEAD_LEAVE(i->d_dead)) {
+        return;
+    }
+    DebugIn_GoOut(&i->d_in_done);
+    #endif
+}

flow/PacketPassInterface.h

@@ -32,235 +32,117 @@
 
 #include <misc/dead.h>
 #include <misc/debug.h>
+#include <misc/debugin.h>
 #include <system/DebugObject.h>
+#include <system/BPending.h>
 
-/**
- * Handler called at the receiver when {@link PacketPassInterface_Sender_Send} is called
- * from the sender.
- * It is guaranteed that the interface is in not sending state.
- * It is guaranteed that the handler is not being called from within Send or Cancel handlers.
- *
- * @param user value supplied to {@link PacketPassInterface_Init}
- * @param data pointer to packet being sent. May be NULL if data_len=0.
- * @param data_len length of the packet being sent. Will be >=0 and <=MTU.
- * @return - 1 if the receiver accepts the packet immediately. The interface remains in
- *           not sending state. The receiver may not use the provided data after the handler
- *           returns.
- *         - 0 if the receiver cannot accept the packet immediately. The interface enters
- *           sending state as the handler returns. The receiver may use the provided data
- *           as long as it needs to. When it's done processing the packet and doesn't need
- *           the data any more, it must call {@link PacketPassInterface_Done}.
- */
-typedef int (*PacketPassInterface_handler_send) (void *user, uint8_t *data, int data_len);
+typedef void (*PacketPassInterface_handler_send) (void *user, uint8_t *data, int data_len);
 
-/**
- * Handler called at the receiver when {@link PacketPassInterface_Sender_Cancel} is called
- * from the sender.
- * The buffer is still available inside the handler. It is no longer available
- * after the handler returns.
- * It is guaranteed that the interface is in sending state.
- * The interface enters not sending state as the handler returns.
- * It is guaranteed that the handler is not being called from within Send or Cancel handlers.
- *
- * @param user value supplied to {@link PacketPassInterface_Init}
- */
 typedef void (*PacketPassInterface_handler_cancel) (void *user);
 
-/**
- * Handler called at the sender when {@link PacketPassInterface_Done} is called from the receiver.
- * The receiver will no longer use the packet it was provided with.
- * It is guaranteed that the interface was in sending state.
- * The interface enters not sending state before the handler is called.
- * It is guaranteed that the handler is not being called from within Send, Cancel or Done handlers.
- *
- * @param user value supplied to {@link PacketPassInterface_Sender_Init}
- */
 typedef void (*PacketPassInterface_handler_done) (void *user);
 
-/**
- * Interface allowing a packet sender to pass data packets to a packet receiver.
- * The sender passes a packet by providing the receiver with a pointer
- * to a packet. The receiver may then either accept the packet immediately,
- * or tell the sender to wait for the packet to be processed and inform it
- * when it's done.
- */
 typedef struct {
-    DebugObject d_obj;
-    
-    // receiver data
+    // provider data
     int mtu;
-    PacketPassInterface_handler_send handler_send;
+    PacketPassInterface_handler_send handler_operation;
     PacketPassInterface_handler_cancel handler_cancel;
-    void *user_receiver;
+    void *user_provider;
     
-    // sender data
+    // user data
     PacketPassInterface_handler_done handler_done;
-    void *user_sender;
+    void *user_user;
+    
+    // jobs
+    BPending job_operation;
+    BPending job_done;
+    
+    // packet supplied by user
+    uint8_t *buf;
+    int buf_len;
     
-    // debug vars
+    DebugObject d_obj;
     #ifndef NDEBUG
-    dead_t debug_dead;
-    int debug_busy;
-    int debug_in_send;
-    int debug_in_done;
+    DebugIn d_in_operation;
+    DebugIn d_in_cancel;
+    DebugIn d_in_done;
+    dead_t d_dead;
+    int d_user_busy;
     #endif
 } PacketPassInterface;
 
-/**
- * Initializes the interface. The sender portion must also be initialized
- * with {@link PacketPassInterface_Sender_Init} before I/O can start.
- * The interface is initialized in not sending state.
- *
- * @param i the object
- * @param mtu maximum packet size the receiver can accept. Must be >=0.
- * @param handler_send handler called when the sender wants to send a packet
- * @param user arbitrary value that will be passed to receiver callback functions
- */
-static void PacketPassInterface_Init (PacketPassInterface *i, int mtu, PacketPassInterface_handler_send handler_send, void *user);
+static void PacketPassInterface_Init (PacketPassInterface *i, int mtu, PacketPassInterface_handler_send handler_operation, void *user, BPendingGroup *pg);
 
-/**
- * Frees the interface.
- *
- * @param i the object
- */
 static void PacketPassInterface_Free (PacketPassInterface *i);
 
-/**
- * Enables cancel functionality for the interface.
- * May only be called once for the interface.
- *
- * @param i the object
- * @param handler_cancel callback function invoked when the sender wants to cancel sending
- */
 static void PacketPassInterface_EnableCancel (PacketPassInterface *i, PacketPassInterface_handler_cancel handler_cancel);
 
-/**
- * Notifies the sender that the receiver has finished processing the packet being sent.
- * The receiver must not use the data it was provided any more.
- * The interface must be in sending state.
- * The interface enters not sending state before notifying the sender.
- * Must not be called from within Send, Cancel or Done handlers.
- *
- * Be aware that the sender may attempt to send packets from within this function.
- *
- * @param i the object
- */
 static void PacketPassInterface_Done (PacketPassInterface *i);
 
-/**
- * Returns the maximum packet size the receiver can accept.
- *
- * @return maximum packet size. Will be >=0.
- */
 static int PacketPassInterface_GetMTU (PacketPassInterface *i);
 
-/**
- * Initializes the sender portion of the interface.
- *
- * @param i the object
- * @param handler_done handler called when the receiver has finished processing a packet
- * @param user arbitrary value that will be passed to sender callback functions
- */
 static void PacketPassInterface_Sender_Init (PacketPassInterface *i, PacketPassInterface_handler_done handler_done, void *user);
 
-/**
- * Attempts to send a packet.
- * The interface must be in not sending state.
- * Must not be called from within Send or Cancel handlers.
- *
- * @param i the object
- * @param data pointer to the packet to send. If the size of the packet is zero, this argument
- *             is ignored.
- * @param data_len length of the packet. Must be >=0 and <=MTU.
- * @return - 1 if the packet was accepted by the receiver. The packet is no longer needed.
- *           The interface remains in not sending state.
- *         - 0 if the packet could not be accepted immediately and is being processed.
- *           The interface enters sending state, and the packet must stay accessible while the
- *           receiver is processing it. When the receiver is done processing it, the
- *           {@link PacketPassInterface_handler_done} handler will be called.
- */
-static int PacketPassInterface_Sender_Send (PacketPassInterface *i, uint8_t *data, int data_len);
+static void PacketPassInterface_Sender_Send (PacketPassInterface *i, uint8_t *data, int data_len);
 
-/**
- * Cancels sending a packet.
- * Cancel functionality must be available for the interface.
- * The buffer must still be available while calling this.
- * The buffer is no longer needed after this function returns.
- * The interface must be in sending state.
- * The interface enters not sending state.
- * Must not be called from within Send or Cancel handlers.
- *
- * @param i the object
- */
 static void PacketPassInterface_Sender_Cancel (PacketPassInterface *i);
 
-/**
- * Determines if the interface supports cancel functionality.
- * 
- * @param i the object
- * @return 1 if the interface supports cancel functionality, 0 if not
- */
 static int PacketPassInterface_HasCancel (PacketPassInterface *i);
 
 #ifndef NDEBUG
 
-/**
- * Determines if we are in a Send or Cancel call.
- * Only available if NDEBUG is not defined.
- * 
- * @param i the object
- * @return 1 if in a Send or Cancel call, 0 if not
- */
 static int PacketPassInterface_InClient (PacketPassInterface *i);
 
-/**
- * Determines if we are in a Done call.
- * Only available if NDEBUG is not defined.
- * 
- * @param i the object
- * @return 1 if in a Done call, 0 if not
- */
 static int PacketPassInterface_InDone (PacketPassInterface *i);
 
 #endif
 
-void PacketPassInterface_Init (PacketPassInterface *i, int mtu, PacketPassInterface_handler_send handler_send, void *user)
+void _PacketPassInterface_job_operation (PacketPassInterface *i);
+void _PacketPassInterface_job_done (PacketPassInterface *i);
+
+void PacketPassInterface_Init (PacketPassInterface *i, int mtu, PacketPassInterface_handler_send handler_operation, void *user, BPendingGroup *pg)
 {
     ASSERT(mtu >= 0)
     
+    // init arguments
     i->mtu = mtu;
-    i->handler_send = handler_send;
+    i->handler_operation = handler_operation;
     i->handler_cancel = NULL;
-    i->user_receiver = user;
+    i->user_provider = user;
+    
+    // set no user
     i->handler_done = NULL;
-    i->user_sender = NULL;
     
-    // init debugging
-    #ifndef NDEBUG
-    DEAD_INIT(i->debug_dead);
-    i->debug_busy = 0;
-    i->debug_in_send = 0;
-    i->debug_in_done = 0;
-    #endif
+    // init jobs
+    BPending_Init(&i->job_operation, pg, (BPending_handler)_PacketPassInterface_job_operation, i);
+    BPending_Init(&i->job_done, pg, (BPending_handler)_PacketPassInterface_job_done, i);
     
-    // init debug object
     DebugObject_Init(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_Init(&i->d_in_operation);
+    DebugIn_Init(&i->d_in_cancel);
+    DebugIn_Init(&i->d_in_done);
+    DEAD_INIT(i->d_dead);
+    i->d_user_busy = 0;
+    #endif
 }
 
 void PacketPassInterface_Free (PacketPassInterface *i)
 {
-    // free debug object
-    DebugObject_Free(&i->d_obj);
-    
-    // free debugging
     #ifndef NDEBUG
-    DEAD_KILL(i->debug_dead);
+    DEAD_KILL(i->d_dead);
     #endif
+    DebugObject_Free(&i->d_obj);
+    
+    // free jobs
+    BPending_Free(&i->job_done);
+    BPending_Free(&i->job_operation);
 }
 
 void PacketPassInterface_EnableCancel (PacketPassInterface *i, PacketPassInterface_handler_cancel handler_cancel)
 {
     ASSERT(!i->handler_cancel)
+    ASSERT(!i->handler_done)
     ASSERT(handler_cancel)
     
     i->handler_cancel = handler_cancel;
@@ -268,91 +150,98 @@ void PacketPassInterface_EnableCancel (PacketPassInterface *i, PacketPassInterfa
 
 void PacketPassInterface_Done (PacketPassInterface *i)
 {
-    ASSERT(i->debug_busy)
-    ASSERT(!i->debug_in_send)
-    ASSERT(!i->debug_in_done)
-    
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len >= 0)
+    ASSERT(i->buf_len <= i->mtu)
+    ASSERT(i->handler_done)
+    ASSERT(!BPending_IsSet(&i->job_operation))
+    DebugObject_Access(&i->d_obj);
     #ifndef NDEBUG
-    i->debug_busy = 0;
-    i->debug_in_done = 1;
-    DEAD_ENTER(i->debug_dead)
+    DebugIn_AmOut(&i->d_in_cancel);
+    DebugIn_AmOut(&i->d_in_done);
     #endif
     
-    i->handler_done(i->user_sender);
-    
-    #ifndef NDEBUG
-    if (DEAD_LEAVE(i->debug_dead)) {
-        return;
-    }
-    i->debug_in_done = 0;
-    #endif
+    BPending_Set(&i->job_done);
 }
 
 int PacketPassInterface_GetMTU (PacketPassInterface *i)
 {
+    DebugObject_Access(&i->d_obj);
+    
     return i->mtu;
 }
 
 void PacketPassInterface_Sender_Init (PacketPassInterface *i, PacketPassInterface_handler_done handler_done, void *user)
 {
+    ASSERT(handler_done)
+    ASSERT(!i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    
     i->handler_done = handler_done;
-    i->user_sender = user;
+    i->user_user = user;
 }
 
-int PacketPassInterface_Sender_Send (PacketPassInterface *i, uint8_t *data, int data_len)
+void PacketPassInterface_Sender_Send (PacketPassInterface *i, uint8_t *data, int data_len)
 {
-    ASSERT(!i->debug_busy)
-    ASSERT(!i->debug_in_send)
     ASSERT(data_len >= 0)
     ASSERT(data_len <= i->mtu)
     ASSERT(!(data_len > 0) || data)
-    
+    ASSERT(!i->d_user_busy)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
     #ifndef NDEBUG
-    i->debug_in_send = 1;
-    DEAD_ENTER(i->debug_dead)
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_cancel);
     #endif
     
-    int res = i->handler_send(i->user_receiver, data, data_len);
+    i->buf = data;
+    i->buf_len = data_len;
     
     #ifndef NDEBUG
-    if (DEAD_LEAVE(i->debug_dead)) {
-        return -1;
-    }
-    i->debug_in_send = 0;
-    ASSERT(res == 0 || res == 1)
-    if (!res) {
-        i->debug_busy = 1;
-    }
+    i->d_user_busy = 1;
     #endif
-
-    return res;
+    
+    BPending_Set(&i->job_operation);
 }
 
 void PacketPassInterface_Sender_Cancel (PacketPassInterface *i)
 {
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len >= 0)
+    ASSERT(i->buf_len <= i->mtu)
     ASSERT(i->handler_cancel)
-    ASSERT(i->debug_busy)
-    ASSERT(!i->debug_in_send)
-    
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
     #ifndef NDEBUG
-    i->debug_in_send = 1;
-    DEAD_ENTER(i->debug_dead)
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_cancel);
     #endif
     
-    i->handler_cancel(i->user_receiver);
+    BPending_Unset(&i->job_operation);
+    BPending_Unset(&i->job_done);
     
     #ifndef NDEBUG
-    if (DEAD_LEAVE(i->debug_dead)) {
-        return;
-    }
-    ASSERT(i->debug_in_send)
-    i->debug_in_send = 0;
-    i->debug_busy = 0;
+    i->d_user_busy = 0;
     #endif
+    
+    if (!BPending_IsSet(&i->job_operation) && !BPending_IsSet(&i->job_done)) {
+        #ifndef NDEBUG
+        DebugIn_GoIn(&i->d_in_cancel);
+        DEAD_ENTER(i->d_dead)
+        #endif
+        i->handler_cancel(i->user_provider);
+        #ifndef NDEBUG
+        ASSERT(!DEAD_KILLED)
+        DEAD_LEAVE(i->d_dead);
+        DebugIn_GoOut(&i->d_in_cancel);
+        #endif
+    }
 }
 
 int PacketPassInterface_HasCancel (PacketPassInterface *i)
 {
+    DebugObject_Access(&i->d_obj);
+    
     return !!i->handler_cancel;
 }
 
@@ -360,12 +249,16 @@ int PacketPassInterface_HasCancel (PacketPassInterface *i)
 
 int PacketPassInterface_InClient (PacketPassInterface *i)
 {
-    return i->debug_in_send;
+    DebugObject_Access(&i->d_obj);
+    
+    return DebugIn_In(&i->d_in_operation);
 }
 
 int PacketPassInterface_InDone (PacketPassInterface *i)
 {
-    return i->debug_in_done;
+    DebugObject_Access(&i->d_obj);
+    
+    return DebugIn_In(&i->d_in_done);
 }
 
 #endif

flow/PacketPassNotifier.c

@@ -25,91 +25,75 @@
 #include <flow/PacketPassNotifier.h>
 
 static int call_handler (PacketPassNotifier *o, uint8_t *data, int data_len);
-static int input_handler_send (PacketPassNotifier *o, uint8_t *data, int data_len);
+static void input_handler_send (PacketPassNotifier *o, uint8_t *data, int data_len);
 static void input_handler_cancel (PacketPassNotifier *o);
 static void output_handler_done (PacketPassNotifier *o);
 
 int call_handler (PacketPassNotifier *o, uint8_t *data, int data_len)
 {
     ASSERT(o->handler)
-    ASSERT(!o->in_handler)
-    
-    #ifndef NDEBUG
-    o->in_handler = 1;
-    #endif
+    DebugIn_AmOut(&o->d_in_handler);
+    DebugObject_Access(&o->d_obj);
     
+    DebugIn_GoIn(&o->d_in_handler);
     DEAD_ENTER(o->dead)
     o->handler(o->handler_user, data, data_len);
     if (DEAD_LEAVE(o->dead)) {
         return -1;
     }
-    
-    #ifndef NDEBUG
-    o->in_handler = 0;
-    #endif
+    DebugIn_GoOut(&o->d_in_handler);
     
     return 0;
 }
 
-int input_handler_send (PacketPassNotifier *o, uint8_t *data, int data_len)
+void input_handler_send (PacketPassNotifier *o, uint8_t *data, int data_len)
 {
-    ASSERT(!o->in_have)
-    ASSERT(!o->in_handler)
+    ASSERT(!o->d_in_have)
+    DebugIn_AmOut(&o->d_in_handler);
+    DebugObject_Access(&o->d_obj);
+    
+    // schedule send
+    PacketPassInterface_Sender_Send(o->output, data, data_len);
     
     // if we have a handler, call it
     if (o->handler) {
         if (call_handler(o, data, data_len) < 0) {
-            return -1;
+            return;
         }
     }
     
-    // call send on output
-    DEAD_ENTER(o->dead)
-    int res = PacketPassInterface_Sender_Send(o->output, data, data_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    if (!res) {
-        // output blocking, continue in output_handler_done
-        #ifndef NDEBUG
-        o->in_have = 1;
-        #endif
-        return 0;
-    }
-    
-    return 1;
+    #ifndef NDEBUG
+    o->d_in_have = 1;
+    #endif
 }
 
 void input_handler_cancel (PacketPassNotifier *o)
 {
-    ASSERT(o->in_have)
-    ASSERT(!o->in_handler)
+    ASSERT(o->d_in_have)
+    DebugIn_AmOut(&o->d_in_handler);
+    DebugObject_Access(&o->d_obj);
+    
+    PacketPassInterface_Sender_Cancel(o->output);
     
     #ifndef NDEBUG
-    o->in_have = 0;
+    o->d_in_have = 0;
     #endif
-    
-    PacketPassInterface_Sender_Cancel(o->output);
-    return;
 }
 
 void output_handler_done (PacketPassNotifier *o)
 {
-    ASSERT(o->in_have)
-    ASSERT(!o->in_handler)
+    ASSERT(o->d_in_have)
+    DebugIn_AmOut(&o->d_in_handler);
+    DebugObject_Access(&o->d_obj);
+    
+    PacketPassInterface_Done(&o->input);
     
     #ifndef NDEBUG
-    o->in_have = 0;
+    o->d_in_have = 0;
     #endif
-    
-    PacketPassInterface_Done(&o->input);
-    return;
 }
 
-void PacketPassNotifier_Init (PacketPassNotifier *o, PacketPassInterface *output)
+void PacketPassNotifier_Init (PacketPassNotifier *o, PacketPassInterface *output, BPendingGroup *pg)
 {
     // init arguments
     o->output = output;
@@ -118,7 +102,7 @@ void PacketPassNotifier_Init (PacketPassNotifier *o, PacketPassInterface *output
     DEAD_INIT(o->dead);
     
     // init input
-    PacketPassInterface_Init(&o->input, PacketPassInterface_GetMTU(o->output), (PacketPassInterface_handler_send)input_handler_send, o);
+    PacketPassInterface_Init(&o->input, PacketPassInterface_GetMTU(o->output), (PacketPassInterface_handler_send)input_handler_send, o, pg);
     if (PacketPassInterface_HasCancel(o->output)) {
         PacketPassInterface_EnableCancel(&o->input, (PacketPassInterface_handler_cancel)input_handler_cancel);
     }
@@ -129,19 +113,15 @@ void PacketPassNotifier_Init (PacketPassNotifier *o, PacketPassInterface *output
     // set no handler
     o->handler = NULL;
     
-    // init debugging
+    DebugObject_Init(&o->d_obj);
+    DebugIn_Init(&o->d_in_handler);
     #ifndef NDEBUG
-    o->in_have = 0;
-    o->in_handler = 0;
+    o->d_in_have = 0;
     #endif
-    
-    // init debug object
-    DebugObject_Init(&o->d_obj);
 }
 
 void PacketPassNotifier_Free (PacketPassNotifier *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // free input
@@ -153,11 +133,15 @@ void PacketPassNotifier_Free (PacketPassNotifier *o)
 
 PacketPassInterface * PacketPassNotifier_GetInput (PacketPassNotifier *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->input;
 }
 
 void PacketPassNotifier_SetHandler (PacketPassNotifier *o, PacketPassNotifier_handler_notify handler, void *user)
 {
+    DebugObject_Access(&o->d_obj);
+    
     o->handler = handler;
     o->handler_user = user;
 }

flow/PacketPassNotifier.h

@@ -31,6 +31,7 @@
 #include <stdint.h>
 
 #include <misc/dead.h>
+#include <misc/debugin.h>
 #include <system/DebugObject.h>
 #include <flow/PacketPassInterface.h>
 
@@ -48,15 +49,15 @@ typedef void (*PacketPassNotifier_handler_notify) (void *user, uint8_t *data, in
  * passing a packet from input to output.
  */
 typedef struct {
-    DebugObject d_obj;
     dead_t dead;
     PacketPassInterface input;
     PacketPassInterface *output;
     PacketPassNotifier_handler_notify handler;
     void *handler_user;
+    DebugObject d_obj;
+    DebugIn d_in_handler;
     #ifndef NDEBUG
-    int in_have;
-    int in_handler;
+    int d_in_have;
     #endif
 } PacketPassNotifier;
 
@@ -65,8 +66,9 @@ typedef struct {
  *
  * @param o the object
  * @param output output interface
+ * @param pg pending group
  */
-void PacketPassNotifier_Init (PacketPassNotifier *o, PacketPassInterface *output);
+void PacketPassNotifier_Init (PacketPassNotifier *o, PacketPassInterface *output, BPendingGroup *pg);
 
 /**
  * Frees the object.

flow/PacketPassPriorityQueue.c

@@ -27,132 +27,67 @@
 
 #include <flow/PacketPassPriorityQueue.h>
 
-static int call_send (PacketPassPriorityQueue *m, uint8_t *data, int data_len)
+static int int_comparator (void *user, int *prio1, int *prio2)
 {
-    DebugIn_GoIn(&m->in_output);
-    DEAD_ENTER(m->dead)
-    int res = PacketPassInterface_Sender_Send(m->output, data, data_len);
-    if (DEAD_LEAVE(m->dead)) {
+    if (*prio1 < *prio2) {
         return -1;
     }
-    DebugIn_GoOut(&m->in_output);
-    
-    ASSERT(!m->freeing)
-    ASSERT(res == 0 || res == 1)
-    
-    return res;
-}
-
-static int call_cancel (PacketPassPriorityQueue *m)
-{
-    DebugIn_GoIn(&m->in_output);
-    DEAD_ENTER(m->dead)
-    PacketPassInterface_Sender_Cancel(m->output);
-    if (DEAD_LEAVE(m->dead)) {
-        return -1;
+    if (*prio1 > *prio2) {
+        return 1;
     }
-    DebugIn_GoOut(&m->in_output);
-    
-    ASSERT(!m->freeing)
-    
     return 0;
 }
 
-static int call_done (PacketPassPriorityQueue *m, PacketPassPriorityQueueFlow *flow)
+static void schedule (PacketPassPriorityQueue *m)
 {
-    DEAD_ENTER(m->dead)
-    PacketPassInterface_Done(&flow->input);
-    if (DEAD_LEAVE(m->dead)) {
-        return -1;
-    }
-    
     ASSERT(!m->freeing)
+    ASSERT(!m->sending_flow)
+    ASSERT(BHeap_GetFirst(&m->queued_heap))
     
-    return 0;
+    // get first queued flow
+    BHeapNode *heap_node = BHeap_GetFirst(&m->queued_heap);
+    PacketPassPriorityQueueFlow *qflow = UPPER_OBJECT(heap_node, PacketPassPriorityQueueFlow, queued.heap_node);
+    ASSERT(qflow->is_queued)
+    
+    // remove flow from queue
+    BHeap_Remove(&m->queued_heap, &qflow->queued.heap_node);
+    qflow->is_queued = 0;
+    
+    // schedule send
+    PacketPassInterface_Sender_Send(m->output, qflow->queued.data, qflow->queued.data_len);
+    m->sending_flow = qflow;
+    m->sending_len = qflow->queued.data_len;
 }
 
-static void process_queue (PacketPassPriorityQueue *m)
+static void schedule_job_handler (PacketPassPriorityQueue *m)
 {
     ASSERT(!m->freeing)
     ASSERT(!m->sending_flow)
+    DebugObject_Access(&m->d_obj);
     
-    do {
-        // get first queued flow
-        BHeapNode *heap_node = BHeap_GetFirst(&m->queued_heap);
-        if (!heap_node) {
-            return;
-        }
-        PacketPassPriorityQueueFlow *qflow = UPPER_OBJECT(heap_node, PacketPassPriorityQueueFlow, queued.heap_node);
-        ASSERT(qflow->is_queued)
-        
-        // remove flow from queue
-        BHeap_Remove(&m->queued_heap, &qflow->queued.heap_node);
-        qflow->is_queued = 0;
-        
-        // try to send the packet
-        int res = call_send(m, qflow->queued.data, qflow->queued.data_len);
-        if (res < 0) {
-            return;
-        }
-        
-        if (res == 0) {
-            // sending in progress
-            m->sending_flow = qflow;
-            m->sending_len = qflow->queued.data_len;
-            return;
-        }
-        
-        // notify sender
-        if (call_done(m, qflow) < 0) {
-            return;
-        }
-    } while (!m->sending_flow);
-}
-
-static int int_comparator (void *user, int *prio1, int *prio2)
-{
-    if (*prio1 < *prio2) {
-        return -1;
-    }
-    if (*prio1 > *prio2) {
-        return 1;
+    if (BHeap_GetFirst(&m->queued_heap)) {
+        schedule(m);
     }
-    return 0;
 }
 
-static int input_handler_send (PacketPassPriorityQueueFlow *flow, uint8_t *data, int data_len)
+static void input_handler_send (PacketPassPriorityQueueFlow *flow, uint8_t *data, int data_len)
 {
     ASSERT(!flow->m->freeing)
     ASSERT(flow != flow->m->sending_flow)
     ASSERT(!flow->is_queued)
-    DebugIn_AmOut(&flow->m->in_output);
+    DebugObject_Access(&flow->d_obj);
     
     PacketPassPriorityQueue *m = flow->m;
     
-    // if nothing is being sent and queue is empty, send immediately without queueing
-    if (!m->sending_flow && !BHeap_GetFirst(&m->queued_heap)) {
-        int res = call_send(m, data, data_len);
-        if (res < 0) {
-            return -1;
-        }
-        
-        if (res == 0) {
-            // output busy, continue in output_handler_done
-            m->sending_flow = flow;
-            m->sending_len = data_len;
-            return 0;
-        }
-        
-        return 1;
-    }
-    
-    // add flow to queue
+    // queue flow
     flow->queued.data = data;
     flow->queued.data_len = data_len;
     BHeap_Insert(&m->queued_heap, &flow->queued.heap_node);
     flow->is_queued = 1;
     
-    return 0;
+    if (!m->sending_flow && !BPending_IsSet(&m->schedule_job)) {
+        schedule(m);
+    }
 }
 
 static void output_handler_done (PacketPassPriorityQueue *m)
@@ -160,13 +95,19 @@ static void output_handler_done (PacketPassPriorityQueue *m)
     ASSERT(!m->freeing)
     ASSERT(m->sending_flow)
     ASSERT(!m->sending_flow->is_queued)
-    DebugIn_AmOut(&m->in_output);
+    ASSERT(!BPending_IsSet(&m->schedule_job))
     
     PacketPassPriorityQueueFlow *flow = m->sending_flow;
     
     // sending finished
     m->sending_flow = NULL;
     
+    // schedule schedule
+    BPending_Set(&m->schedule_job);
+    
+    // finish flow packet
+    PacketPassInterface_Done(&flow->input);
+    
     // call busy handler if set
     if (flow->handler_busy) {
         // handler is one-shot, unset it before calling
@@ -174,42 +115,14 @@ static void output_handler_done (PacketPassPriorityQueue *m)
         flow->handler_busy = NULL;
         
         // call handler
-        DEAD_ENTER_N(m, m->dead)
-        DEAD_ENTER_N(flow, flow->dead)
+        DEAD_ENTER(m->dead)
         handler(flow->user);
-        DEAD_LEAVE_N(m, m->dead);
-        DEAD_LEAVE_N(flow, flow->dead);
-        if (DEAD_KILLED_N(m)) {
+        if (DEAD_LEAVE(m->dead)) {
             return;
         }
-        if (DEAD_KILLED_N(flow)) {
-            flow = NULL;
-        }
         
         ASSERT(!m->freeing)
-    }
-    
-    // report completion to sender
-    if (flow) {
-        if (call_done(m, flow) < 0) {
-            return;
-        }
-    }
-    
-    // process queued flows
-    if (!m->sending_flow) {
-        process_queue(m);
-        return;
-    }
-}
-
-static void job_handler (PacketPassPriorityQueue *m)
-{
-    ASSERT(!m->freeing)
-    
-    if (!m->sending_flow) {
-        process_queue(m);
-        return;
+        ASSERT(!m->sending_flow)
     }
 }
 
@@ -217,6 +130,7 @@ void PacketPassPriorityQueue_Init (PacketPassPriorityQueue *m, PacketPassInterfa
 {
     // init arguments
     m->output = output;
+    m->pg = pg;
     
     // init dead var
     DEAD_INIT(m->dead);
@@ -236,16 +150,10 @@ void PacketPassPriorityQueue_Init (PacketPassPriorityQueue *m, PacketPassInterfa
     // not using cancel
     m->use_cancel = 0;
     
-    // init continue job
-    BPending_Init(&m->continue_job, pg, (BPending_handler)job_handler, m);
+    // init schedule job
+    BPending_Init(&m->schedule_job, m->pg, (BPending_handler)schedule_job_handler, m);
     
-    // init debug counter
     DebugCounter_Init(&m->d_ctr);
-    
-    // init debug in output
-    DebugIn_Init(&m->in_output);
-    
-    // init debug object
     DebugObject_Init(&m->d_obj);
 }
 
@@ -256,8 +164,8 @@ void PacketPassPriorityQueue_Free (PacketPassPriorityQueue *m)
     DebugCounter_Free(&m->d_ctr);
     DebugObject_Free(&m->d_obj);
     
-    // free continue job
-    BPending_Free(&m->continue_job);
+    // free schedule job
+    BPending_Free(&m->schedule_job);
     
     // free dead var
     DEAD_KILL(m->dead);
@@ -267,6 +175,7 @@ void PacketPassPriorityQueue_EnableCancel (PacketPassPriorityQueue *m)
 {
     ASSERT(!m->use_cancel)
     ASSERT(PacketPassInterface_HasCancel(m->output))
+    DebugObject_Access(&m->d_obj);
     
     // using cancel
     m->use_cancel = 1;
@@ -274,34 +183,30 @@ void PacketPassPriorityQueue_EnableCancel (PacketPassPriorityQueue *m)
 
 void PacketPassPriorityQueue_PrepareFree (PacketPassPriorityQueue *m)
 {
+    DebugObject_Access(&m->d_obj);
+    
     m->freeing = 1;
 }
 
 void PacketPassPriorityQueueFlow_Init (PacketPassPriorityQueueFlow *flow, PacketPassPriorityQueue *m, int priority)
 {
     ASSERT(!m->freeing)
-    DebugIn_AmOut(&m->in_output);
+    DebugObject_Access(&m->d_obj);
     
     // init arguments
     flow->m = m;
     flow->priority = priority;
     
-    // init dead var
-    DEAD_INIT(flow->dead);
-    
     // have no canfree handler
     flow->handler_busy = NULL;
     
     // init input
-    PacketPassInterface_Init(&flow->input, PacketPassInterface_GetMTU(flow->m->output), (PacketPassInterface_handler_send)input_handler_send, flow);
+    PacketPassInterface_Init(&flow->input, PacketPassInterface_GetMTU(flow->m->output), (PacketPassInterface_handler_send)input_handler_send, flow, m->pg);
     
     // is not queued
     flow->is_queued = 0;
     
-    // increment debug counter
     DebugCounter_Increment(&m->d_ctr);
-    
-    // init debug object
     DebugObject_Init(&flow->d_obj);
 }
 
@@ -309,16 +214,15 @@ void PacketPassPriorityQueueFlow_Free (PacketPassPriorityQueueFlow *flow)
 {
     if (!flow->m->freeing) {
         ASSERT(flow != flow->m->sending_flow)
-        DebugIn_AmOut(&flow->m->in_output);
     }
     DebugCounter_Decrement(&flow->m->d_ctr);
     DebugObject_Free(&flow->d_obj);
     
     PacketPassPriorityQueue *m = flow->m;
     
-    // remove current flow
-    if (flow == flow->m->sending_flow) {
-        flow->m->sending_flow = NULL;
+    // remove from current flow
+    if (flow == m->sending_flow) {
+        m->sending_flow = NULL;
     }
     
     // remove from queue
@@ -328,15 +232,20 @@ void PacketPassPriorityQueueFlow_Free (PacketPassPriorityQueueFlow *flow)
     
     // free input
     PacketPassInterface_Free(&flow->input);
-    
-    // free dead var
-    DEAD_KILL(flow->dead);
+}
+
+void PacketPassPriorityQueueFlow_AssertFree (PacketPassPriorityQueueFlow *flow)
+{
+    if (!flow->m->freeing) {
+        ASSERT(flow != flow->m->sending_flow)
+    }
+    DebugObject_Access(&flow->d_obj);
 }
 
 int PacketPassPriorityQueueFlow_IsBusy (PacketPassPriorityQueueFlow *flow)
 {
     ASSERT(!flow->m->freeing)
-    DebugIn_AmOut(&flow->m->in_output);
+    DebugObject_Access(&flow->d_obj);
     
     return (flow == flow->m->sending_flow);
 }
@@ -346,27 +255,26 @@ void PacketPassPriorityQueueFlow_Release (PacketPassPriorityQueueFlow *flow)
     ASSERT(flow->m->use_cancel)
     ASSERT(flow == flow->m->sending_flow)
     ASSERT(!flow->m->freeing)
-    DebugIn_AmOut(&flow->m->in_output);
+    ASSERT(!BPending_IsSet(&flow->m->schedule_job))
+    DebugObject_Access(&flow->d_obj);
     
     PacketPassPriorityQueue *m = flow->m;
     
     // cancel current packet
-    if (call_cancel(m) < 0) {
-        return;
-    }
+    PacketPassInterface_Sender_Cancel(m->output);
     
     // set no sending flow
     m->sending_flow = NULL;
     
-    // set continue job
-    BPending_Set(&m->continue_job);
+    // schedule schedule
+    BPending_Set(&m->schedule_job);
 }
 
 void PacketPassPriorityQueueFlow_SetBusyHandler (PacketPassPriorityQueueFlow *flow, PacketPassPriorityQueue_handler_busy handler, void *user)
 {
     ASSERT(flow == flow->m->sending_flow)
     ASSERT(!flow->m->freeing)
-    DebugIn_AmOut(&flow->m->in_output);
+    DebugObject_Access(&flow->d_obj);
     
     flow->handler_busy = handler;
     flow->user = user;
@@ -374,5 +282,7 @@ void PacketPassPriorityQueueFlow_SetBusyHandler (PacketPassPriorityQueueFlow *fl
 
 PacketPassInterface * PacketPassPriorityQueueFlow_GetInput (PacketPassPriorityQueueFlow *flow)
 {
+    DebugObject_Access(&flow->d_obj);
+    
     return &flow->input;
 }

flow/PacketPassPriorityQueue.h

@@ -30,11 +30,10 @@
 #include <stdint.h>
 
 #include <misc/dead.h>
-#include <misc/debugin.h>
 #include <system/DebugObject.h>
+#include <system/BPending.h>
 #include <misc/debugcounter.h>
 #include <structure/BHeap.h>
-#include <system/BPending.h>
 #include <flow/PacketPassInterface.h>
 
 typedef void (*PacketPassPriorityQueue_handler_busy) (void *user);
@@ -52,14 +51,13 @@ typedef struct {
     BHeap queued_heap;
     int freeing;
     int use_cancel;
-    BPending continue_job;
+    BPending schedule_job;
+    BPendingGroup *pg;
     DebugCounter d_ctr;
-    DebugIn in_output;
     DebugObject d_obj;
 } PacketPassPriorityQueue;
 
 typedef struct PacketPassPriorityQueueFlow_s {
-    dead_t dead;
     PacketPassPriorityQueue *m;
     PacketPassPriorityQueue_handler_busy handler_busy;
     void *user;
@@ -132,6 +130,14 @@ void PacketPassPriorityQueueFlow_Init (PacketPassPriorityQueueFlow *flow, Packet
  */
 void PacketPassPriorityQueueFlow_Free (PacketPassPriorityQueueFlow *flow);
 
+/**
+ * Does nothing.
+ * It must be possible to free the flow (see {@link PacketPassPriorityQueueFlow}).
+ * 
+ * @param flow the object
+ */
+void PacketPassPriorityQueueFlow_AssertFree (PacketPassPriorityQueueFlow *flow);
+
 /**
  * Determines if the flow is busy. If the flow is considered busy, it must not
  * be freed.

flow/PacketProtoDecoder.c

@@ -30,132 +30,21 @@
 #include <flow/PacketProtoDecoder.h>
 
 static void report_error (PacketProtoDecoder *enc, int error);
-static int call_recv (PacketProtoDecoder *enc, uint8_t *data, int avail);
-static int call_send (PacketProtoDecoder *enc, uint8_t *data, int len);
-static void receive_data (PacketProtoDecoder *enc);
+static void process_data (PacketProtoDecoder *enc);
 static void input_handler_done (PacketProtoDecoder *enc, int data_len);
-static int parse_and_send (PacketProtoDecoder *enc);
 static void output_handler_done (PacketProtoDecoder *enc);
-static void job_handler (PacketProtoDecoder *enc);
 
 void report_error (PacketProtoDecoder *enc, int error)
 {
-    #ifndef NDEBUG
     DEAD_ENTER(enc->dead)
-    #endif
-    
     FlowErrorReporter_ReportError(&enc->rep, &error);
-    
-    #ifndef NDEBUG
     ASSERT(DEAD_KILLED)
     DEAD_LEAVE(enc->dead);
-    #endif
-}
-
-int call_recv (PacketProtoDecoder *enc, uint8_t *data, int avail)
-{
-    ASSERT(avail > 0)
-    ASSERT(!StreamRecvInterface_InClient(enc->input))
-    
-    DEAD_ENTER(enc->dead)
-    int res = StreamRecvInterface_Receiver_Recv(enc->input, data, avail);
-    if (DEAD_LEAVE(enc->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res >= 0)
-    ASSERT(res <= avail)
-    
-    return res;
 }
 
-int call_send (PacketProtoDecoder *enc, uint8_t *data, int len)
+void process_data (PacketProtoDecoder *enc)
 {
-    ASSERT(len >= 0)
-    ASSERT(len <= enc->output_mtu)
-    ASSERT(!PacketPassInterface_InClient(enc->output))
-    
-    DEAD_ENTER(enc->dead)
-    int res = PacketPassInterface_Sender_Send(enc->output, data, len);
-    if (DEAD_LEAVE(enc->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    return res;
-}
-
-void receive_data (PacketProtoDecoder *enc)
-{
-    ASSERT(!enc->receiving)
-    ASSERT(!enc->sending)
-    ASSERT(enc->buf_start + enc->buf_used < enc->buf_size)
-    ASSERT(!StreamRecvInterface_InClient(enc->input))
-    ASSERT(!PacketPassInterface_InClient(enc->output))
-    
     do {
-        // receive data
-        int res;
-        if ((res = call_recv(
-            enc,
-            enc->buf + (enc->buf_start + enc->buf_used),
-            enc->buf_size - (enc->buf_start + enc->buf_used)
-        )) < 0) {
-            return;
-        }
-        
-        if (res == 0) {
-            // input busy, continue in input_handler_done
-            enc->receiving = 1;
-            break;
-        }
-        
-        // update buffer
-        enc->buf_used += res;
-        
-        // parse and send data
-        if (parse_and_send(enc) < 0) {
-            return;
-        }
-    } while (!enc->sending && enc->buf_start + enc->buf_used < enc->buf_size);
-}
-
-static void input_handler_done (PacketProtoDecoder *enc, int data_len)
-{
-    ASSERT(enc->receiving)
-    ASSERT(!enc->sending)
-    ASSERT(enc->buf_start + enc->buf_used < enc->buf_size)
-    ASSERT(data_len > 0)
-    ASSERT(data_len <= enc->buf_size - (enc->buf_start + enc->buf_used))
-    ASSERT(!StreamRecvInterface_InClient(enc->input))
-    ASSERT(!PacketPassInterface_InClient(enc->output))
-    
-    // set not receiving
-    enc->receiving = 0;
-    
-    // update buffer
-    enc->buf_used += data_len;
-    
-    // parse and send data
-    if (parse_and_send(enc) < 0) {
-        return;
-    }
-    
-    // continue receiving
-    if (!enc->sending && enc->buf_start + enc->buf_used < enc->buf_size) {
-        receive_data(enc);
-        return;
-    }
-}
-
-int parse_and_send (PacketProtoDecoder *enc)
-{
-    ASSERT(!enc->sending)
-    ASSERT(!StreamRecvInterface_InClient(enc->input))
-    ASSERT(!PacketPassInterface_InClient(enc->output))
-    
-    while (1) {
         uint8_t *data = enc->buf + enc->buf_start;
         int left = enc->buf_used;
         
@@ -171,7 +60,7 @@ int parse_and_send (PacketProtoDecoder *enc)
         // check data length
         if (data_len > enc->output_mtu) {
             report_error(enc, PACKETPROTODECODER_ERROR_TOOLONG);
-            return -1;
+            return;
         }
         
         // check if whole packet was received
@@ -184,17 +73,9 @@ int parse_and_send (PacketProtoDecoder *enc)
         enc->buf_used -= sizeof(struct packetproto_header) + data_len;
         
         // submit packet
-        int res;
-        if ((res = call_send(enc, data, data_len)) < 0) {
-            return -1;
-        }
-        
-        if (!res) {
-            // output busy, continue in output_handler_done
-            enc->sending = 1;
-            return 0;
-        }
-    }
+        PacketPassInterface_Sender_Send(enc->output, data, data_len);
+        return;
+    } while (0);
     
     // if we reached the end of the buffer, wrap around to allow more data to be received
     if (enc->buf_start + enc->buf_used == enc->buf_size) {
@@ -202,33 +83,30 @@ int parse_and_send (PacketProtoDecoder *enc)
         enc->buf_start = 0;
     }
     
-    return 0;
+    // receive data
+    StreamRecvInterface_Receiver_Recv(enc->input, enc->buf + (enc->buf_start + enc->buf_used), enc->buf_size - ((enc->buf_start + enc->buf_used)));
 }
 
-void output_handler_done (PacketProtoDecoder *enc)
+static void input_handler_done (PacketProtoDecoder *enc, int data_len)
 {
-    ASSERT(enc->sending)
-    ASSERT(!StreamRecvInterface_InClient(enc->input))
-    ASSERT(!PacketPassInterface_InClient(enc->output))
-    
-    // set not sending
-    enc->sending = 0;
+    ASSERT(data_len > 0)
+    ASSERT(data_len <= enc->buf_size - (enc->buf_start + enc->buf_used))
+    DebugObject_Access(&enc->d_obj);
     
-    // continue parsing and sending
-    if (parse_and_send(enc) < 0) {
-        return;
-    }
+    // update buffer
+    enc->buf_used += data_len;
     
-    // continue receiving
-    if (!enc->receiving && !enc->sending && enc->buf_start + enc->buf_used < enc->buf_size) {
-        receive_data(enc);
-        return;
-    }
+    // process data
+    process_data(enc);
+    return;
 }
 
-void job_handler (PacketProtoDecoder *enc)
+void output_handler_done (PacketProtoDecoder *enc)
 {
-    receive_data(enc);
+    DebugObject_Access(&enc->d_obj);
+    
+    // process data
+    process_data(enc);
     return;
 }
 
@@ -261,17 +139,9 @@ int PacketProtoDecoder_Init (PacketProtoDecoder *enc, FlowErrorReporter rep, Str
         goto fail0;
     }
     
-    // set not receiving
-    enc->receiving = 0;
+    // start receiving
+    StreamRecvInterface_Receiver_Recv(enc->input, enc->buf, enc->buf_size);
     
-    // set not sending
-    enc->sending = 0;
-    
-    // init start job
-    BPending_Init(&enc->start_job, pg, (BPending_handler)job_handler, enc);
-    BPending_Set(&enc->start_job);
-    
-    // init debug object
     DebugObject_Init(&enc->d_obj);
     
     return 1;
@@ -282,12 +152,8 @@ fail0:
 
 void PacketProtoDecoder_Free (PacketProtoDecoder *enc)
 {
-    // free debug object
     DebugObject_Free(&enc->d_obj);
     
-    // free start job
-    BPending_Free(&enc->start_job);
-    
     // free buffer
     free(enc->buf);
     
@@ -297,6 +163,8 @@ void PacketProtoDecoder_Free (PacketProtoDecoder *enc)
 
 void PacketProtoDecoder_Reset (PacketProtoDecoder *enc)
 {
+    DebugObject_Access(&enc->d_obj);
+    
     enc->buf_start += enc->buf_used;
     enc->buf_used = 0;
 }

flow/PacketProtoDecoder.h

@@ -29,15 +29,14 @@
 
 #include <stdint.h>
 
+#include <protocol/packetproto.h>
 #include <misc/debug.h>
 #include <misc/dead.h>
 #include <misc/balign.h>
+#include <system/DebugObject.h>
 #include <flow/StreamRecvInterface.h>
 #include <flow/PacketPassInterface.h>
 #include <flow/error.h>
-#include <protocol/packetproto.h>
-#include <system/DebugObject.h>
-#include <system/BPending.h>
 
 #define PACKETPROTODECODER_ERROR_TOOLONG 1
 
@@ -54,7 +53,6 @@
  *       a packet length value which is too big,
  */
 typedef struct {
-    DebugObject d_obj;
     dead_t dead;
     FlowErrorReporter rep;
     StreamRecvInterface *input;
@@ -64,9 +62,7 @@ typedef struct {
     int buf_start;
     int buf_used;
     uint8_t *buf;
-    int receiving;
-    int sending;
-    BPending start_job;
+    DebugObject d_obj;
 } PacketProtoDecoder;
 
 /**

flow/PacketProtoEncoder.c

@@ -30,7 +30,7 @@
 #include <flow/PacketProtoEncoder.h>
 
 static int encode_packet (PacketProtoEncoder *enc, uint8_t *data, int in_len);
-static int output_handler_recv (PacketProtoEncoder *enc, uint8_t *data, int *out_len);
+static void output_handler_recv (PacketProtoEncoder *enc, uint8_t *data);
 static void input_handler_done (PacketProtoEncoder *enc, int in_len);
 
 int encode_packet (PacketProtoEncoder *enc, uint8_t *data, int in_len)
@@ -42,58 +42,37 @@ int encode_packet (PacketProtoEncoder *enc, uint8_t *data, int in_len)
     return PACKETPROTO_ENCLEN(in_len);
 }
 
-int output_handler_recv (PacketProtoEncoder *enc, uint8_t *data, int *out_len)
+void output_handler_recv (PacketProtoEncoder *enc, uint8_t *data)
 {
     ASSERT(!enc->output_packet)
     ASSERT(data)
+    DebugObject_Access(&enc->d_obj);
     
-    // call recv on input
-    int in_len;
-    DEAD_ENTER(enc->dead)
-    int res = PacketRecvInterface_Receiver_Recv(enc->input, data + sizeof(struct packetproto_header), &in_len);
-    if (DEAD_LEAVE(enc->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    if (!res) {
-        // input busy, continue in input_handler_done
-        enc->output_packet = data;
-        return 0;
-    }
-    
-    // encode
-    *out_len = encode_packet(enc, data, in_len);
-    
-    return 1;
+    // schedule receive
+    PacketRecvInterface_Receiver_Recv(enc->input, data + sizeof(struct packetproto_header));
+    enc->output_packet = data;
 }
 
 void input_handler_done (PacketProtoEncoder *enc, int in_len)
 {
     ASSERT(enc->output_packet)
+    DebugObject_Access(&enc->d_obj);
     
     // encode
     int out_len = encode_packet(enc, enc->output_packet, in_len);
     
-    // set no output packet
+    // finish output packet
     enc->output_packet = NULL;
-    
-    // notify output
     PacketRecvInterface_Done(&enc->output, out_len);
-    return;
 }
 
-void PacketProtoEncoder_Init (PacketProtoEncoder *enc, PacketRecvInterface *input)
+void PacketProtoEncoder_Init (PacketProtoEncoder *enc, PacketRecvInterface *input, BPendingGroup *pg)
 {
     ASSERT(PacketRecvInterface_GetMTU(input) <= PACKETPROTO_MAXPAYLOAD)
     
     // init arguments
     enc->input = input;
     
-    // init dead var
-    DEAD_INIT(enc->dead);
-    
     // init input
     PacketRecvInterface_Receiver_Init(enc->input, (PacketRecvInterface_handler_done)input_handler_done, enc);
     
@@ -102,29 +81,27 @@ void PacketProtoEncoder_Init (PacketProtoEncoder *enc, PacketRecvInterface *inpu
         &enc->output,
         PACKETPROTO_ENCLEN(PacketRecvInterface_GetMTU(enc->input)),
         (PacketRecvInterface_handler_recv)output_handler_recv,
-        enc
+        enc,
+        pg
     );
     
     // set no output packet
     enc->output_packet = NULL;
     
-    // init debug object
     DebugObject_Init(&enc->d_obj);
 }
 
 void PacketProtoEncoder_Free (PacketProtoEncoder *enc)
 {
-    // free debug object
     DebugObject_Free(&enc->d_obj);
 
     // free input
     PacketRecvInterface_Free(&enc->output);
-    
-    // free dead var
-    DEAD_KILL(enc->dead);
 }
 
 PacketRecvInterface * PacketProtoEncoder_GetOutput (PacketProtoEncoder *enc)
 {
+    DebugObject_Access(&enc->d_obj);
+    
     return &enc->output;
 }

flow/PacketProtoEncoder.h

@@ -29,7 +29,6 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <system/DebugObject.h>
 #include <flow/PacketRecvInterface.h>
 
@@ -40,11 +39,10 @@
  * Output is with {@link PacketRecvInterface}.
  */
 typedef struct {
-    DebugObject d_obj;
-    dead_t dead;
     PacketRecvInterface *input;
     PacketRecvInterface output;
     uint8_t *output_packet;
+    DebugObject d_obj;
 } PacketProtoEncoder;
 
 /**
@@ -52,8 +50,9 @@ typedef struct {
  *
  * @param enc the object
  * @param input input interface. Its MTU must be <=PACKETPROTO_MAXPAYLOAD.
+ * @param pg pending group
  */
-void PacketProtoEncoder_Init (PacketProtoEncoder *enc, PacketRecvInterface *input);
+void PacketProtoEncoder_Init (PacketProtoEncoder *enc, PacketRecvInterface *input, BPendingGroup *pg);
 
 /**
  * Frees the object.

flow/PacketProtoFlow.c

@@ -33,24 +33,23 @@ int PacketProtoFlow_Init (PacketProtoFlow *o, int input_mtu, int num_packets, Pa
     ASSERT(PacketPassInterface_GetMTU(output) >= PACKETPROTO_ENCLEN(input_mtu))
     
     // init async input
-    PacketBufferAsyncInput_Init(&o->ainput, input_mtu);
+    BufferWriter_Init(&o->ainput, input_mtu, pg);
     
     // init encoder
-    PacketProtoEncoder_Init(&o->encoder, PacketBufferAsyncInput_GetOutput(&o->ainput));
+    PacketProtoEncoder_Init(&o->encoder, BufferWriter_GetOutput(&o->ainput), pg);
     
     // init buffer
     if (!PacketBuffer_Init(&o->buffer, PacketProtoEncoder_GetOutput(&o->encoder), output, num_packets, pg)) {
         goto fail0;
     }
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
     
     return 1;
     
 fail0:
     PacketProtoEncoder_Free(&o->encoder);
-    PacketBufferAsyncInput_Free(&o->ainput);
+    BufferWriter_Free(&o->ainput);
     return 0;
 }
 
@@ -65,12 +64,12 @@ void PacketProtoFlow_Free (PacketProtoFlow *o)
     PacketProtoEncoder_Free(&o->encoder);
     
     // free async input
-    PacketBufferAsyncInput_Free(&o->ainput);
+    BufferWriter_Free(&o->ainput);
 }
 
-BestEffortPacketWriteInterface * PacketProtoFlow_GetInput (PacketProtoFlow *o)
+BufferWriter * PacketProtoFlow_GetInput (PacketProtoFlow *o)
 {
     DebugObject_Access(&o->d_obj);
     
-    return PacketBufferAsyncInput_GetInput(&o->ainput);
+    return &o->ainput;
 }

flow/PacketProtoFlow.h

@@ -21,7 +21,7 @@
  * 
  * @section DESCRIPTION
  * 
- * Buffer which encodes packets with PacketProto, with {@link BestEffortPacketWriteInterface}
+ * Buffer which encodes packets with PacketProto, with {@link BufferWriter}
  * input and {@link PacketPassInterface} output.
  */
 
@@ -29,17 +29,16 @@
 #define BADVPN_FLOW_PACKETPROTOFLOW_H
 
 #include <system/DebugObject.h>
-#include <system/BPending.h>
-#include <flow/PacketBufferAsyncInput.h>
+#include <flow/BufferWriter.h>
 #include <flow/PacketProtoEncoder.h>
 #include <flow/PacketBuffer.h>
 
 /**
- * Buffer which encodes packets with PacketProto, with {@link BestEffortPacketWriteInterface}
+ * Buffer which encodes packets with PacketProto, with {@link BufferWriter}
  * input and {@link PacketPassInterface} output.
  */
 typedef struct {
-    PacketBufferAsyncInput ainput;
+    BufferWriter ainput;
     PacketProtoEncoder encoder;
     PacketBuffer buffer;
     DebugObject d_obj;
@@ -66,13 +65,10 @@ void PacketProtoFlow_Free (PacketProtoFlow *o);
 
 /**
  * Returns the input interface.
- * Its MTU will be as in {@link PacketProtoFlow_Init}.
- * Be aware that it may not be possible to write packets immediately after initializing
- * the object; the object starts its internal I/O with a {@link BPending} job.
  * 
  * @param o the object
  * @return input interface
  */
-BestEffortPacketWriteInterface * PacketProtoFlow_GetInput (PacketProtoFlow *o);
+BufferWriter * PacketProtoFlow_GetInput (PacketProtoFlow *o);
 
 #endif

flow/PacketRecvBlocker.c

@@ -24,41 +24,35 @@
 
 #include <flow/PacketRecvBlocker.h>
 
-static int output_handler_recv (PacketRecvBlocker *o, uint8_t *data, int *data_len)
+static void output_handler_recv (PacketRecvBlocker *o, uint8_t *data)
 {
     ASSERT(!o->out_have)
+    DebugObject_Access(&o->d_obj);
     
     // remember packet
     o->out_have = 1;
     o->out = data;
     o->out_input_blocking = 0;
-    
-    return 0;
 }
 
 static void input_handler_done (PacketRecvBlocker *o, int data_len)
 {
     ASSERT(o->out_have)
     ASSERT(o->out_input_blocking)
+    DebugObject_Access(&o->d_obj);
     
     // have no output packet
     o->out_have = 0;
-    
-    // inform output we received something
     PacketRecvInterface_Done(&o->output, data_len);
-    return;
 }
 
-void PacketRecvBlocker_Init (PacketRecvBlocker *o, PacketRecvInterface *input)
+void PacketRecvBlocker_Init (PacketRecvBlocker *o, PacketRecvInterface *input, BPendingGroup *pg)
 {
     // init arguments
     o->input = input;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init output
-    PacketRecvInterface_Init(&o->output, PacketRecvInterface_GetMTU(o->input), (PacketRecvInterface_handler_recv)output_handler_recv, o);
+    PacketRecvInterface_Init(&o->output, PacketRecvInterface_GetMTU(o->input), (PacketRecvInterface_handler_recv)output_handler_recv, o, pg);
     
     // have no output packet
     o->out_have = 0;
@@ -66,55 +60,33 @@ void PacketRecvBlocker_Init (PacketRecvBlocker *o, PacketRecvInterface *input)
     // init input
     PacketRecvInterface_Receiver_Init(o->input, (PacketRecvInterface_handler_done)input_handler_done, o);
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
 void PacketRecvBlocker_Free (PacketRecvBlocker *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // free output
     PacketRecvInterface_Free(&o->output);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }
 
 PacketRecvInterface * PacketRecvBlocker_GetOutput (PacketRecvBlocker *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->output;
 }
 
 void PacketRecvBlocker_AllowBlockedPacket (PacketRecvBlocker *o)
 {
-    ASSERT(!PacketRecvInterface_InClient(o->input))
+    DebugObject_Access(&o->d_obj);
     
     if (!o->out_have || o->out_input_blocking) {
         return;
     }
     
-    // receive from input
-    int in_len;
-    DEAD_ENTER(o->dead)
-    int res = PacketRecvInterface_Receiver_Recv(o->input, o->out, &in_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    if (!res) {
-        // input blocking, continue in input_handler_done
-        o->out_input_blocking = 1;
-        return;
-    }
-    
-    // have no output packet
-    o->out_have = 0;
-    
-    // inform output we received something
-    PacketRecvInterface_Done(&o->output, in_len);
-    return;
+    // schedule receive
+    PacketRecvInterface_Receiver_Recv(o->input, o->out);
+    o->out_input_blocking = 1;
 }

flow/PacketRecvBlocker.h

@@ -30,7 +30,6 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <system/DebugObject.h>
 #include <flow/PacketRecvInterface.h>
 
@@ -39,13 +38,12 @@
  * passes a single blocked call on to input when the user wants so.
  */
 typedef struct {
-    DebugObject d_obj;
-    dead_t dead;
     PacketRecvInterface output;
     int out_have;
     uint8_t *out;
     int out_input_blocking;
     PacketRecvInterface *input;
+    DebugObject d_obj;
 } PacketRecvBlocker;
 
 /**
@@ -54,7 +52,7 @@ typedef struct {
  * @param o the object
  * @param input input interface
  */
-void PacketRecvBlocker_Init (PacketRecvBlocker *o, PacketRecvInterface *input);
+void PacketRecvBlocker_Init (PacketRecvBlocker *o, PacketRecvInterface *input, BPendingGroup *pg);
 
 /**
  * Frees the object.

flow/PacketRecvConnector.c

@@ -26,57 +26,23 @@
 
 #include <flow/PacketRecvConnector.h>
 
-static int output_handler_recv (PacketRecvConnector *o, uint8_t *data, int *data_len)
+static void output_handler_recv (PacketRecvConnector *o, uint8_t *data)
 {
     ASSERT(!o->out_have)
-    ASSERT(!o->input || !o->in_blocking)
-    
-    // if we have no input, remember output packet
-    if (!o->input) {
-        o->out_have = 1;
-        o->out = data;
-        return 0;
+    if (o->input) {
+        ASSERT(!o->in_blocking)
     }
+    DebugObject_Access(&o->d_obj);
     
-    // try to receive the packet
-    int res;
-    while (1) {
-        DEAD_ENTER_N(obj, o->dead)
-        DEAD_ENTER_N(inp, o->input_dead)
-        res = PacketRecvInterface_Receiver_Recv(o->input, data, data_len);
-        DEAD_LEAVE_N(obj, o->dead);
-        DEAD_LEAVE_N(inp, o->input_dead);
-        if (DEAD_KILLED_N(obj)) {
-            return -1;
-        }
-        if (DEAD_KILLED_N(inp)) {
-            if (!o->input) {
-                // lost input
-                o->out_have = 1;
-                o->out = data;
-                return 0;
-            }
-            // got a new input, retry
-            continue;
-        }
-        break;
-    };
-    
-    ASSERT(res == 0 || res == 1)
-    if (res) {
-        ASSERT(*data_len >= 0)
-        ASSERT(*data_len <= o->output_mtu)
-    }
+    // remember output packet
+    o->out_have = 1;
+    o->out = data;
     
-    if (!res) {
-        // input blocking
-        o->out_have = 1;
-        o->out = data;
+    if (o->input) {
+        // schedule receive
+        PacketRecvInterface_Receiver_Recv(o->input, o->out);
         o->in_blocking = 1;
-        return 0;
     }
-    
-    return 1;
 }
 
 static void input_handler_done (PacketRecvConnector *o, int data_len)
@@ -84,58 +50,16 @@ static void input_handler_done (PacketRecvConnector *o, int data_len)
     ASSERT(o->out_have)
     ASSERT(o->input)
     ASSERT(o->in_blocking)
-    
-    // have no output packet
-    o->out_have = 0;
+    DebugObject_Access(&o->d_obj);
     
     // input not blocking any more
     o->in_blocking = 0;
     
-    // allow output to receive more packets
-    PacketRecvInterface_Done(&o->output, data_len);
-    return;
-}
-
-static void job_handler (PacketRecvConnector *o)
-{
-    ASSERT(o->input)
-    ASSERT(!o->in_blocking)
-    ASSERT(o->out_have)
-    
-    // try to receive the packet
-    int in_len;
-    DEAD_ENTER_N(obj, o->dead)
-    DEAD_ENTER_N(inp, o->input_dead)
-    int res = PacketRecvInterface_Receiver_Recv(o->input, o->out, &in_len);
-    DEAD_LEAVE_N(obj, o->dead);
-    DEAD_LEAVE_N(inp, o->input_dead);
-    if (DEAD_KILLED_N(obj)) {
-        return;
-    }
-    if (DEAD_KILLED_N(inp)) {
-        // lost current input. Do nothing here.
-        // If we gained a new one, its own job is responsible for it.
-        return;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    if (res) {
-        ASSERT(in_len >= 0)
-        ASSERT(in_len <= o->output_mtu)
-    }
-    
-    if (!res) {
-        // input blocking
-        o->in_blocking = 1;
-        return;
-    }
-    
     // have no output packet
     o->out_have = 0;
     
     // allow output to receive more packets
-    PacketRecvInterface_Done(&o->output, in_len);
-    return;
+    PacketRecvInterface_Done(&o->output, data_len);
 }
 
 void PacketRecvConnector_Init (PacketRecvConnector *o, int mtu, BPendingGroup *pg)
@@ -145,11 +69,8 @@ void PacketRecvConnector_Init (PacketRecvConnector *o, int mtu, BPendingGroup *p
     // init arguments
     o->output_mtu = mtu;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init output
-    PacketRecvInterface_Init(&o->output, o->output_mtu, (PacketRecvInterface_handler_recv)output_handler_recv, o);
+    PacketRecvInterface_Init(&o->output, o->output_mtu, (PacketRecvInterface_handler_recv)output_handler_recv, o, pg);
     
     // have no output packet
     o->out_have = 0;
@@ -157,35 +78,21 @@ void PacketRecvConnector_Init (PacketRecvConnector *o, int mtu, BPendingGroup *p
     // have no input
     o->input = NULL;
     
-    // init continue job
-    BPending_Init(&o->continue_job, pg, (BPending_handler)job_handler, o);
-    
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
 void PacketRecvConnector_Free (PacketRecvConnector *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
     
-    // free continue job
-    BPending_Free(&o->continue_job);
-    
-    // free input dead var
-    if (o->input) {
-        DEAD_KILL(o->input_dead);
-    }
-    
     // free output
     PacketRecvInterface_Free(&o->output);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }
 
 PacketRecvInterface * PacketRecvConnector_GetOutput (PacketRecvConnector *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->output;
 }
 
@@ -193,6 +100,7 @@ void PacketRecvConnector_ConnectInput (PacketRecvConnector *o, PacketRecvInterfa
 {
     ASSERT(!o->input)
     ASSERT(PacketRecvInterface_GetMTU(input) <= o->output_mtu)
+    DebugObject_Access(&o->d_obj);
     
     // set input
     o->input = input;
@@ -200,27 +108,20 @@ void PacketRecvConnector_ConnectInput (PacketRecvConnector *o, PacketRecvInterfa
     // init input
     PacketRecvInterface_Receiver_Init(o->input, (PacketRecvInterface_handler_done)input_handler_done, o);
     
-    // init input dead var
-    DEAD_INIT(o->input_dead);
-    
     // set input not blocking
     o->in_blocking = 0;
     
-    // if we have an input packet, set continue job
+    // if we have an output packet, schedule receive
     if (o->out_have) {
-        BPending_Set(&o->continue_job);
+        PacketRecvInterface_Receiver_Recv(o->input, o->out);
+        o->in_blocking = 1;
     }
 }
 
 void PacketRecvConnector_DisconnectInput (PacketRecvConnector *o)
 {
     ASSERT(o->input)
-    
-    // unset continue job (in case it wasn't called yet)
-    BPending_Unset(&o->continue_job);
-    
-    // free dead var
-    DEAD_KILL(o->input_dead);
+    DebugObject_Access(&o->d_obj);
     
     // set no input
     o->input = NULL;

flow/PacketRecvConnector.h

@@ -30,9 +30,7 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <system/DebugObject.h>
-#include <system/BPending.h>
 #include <flow/PacketRecvInterface.h>
 
 /**
@@ -40,16 +38,13 @@
  * connected and disconnected on the fly.
  */
 typedef struct {
-    DebugObject d_obj;
-    dead_t dead;
     PacketRecvInterface output;
     int output_mtu;
     int out_have;
     uint8_t *out;
     PacketRecvInterface *input;
-    dead_t input_dead;
     int in_blocking;
-    BPending continue_job;
+    DebugObject d_obj;
 } PacketRecvConnector;
 
 /**

flow/PacketRecvInterface.c

@@ -0,0 +1,77 @@
+/**
+ * @file PacketRecvInterface.c
+ * @author Ambroz Bizjak <[email protected]>
+ * 
+ * @section LICENSE
+ * 
+ * This file is part of BadVPN.
+ * 
+ * BadVPN is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ * 
+ * BadVPN is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <flow/PacketRecvInterface.h>
+
+void _PacketRecvInterface_job_operation (PacketRecvInterface *i)
+{
+    ASSERT(i->d_user_busy)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_done);
+    #endif
+    
+    // call operation handler
+    #ifndef NDEBUG
+    DebugIn_GoIn(&i->d_in_operation);
+    DEAD_ENTER(i->d_dead)
+    #endif
+    i->handler_operation(i->user_provider, i->buf);
+    #ifndef NDEBUG
+    if (DEAD_LEAVE(i->d_dead)) {
+        return;
+    }
+    DebugIn_GoOut(&i->d_in_operation);
+    #endif
+}
+
+void _PacketRecvInterface_job_done (PacketRecvInterface *i)
+{
+    ASSERT(i->d_user_busy)
+    ASSERT(i->done_len >= 0)
+    ASSERT(i->done_len <= i->mtu)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_done);
+    #endif
+    
+    #ifndef NDEBUG
+    i->d_user_busy = 0;
+    #endif
+    
+    // call done handler
+    #ifndef NDEBUG
+    DebugIn_GoIn(&i->d_in_done);
+    DEAD_ENTER(i->d_dead)
+    #endif
+    i->handler_done(i->user_user, i->done_len);
+    #ifndef NDEBUG
+    if (DEAD_LEAVE(i->d_dead)) {
+        return;
+    }
+    DebugIn_GoOut(&i->d_in_done);
+    #endif
+}

flow/PacketRecvInterface.h

@@ -32,263 +32,170 @@
 
 #include <misc/dead.h>
 #include <misc/debug.h>
+#include <misc/debugin.h>
 #include <system/DebugObject.h>
+#include <system/BPending.h>
 
-/**
- * Handler called at the sender when {@link PacketRecvInterface_Receiver_Recv} is called
- * from the receiver.
- * It is guaranteed that the interface is in not receiving state.
- * It is guaranteed that the handler is not being called from within Recv or Cancel handlers.
- *
- * @param user value supplied to {@link PacketRecvInterface_Init}
- * @param data pointer to the buffer where the packet is to be written. Will have space
- *             for MTU bytes. May be NULL if MTU is 0.
- * @param data_len if the packet was written immediately, must be set to its length
- * @return - 1 if the sender provides a packet immediately. The interface remains in
- *           not receiving state. The sender may not use the provided buffer after the handler
- *           returns.
- *         - 0 if the sender cannot provide a packet immediately. The interface enters
- *           receiving state as the handler returns. The sender must write a packet to the
- *           provided buffer and call {@link PacketRecvInterface_Done} when it's done.
- */
-typedef int (*PacketRecvInterface_handler_recv) (void *user, uint8_t *data, int *data_len);
+typedef void (*PacketRecvInterface_handler_recv) (void *user, uint8_t *data);
 
-/**
- * Handler called at the receiver when {@link PacketRecvInterface_Done} is called from the sender.
- * The sender will no longer use the buffer it was provided with.
- * It is guaranteed that the interface was in receiving state.
- * The interface enters not receiving state before the handler is called.
- * It is guaranteed that the handler is not being called from within Recv, Cancel or Done handlers.
- *
- * @param user value supplied to {@link PacketRecvInterface_Receiver_Init}
- * @param data_len size of the packet that was written to the buffer. Will be >=0 and <=MTU.
- */
 typedef void (*PacketRecvInterface_handler_done) (void *user, int data_len);
 
-/**
- * Interface allowing a packet receiver to receive data packets from a packet sender.
- * The receiver receives a packet by providing the sender with a buffer. The sender
- * may then either provide the packet immediately, or tell the receiver to wait for
- * the packet to be available and inform it when it's done.
- */
 typedef struct {
-    DebugObject d_obj;
-    
-    // sender data
+    // provider data
     int mtu;
-    PacketRecvInterface_handler_recv handler_recv;
-    void *user_sender;
+    PacketRecvInterface_handler_recv handler_operation;
+    void *user_provider;
 
-    // receiver data
+    // user data
     PacketRecvInterface_handler_done handler_done;
-    void *user_receiver;
+    void *user_user;
+    
+    // jobs
+    BPending job_operation;
+    BPending job_done;
     
-    // debug vars
+    // buffer supplied by user
+    uint8_t *buf;
+    
+    // length supplied by done
+    int done_len;
+    
+    DebugObject d_obj;
     #ifndef NDEBUG
-    dead_t debug_dead;
-    int debug_busy;
-    int debug_in_recv;
-    int debug_in_done;
+    DebugIn d_in_operation;
+    DebugIn d_in_done;
+    dead_t d_dead;
+    int d_user_busy;
     #endif
 } PacketRecvInterface;
 
-/**
- * Initializes the interface. The receiver portion must also be initialized
- * with {@link PacketRecvInterface_Receiver_Init} before I/O can start.
- * The interface is initialized in not receiving state.
- *
- * @param i the object
- * @param mtu maximum packet size the sender can provide. Must be >=0.
- * @param handler_recv handler called when the receiver wants to receive a packet
- * @param user arbitrary value that will be passed to sender callback functions
- */
-static void PacketRecvInterface_Init (PacketRecvInterface *i, int mtu, PacketRecvInterface_handler_recv handler_recv, void *user);
+static void PacketRecvInterface_Init (PacketRecvInterface *i, int mtu, PacketRecvInterface_handler_recv handler_operation, void *user, BPendingGroup *pg);
 
-/**
- * Frees the interface.
- *
- * @param i the object
- */
 static void PacketRecvInterface_Free (PacketRecvInterface *i);
 
-/**
- * Notifies the receiver that the sender has finished providing the packet being received.
- * The sender must not use the buffer it was provided any more.
- * The interface must be in receiving state.
- * The interface enters not receiving state before notifying the receiver.
- * Must not be called from within Recv, Cancel or Done handlers.
- *
- * Be aware that the receiver may attempt to receive packets from within this function.
- *
- * @param i the object
- * @param data_len size of the packet written to the buffer. Must be >=0 and <=MTU.
- */
 static void PacketRecvInterface_Done (PacketRecvInterface *i, int data_len);
 
-/**
- * Returns the maximum packet size the sender can provide.
- *
- * @return maximum packet size. Will be >=0.
- */
 static int PacketRecvInterface_GetMTU (PacketRecvInterface *i);
 
-/**
- * Initializes the receiver portion of the interface.
- *
- * @param i the object
- * @param handler_done handler called when the sender has finished providing a packet
- * @param user arbitrary value that will be passed to receiver callback functions
- */
 static void PacketRecvInterface_Receiver_Init (PacketRecvInterface *i, PacketRecvInterface_handler_done handler_done, void *user);
 
-/**
- * Attempts to receive a packet.
- * The interface must be in not receiving state.
- * Must not be called from within Recv or Cancel handlers.
- *
- * @param i the object
- * @param data pointer to the buffer where the packet is to be written. Must have space
- *             for MTU bytes. Ignored if MTU is 0.
- * @param data_len will contain the size of the packet if it was provided immediately
- * @return - 1 if a packet was provided by the sender immediately. The buffer is no longer needed.
- *           The interface remains in not receiving state.
- *         - 0 if a packet could not be provided immediately.
- *           The interface enters receiving state, and the buffer must stay accessible while the
- *           sender is providing the packet. When the sender is done providing it, the
- *           {@link PacketRecvInterface_handler_done} handler will be called.
- */
-static int PacketRecvInterface_Receiver_Recv (PacketRecvInterface *i, uint8_t *data, int *data_len);
+static void PacketRecvInterface_Receiver_Recv (PacketRecvInterface *i, uint8_t *data);
 
 #ifndef NDEBUG
 
-/**
- * Determines if we are in a Recv call.
- * Only available if NDEBUG is not defined.
- * 
- * @param i the object
- * @return 1 if in a Recv call, 0 if not
- */
 static int PacketRecvInterface_InClient (PacketRecvInterface *i);
 
-/**
- * Determines if we are in a Done call.
- * Only available if NDEBUG is not defined.
- * 
- * @param i the object
- * @return 1 if in a Done call, 0 if not
- */
 static int PacketRecvInterface_InDone (PacketRecvInterface *i);
 
 #endif
 
-void PacketRecvInterface_Init (PacketRecvInterface *i, int mtu, PacketRecvInterface_handler_recv handler_recv, void *user)
+void _PacketRecvInterface_job_operation (PacketRecvInterface *i);
+void _PacketRecvInterface_job_done (PacketRecvInterface *i);
+
+void PacketRecvInterface_Init (PacketRecvInterface *i, int mtu, PacketRecvInterface_handler_recv handler_operation, void *user, BPendingGroup *pg)
 {
     ASSERT(mtu >= 0)
     
+    // init arguments
     i->mtu = mtu;
-    i->handler_recv = handler_recv;
-    i->user_sender = user;
+    i->handler_operation = handler_operation;
+    i->user_provider = user;
+    
+    // set no user
     i->handler_done = NULL;
-    i->user_receiver = NULL;
     
-    // init debugging
-    #ifndef NDEBUG
-    DEAD_INIT(i->debug_dead);
-    i->debug_busy = 0;
-    i->debug_in_recv = 0;
-    i->debug_in_done = 0;
-    #endif
+    // init jobs
+    BPending_Init(&i->job_operation, pg, (BPending_handler)_PacketRecvInterface_job_operation, i);
+    BPending_Init(&i->job_done, pg, (BPending_handler)_PacketRecvInterface_job_done, i);
     
-    // init debug object
     DebugObject_Init(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_Init(&i->d_in_operation);
+    DebugIn_Init(&i->d_in_done);
+    DEAD_INIT(i->d_dead);
+    i->d_user_busy = 0;
+    #endif
 }
 
 void PacketRecvInterface_Free (PacketRecvInterface *i)
 {
-    // free debug object
-    DebugObject_Free(&i->d_obj);
-    
-    // free debugging
     #ifndef NDEBUG
-    DEAD_KILL(i->debug_dead);
+    DEAD_KILL(i->d_dead);
     #endif
+    DebugObject_Free(&i->d_obj);
+    
+    // free jobs
+    BPending_Free(&i->job_done);
+    BPending_Free(&i->job_operation);
 }
 
 void PacketRecvInterface_Done (PacketRecvInterface *i, int data_len)
 {
-    ASSERT(i->debug_busy)
-    ASSERT(!i->debug_in_recv)
-    ASSERT(!i->debug_in_done)
     ASSERT(data_len >= 0)
     ASSERT(data_len <= i->mtu)
-    
+    ASSERT(i->d_user_busy)
+    ASSERT(i->handler_done)
+    ASSERT(!BPending_IsSet(&i->job_operation))
+    DebugObject_Access(&i->d_obj);
     #ifndef NDEBUG
-    i->debug_busy = 0;
-    i->debug_in_done = 1;
-    DEAD_ENTER(i->debug_dead)
+    DebugIn_AmOut(&i->d_in_done);
     #endif
     
-    i->handler_done(i->user_receiver, data_len);
+    i->done_len = data_len;
     
-    #ifndef NDEBUG
-    if (DEAD_LEAVE(i->debug_dead)) {
-        return;
-    }
-    i->debug_in_done = 0;
-    #endif
+    BPending_Set(&i->job_done);
 }
 
 int PacketRecvInterface_GetMTU (PacketRecvInterface *i)
 {
+    DebugObject_Access(&i->d_obj);
+    
     return i->mtu;
 }
 
 void PacketRecvInterface_Receiver_Init (PacketRecvInterface *i, PacketRecvInterface_handler_done handler_done, void *user)
 {
+    ASSERT(handler_done)
+    ASSERT(!i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    
     i->handler_done = handler_done;
-    i->user_receiver = user;
+    i->user_user = user;
 }
 
-int PacketRecvInterface_Receiver_Recv (PacketRecvInterface *i, uint8_t *data, int *data_len)
+void PacketRecvInterface_Receiver_Recv (PacketRecvInterface *i, uint8_t *data)
 {
-    ASSERT(!i->debug_busy)
-    ASSERT(!i->debug_in_recv)
     ASSERT(!(i->mtu > 0) || data)
-    ASSERT(data_len)
-    
+    ASSERT(!i->d_user_busy)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
     #ifndef NDEBUG
-    i->debug_in_recv = 1;
-    DEAD_ENTER(i->debug_dead)
+    DebugIn_AmOut(&i->d_in_operation);
     #endif
     
-    int res = i->handler_recv(i->user_sender, data, data_len);
+    BPending_Set(&i->job_operation);
+    
+    i->buf = data;
     
     #ifndef NDEBUG
-    if (DEAD_LEAVE(i->debug_dead)) {
-        return -1;
-    }
-    ASSERT(i->debug_in_recv)
-    i->debug_in_recv = 0;
-    ASSERT(res == 0 || res == 1)
-    ASSERT(!(res == 1) || (*data_len >= 0 && *data_len <= i->mtu))
-    if (!res) {
-        i->debug_busy = 1;
-    }
+    i->d_user_busy = 1;
     #endif
-    
-    return res;
 }
 
 #ifndef NDEBUG
 
 int PacketRecvInterface_InClient (PacketRecvInterface *i)
 {
-    return i->debug_in_recv;
+    DebugObject_Access(&i->d_obj);
+    
+    return DebugIn_In(&i->d_in_operation);
 }
 
 int PacketRecvInterface_InDone (PacketRecvInterface *i)
 {
-    return i->debug_in_done;
+    DebugObject_Access(&i->d_obj);
+    
+    return DebugIn_In(&i->d_in_done);
 }
 
 #endif

flow/PacketRecvNotifier.c

@@ -25,74 +25,43 @@
 #include <flow/PacketRecvNotifier.h>
 
 static int call_handler (PacketRecvNotifier *o, uint8_t *data, int data_len);
-static int output_handler_recv (PacketRecvNotifier *o, uint8_t *data, int *data_len);
+static void output_handler_recv (PacketRecvNotifier *o, uint8_t *data);
 static void input_handler_done (PacketRecvNotifier *o, int data_len);
 
 int call_handler (PacketRecvNotifier *o, uint8_t *data, int data_len)
 {
     ASSERT(o->handler)
-    ASSERT(!o->in_handler)
-    
-    #ifndef NDEBUG
-    o->in_handler = 1;
-    #endif
+    DebugIn_AmOut(&o->d_in_handler);
+    DebugObject_Access(&o->d_obj);
     
+    DebugIn_GoIn(&o->d_in_handler);
     DEAD_ENTER(o->dead)
     o->handler(o->handler_user, data, data_len);
     if (DEAD_LEAVE(o->dead)) {
         return -1;
     }
-    
-    #ifndef NDEBUG
-    o->in_handler = 0;
-    #endif
+    DebugIn_GoOut(&o->d_in_handler);
     
     return 0;
 }
 
-int output_handler_recv (PacketRecvNotifier *o, uint8_t *data, int *data_len)
+void output_handler_recv (PacketRecvNotifier *o, uint8_t *data)
 {
-    ASSERT(!o->out_have)
-    ASSERT(!o->in_handler)
-    
-    DEAD_DECLARE
-    
-    // call recv on input
-    DEAD_ENTER2(o->dead)
-    int res = PacketRecvInterface_Receiver_Recv(o->input, data, data_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    if (!res) {
-        // input blocking, continue in input_handler_done
-        #ifndef NDEBUG
-        o->out_have = 1;
-        #endif
-        o->out = data;
-        return 0;
-    }
+    DebugIn_AmOut(&o->d_in_handler);
+    DebugObject_Access(&o->d_obj);
     
-    // if we have a handler, call it
-    if (o->handler) {
-        if (call_handler(o, data, *data_len) < 0) {
-            return -1;
-        }
-    }
-    
-    return 1;
+    // schedule receive
+    o->out = data;
+    PacketRecvInterface_Receiver_Recv(o->input, o->out);
 }
 
 void input_handler_done (PacketRecvNotifier *o, int data_len)
 {
-    ASSERT(o->out_have)
-    ASSERT(!o->in_handler)
+    DebugIn_AmOut(&o->d_in_handler);
+    DebugObject_Access(&o->d_obj);
     
-    #ifndef NDEBUG
-    o->out_have = 0;
-    #endif
+    // finish packet
+    PacketRecvInterface_Done(&o->output, data_len);
     
     // if we have a handler, call it
     if (o->handler) {
@@ -100,12 +69,9 @@ void input_handler_done (PacketRecvNotifier *o, int data_len)
             return;
         }
     }
-    
-    PacketRecvInterface_Done(&o->output, data_len);
-    return;
 }
 
-void PacketRecvNotifier_Init (PacketRecvNotifier *o, PacketRecvInterface *input)
+void PacketRecvNotifier_Init (PacketRecvNotifier *o, PacketRecvInterface *input, BPendingGroup *pg)
 {
     // set arguments
     o->input = input;
@@ -114,7 +80,7 @@ void PacketRecvNotifier_Init (PacketRecvNotifier *o, PacketRecvInterface *input)
     DEAD_INIT(o->dead);
     
     // init output
-    PacketRecvInterface_Init(&o->output, PacketRecvInterface_GetMTU(o->input), (PacketRecvInterface_handler_recv)output_handler_recv, o);
+    PacketRecvInterface_Init(&o->output, PacketRecvInterface_GetMTU(o->input), (PacketRecvInterface_handler_recv)output_handler_recv, o, pg);
     
     // init input
     PacketRecvInterface_Receiver_Init(o->input, (PacketRecvInterface_handler_done)input_handler_done, o);
@@ -122,21 +88,14 @@ void PacketRecvNotifier_Init (PacketRecvNotifier *o, PacketRecvInterface *input)
     // set no handler
     o->handler = NULL;
     
-    // init debugging
-    #ifndef NDEBUG
-    o->out_have = 0;
-    o->in_handler = 0;
-    #endif
-    
-    // init debug object
     DebugObject_Init(&o->d_obj);
+    DebugIn_Init(&o->d_in_handler);
 }
 
 void PacketRecvNotifier_Free (PacketRecvNotifier *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
-
+    
     // free output
     PacketRecvInterface_Free(&o->output);
     
@@ -146,11 +105,15 @@ void PacketRecvNotifier_Free (PacketRecvNotifier *o)
 
 PacketRecvInterface * PacketRecvNotifier_GetOutput (PacketRecvNotifier *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->output;
 }
 
 void PacketRecvNotifier_SetHandler (PacketRecvNotifier *o, PacketRecvNotifier_handler_notify handler, void *user)
 {
+    DebugObject_Access(&o->d_obj);
+    
     o->handler = handler;
     o->handler_user = user;
 }

flow/PacketRecvNotifier.h

@@ -31,6 +31,7 @@
 #include <stdint.h>
 
 #include <misc/dead.h>
+#include <misc/debugin.h>
 #include <system/DebugObject.h>
 #include <flow/PacketRecvInterface.h>
 
@@ -49,17 +50,14 @@ typedef void (*PacketRecvNotifier_handler_notify) (void *user, uint8_t *data, in
  * providing a packet to output.
  */
 typedef struct {
-    DebugObject d_obj;
     dead_t dead;
     PacketRecvInterface output;
     PacketRecvInterface *input;
     PacketRecvNotifier_handler_notify handler;
     void *handler_user;
     uint8_t *out;
-    #ifndef NDEBUG
-    int out_have;
-    int in_handler;
-    #endif
+    DebugObject d_obj;
+    DebugIn d_in_handler;
 } PacketRecvNotifier;
 
 /**
@@ -67,8 +65,9 @@ typedef struct {
  *
  * @param o the object
  * @param input input interface
+ * @param pg pending group
  */
-void PacketRecvNotifier_Init (PacketRecvNotifier *o, PacketRecvInterface *input);
+void PacketRecvNotifier_Init (PacketRecvNotifier *o, PacketRecvInterface *input, BPendingGroup *pg);
 
 /**
  * Frees the object.

flow/PacketStreamSender.c

@@ -26,57 +26,33 @@
 
 #include <flow/PacketStreamSender.h>
 
-static int send_data (PacketStreamSender *s)
+static void send_data (PacketStreamSender *s)
 {
     ASSERT(s->in_len >= 0)
     
-    while (s->in_used < s->in_len) {
-        // attempt to send something
-        DEAD_ENTER(s->dead)
-        int res = StreamPassInterface_Sender_Send(s->output, s->in + s->in_used, s->in_len - s->in_used);
-        if (DEAD_LEAVE(s->dead)) {
-            return -1;
-        }
-        
-        ASSERT(res >= 0)
-        ASSERT(res <= s->in_len - s->in_used)
-        
-        if (res == 0) {
-            // output busy, continue in output_handler_done
-            return 0;
-        }
-        
-        // update number of bytes sent
-        s->in_used += res;
+    if (s->in_used < s->in_len) {
+        // send more data
+        StreamPassInterface_Sender_Send(s->output, s->in + s->in_used, s->in_len - s->in_used);
+    } else {
+        // finish input packet
+        s->in_len = -1;
+        PacketPassInterface_Done(&s->input);
     }
-    
-    // everything sent
-    s->in_len = -1;
-    
-    return 0;
 }
 
-static int input_handler_send (PacketStreamSender *s, uint8_t *data, int data_len)
+static void input_handler_send (PacketStreamSender *s, uint8_t *data, int data_len)
 {
     ASSERT(s->in_len == -1)
     ASSERT(data_len >= 0)
+    DebugObject_Access(&s->d_obj);
     
     // set input packet
     s->in_len = data_len;
     s->in = data;
     s->in_used = 0;
     
-    // try sending
-    if (send_data(s) < 0) {
-        return -1;
-    }
-    
-    // if we couldn't send everything, block input
-    if (s->in_len >= 0) {
-        return 0;
-    }
-    
-    return 1;
+    // send
+    send_data(s);
 }
 
 static void output_handler_done (PacketStreamSender *s, int data_len)
@@ -84,37 +60,24 @@ static void output_handler_done (PacketStreamSender *s, int data_len)
     ASSERT(s->in_len >= 0)
     ASSERT(data_len > 0)
     ASSERT(data_len <= s->in_len - s->in_used)
+    DebugObject_Access(&s->d_obj);
     
     // update number of bytes sent
     s->in_used += data_len;
     
-    // continue sending
-    if (send_data(s) < 0) {
-        return;
-    }
-    
-    // if we couldn't send everything, keep input blocked
-    if (s->in_len >= 0) {
-        return;
-    }
-    
-    // allow more input
-    PacketPassInterface_Done(&s->input);
-    return;
+    // send
+    send_data(s);
 }
 
-void PacketStreamSender_Init (PacketStreamSender *s, StreamPassInterface *output, int mtu)
+void PacketStreamSender_Init (PacketStreamSender *s, StreamPassInterface *output, int mtu, BPendingGroup *pg)
 {
     ASSERT(mtu >= 0)
     
     // init arguments
     s->output = output;
     
-    // init dead var
-    DEAD_INIT(s->dead);
-    
     // init input
-    PacketPassInterface_Init(&s->input, mtu, (PacketPassInterface_handler_send)input_handler_send, s);
+    PacketPassInterface_Init(&s->input, mtu, (PacketPassInterface_handler_send)input_handler_send, s, pg);
     
     // init output
     StreamPassInterface_Sender_Init(s->output, (StreamPassInterface_handler_done)output_handler_done, s);
@@ -122,23 +85,20 @@ void PacketStreamSender_Init (PacketStreamSender *s, StreamPassInterface *output
     // have no input packet
     s->in_len = -1;
     
-    // init debug object
     DebugObject_Init(&s->d_obj);
 }
 
 void PacketStreamSender_Free (PacketStreamSender *s)
 {
-    // free debug object
     DebugObject_Free(&s->d_obj);
     
     // free input
     PacketPassInterface_Free(&s->input);
-    
-    // free dead var
-    DEAD_KILL(s->dead);
 }
 
 PacketPassInterface * PacketStreamSender_GetInput (PacketStreamSender *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     return &s->input;
 }

flow/PacketStreamSender.h

@@ -30,7 +30,6 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <system/DebugObject.h>
 #include <flow/PacketPassInterface.h>
 #include <flow/StreamPassInterface.h>
@@ -41,7 +40,6 @@
  */
 typedef struct {
     DebugObject d_obj;
-    dead_t dead;
     PacketPassInterface input;
     StreamPassInterface *output;
     int in_len;
@@ -55,8 +53,9 @@ typedef struct {
  * @param s the object
  * @param output output interface
  * @param mtu input MTU. Must be >=0.
+ * @param pg pending group
  */
-void PacketStreamSender_Init (PacketStreamSender *s, StreamPassInterface *output, int mtu);
+void PacketStreamSender_Init (PacketStreamSender *s, StreamPassInterface *output, int mtu, BPendingGroup *pg);
 
 /**
  * Frees the object.

flow/SCKeepaliveSource.c

@@ -25,27 +25,26 @@
 
 #include <flow/SCKeepaliveSource.h>
 
-static int output_handler_recv (SCKeepaliveSource *o, uint8_t *data, int *data_len)
+static void output_handler_recv (SCKeepaliveSource *o, uint8_t *data)
 {
+    DebugObject_Access(&o->d_obj);
+    
     struct sc_header *header = (struct sc_header *)data;
-    header->type = SCID_KEEPALIVE;
+    header->type = htol8(SCID_KEEPALIVE);
     
-    *data_len = sizeof(struct sc_header);
-    return 1;
+    PacketRecvInterface_Done(&o->output, sizeof(struct sc_header));
 }
 
-void SCKeepaliveSource_Init (SCKeepaliveSource *o)
+void SCKeepaliveSource_Init (SCKeepaliveSource *o, BPendingGroup *pg)
 {
     // init output
-    PacketRecvInterface_Init(&o->output, sizeof(struct sc_header), (PacketRecvInterface_handler_recv)output_handler_recv, o);
+    PacketRecvInterface_Init(&o->output, sizeof(struct sc_header), (PacketRecvInterface_handler_recv)output_handler_recv, o, pg);
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
 void SCKeepaliveSource_Free (SCKeepaliveSource *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // free output
@@ -54,5 +53,7 @@ void SCKeepaliveSource_Free (SCKeepaliveSource *o)
 
 PacketRecvInterface * SCKeepaliveSource_GetOutput (SCKeepaliveSource *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->output;
 }

flow/SCKeepaliveSource.h

@@ -42,8 +42,9 @@ typedef struct {
  * Initializes the object.
  *
  * @param o the object
+ * @param pg pending group
  */
-void SCKeepaliveSource_Init (SCKeepaliveSource *o);
+void SCKeepaliveSource_Init (SCKeepaliveSource *o, BPendingGroup *pg);
 
 /**
  * Frees the object.

flow/SPProtoDecoder.c

@@ -134,37 +134,33 @@ static int decode_packet (SPProtoDecoder *o, uint8_t *in, int in_len, uint8_t **
     return 1;
 }
 
-static int input_handler_send (SPProtoDecoder *o, uint8_t *data, int data_len)
+static void input_handler_send (SPProtoDecoder *o, uint8_t *data, int data_len)
 {
     ASSERT(data_len >= 0)
     ASSERT(data_len <= o->input_mtu)
+    DebugObject_Access(&o->d_obj);
     
     // attempt to decode packet
     uint8_t *out;
     int out_len;
     if (!decode_packet(o, data, data_len, &out, &out_len)) {
-        return 1;
-    }
-    
-    // submit decoded packet to output
-    DEAD_ENTER(o->dead)
-    int res = PacketPassInterface_Sender_Send(o->output, out, out_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
+        // cannot decode, finish input packet
+        PacketPassInterface_Done(&o->input);
+    } else {
+        // submit decoded packet to output
+        PacketPassInterface_Sender_Send(o->output, out, out_len);
     }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    return res;
 }
 
 static void output_handler_done (SPProtoDecoder *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
+    // finish input packet
     PacketPassInterface_Done(&o->input);
-    return;
 }
 
-int SPProtoDecoder_Init (SPProtoDecoder *o, PacketPassInterface *output, struct spproto_security_params sp_params, int num_otp_seeds)
+int SPProtoDecoder_Init (SPProtoDecoder *o, PacketPassInterface *output, struct spproto_security_params sp_params, int num_otp_seeds, BPendingGroup *pg)
 {
     ASSERT(spproto_validate_security_params(sp_params))
     ASSERT(!SPPROTO_HAVE_OTP(sp_params) || num_otp_seeds >= 2)
@@ -173,9 +169,6 @@ int SPProtoDecoder_Init (SPProtoDecoder *o, PacketPassInterface *output, struct
     o->output = output;
     o->sp_params = sp_params;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init output
     PacketPassInterface_Sender_Init(o->output, (PacketPassInterface_handler_done)output_handler_done, o);
     
@@ -205,7 +198,7 @@ int SPProtoDecoder_Init (SPProtoDecoder *o, PacketPassInterface *output, struct
     }
     
     // init input
-    PacketPassInterface_Init(&o->input, o->input_mtu, (PacketPassInterface_handler_send)input_handler_send, o);
+    PacketPassInterface_Init(&o->input, o->input_mtu, (PacketPassInterface_handler_send)input_handler_send, o, pg);
     
     // init OTP checker
     if (SPPROTO_HAVE_OTP(o->sp_params)) {
@@ -219,7 +212,6 @@ int SPProtoDecoder_Init (SPProtoDecoder *o, PacketPassInterface *output, struct
         o->have_encryption_key = 0;
     }
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
     
     return 1;
@@ -235,7 +227,6 @@ fail0:
 
 void SPProtoDecoder_Free (SPProtoDecoder *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
 
     // free encryptor
@@ -255,19 +246,19 @@ void SPProtoDecoder_Free (SPProtoDecoder *o)
     if (SPPROTO_HAVE_ENCRYPTION(o->sp_params)) {
         free(o->buf);
     }
-    
-    // kill dead var
-    DEAD_KILL(o->dead);
 }
 
 PacketPassInterface * SPProtoDecoder_GetInput (SPProtoDecoder *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->input;
 }
 
 void SPProtoDecoder_SetEncryptionKey (SPProtoDecoder *o, uint8_t *encryption_key)
 {
     ASSERT(SPPROTO_HAVE_ENCRYPTION(o->sp_params))
+    DebugObject_Access(&o->d_obj);
     
     // free encryptor
     if (o->have_encryption_key) {
@@ -284,6 +275,7 @@ void SPProtoDecoder_SetEncryptionKey (SPProtoDecoder *o, uint8_t *encryption_key
 void SPProtoDecoder_RemoveEncryptionKey (SPProtoDecoder *o)
 {
     ASSERT(SPPROTO_HAVE_ENCRYPTION(o->sp_params))
+    DebugObject_Access(&o->d_obj);
     
     if (o->have_encryption_key) {
         // free encryptor
@@ -297,6 +289,7 @@ void SPProtoDecoder_RemoveEncryptionKey (SPProtoDecoder *o)
 void SPProtoDecoder_AddOTPSeed (SPProtoDecoder *o, uint16_t seed_id, uint8_t *key, uint8_t *iv)
 {
     ASSERT(SPPROTO_HAVE_OTP(o->sp_params))
+    DebugObject_Access(&o->d_obj);
     
     OTPChecker_AddSeed(&o->otpchecker, seed_id, key, iv);
 }
@@ -304,6 +297,7 @@ void SPProtoDecoder_AddOTPSeed (SPProtoDecoder *o, uint16_t seed_id, uint8_t *ke
 void SPProtoDecoder_RemoveOTPSeeds (SPProtoDecoder *o)
 {
     ASSERT(SPPROTO_HAVE_OTP(o->sp_params))
+    DebugObject_Access(&o->d_obj);
     
     OTPChecker_RemoveSeeds(&o->otpchecker);
 }

flow/SPProtoDecoder.h

@@ -42,8 +42,6 @@
  * Output is with {@link PacketPassInterface}.
  */
 typedef struct {
-    DebugObject d_obj;
-    dead_t dead;
     PacketPassInterface *output;
     int output_mtu;
     struct spproto_security_params sp_params;
@@ -56,6 +54,7 @@ typedef struct {
     OTPChecker otpchecker;
     int have_encryption_key;
     BEncryption encryptor;
+    DebugObject d_obj;
 } SPProtoDecoder;
 
 /**
@@ -67,9 +66,10 @@ typedef struct {
  * @param encryption_key if using encryption, the encryption key
  * @param num_otp_seeds if using OTPs, how many OTP seeds to keep for checking
  *                      receiving packets. Must be >=2 if using OTPs.
+ * @param pg pending group
  * @return 1 on success, 0 on failure
  */
-int SPProtoDecoder_Init (SPProtoDecoder *o, PacketPassInterface *output, struct spproto_security_params sp_params, int num_otp_seeds) WARN_UNUSED;
+int SPProtoDecoder_Init (SPProtoDecoder *o, PacketPassInterface *output, struct spproto_security_params sp_params, int num_otp_seeds, BPendingGroup *pg) WARN_UNUSED;
 
 /**
  * Frees the object.

flow/SPProtoEncoder.c

@@ -37,8 +37,8 @@ static int can_encode (SPProtoEncoder *o)
     ASSERT(o->out_have)
     
     return (
-        (!SPPROTO_HAVE_OTP(o->group->sp_params) || OTPGenerator_GetPosition(&o->group->otpgen) < o->group->sp_params.otp_num) &&
-        (!SPPROTO_HAVE_ENCRYPTION(o->group->sp_params) || o->group->have_encryption_key)
+        (!SPPROTO_HAVE_OTP(o->sp_params) || OTPGenerator_GetPosition(&o->otpgen) < o->sp_params.otp_num) &&
+        (!SPPROTO_HAVE_ENCRYPTION(o->sp_params) || o->have_encryption_key)
     );
 }
 
@@ -50,154 +50,122 @@ static int encode_packet (SPProtoEncoder *o)
     ASSERT(can_encode(o))
     
     // plaintext is either output packet or our buffer
-    uint8_t *plaintext = (!SPPROTO_HAVE_ENCRYPTION(o->group->sp_params) ? o->out : o->buf);
+    uint8_t *plaintext = (!SPPROTO_HAVE_ENCRYPTION(o->sp_params) ? o->out : o->buf);
     
     // plaintext begins with header
     uint8_t *header = plaintext;
     
     // plaintext is header + payload
-    int plaintext_len = SPPROTO_HEADER_LEN(o->group->sp_params) + o->in_len;
+    int plaintext_len = SPPROTO_HEADER_LEN(o->sp_params) + o->in_len;
     
     // write OTP
-    if (SPPROTO_HAVE_OTP(o->group->sp_params)) {
+    if (SPPROTO_HAVE_OTP(o->sp_params)) {
         struct spproto_otpdata *header_otpd = (struct spproto_otpdata *)(header + SPPROTO_HEADER_OTPDATA_OFF(o->group->sp_params));
-        header_otpd->seed_id = o->group->otpgen_seed_id;
-        header_otpd->otp = OTPGenerator_GetOTP(&o->group->otpgen);
+        header_otpd->seed_id = o->otpgen_seed_id;
+        header_otpd->otp = OTPGenerator_GetOTP(&o->otpgen);
     }
     
     // write hash
-    if (SPPROTO_HAVE_HASH(o->group->sp_params)) {
-        uint8_t *header_hash = header + SPPROTO_HEADER_HASH_OFF(o->group->sp_params);
+    if (SPPROTO_HAVE_HASH(o->sp_params)) {
+        uint8_t *header_hash = header + SPPROTO_HEADER_HASH_OFF(o->sp_params);
         // zero hash field
-        memset(header_hash, 0, o->group->hash_size);
+        memset(header_hash, 0, o->hash_size);
         // calculate hash
-        uint8_t hash[o->group->hash_size];
-        BHash_calculate(o->group->sp_params.hash_mode, plaintext, plaintext_len, hash);
+        uint8_t hash[o->hash_size];
+        BHash_calculate(o->sp_params.hash_mode, plaintext, plaintext_len, hash);
         // set hash field
-        memcpy(header_hash, hash, o->group->hash_size);
+        memcpy(header_hash, hash, o->hash_size);
     }
     
     int out_len;
     
-    if (SPPROTO_HAVE_ENCRYPTION(o->group->sp_params)) {
+    if (SPPROTO_HAVE_ENCRYPTION(o->sp_params)) {
         // encrypting pad(header + payload)
-        int cyphertext_len = BALIGN_UP_N((plaintext_len + 1), o->group->enc_block_size);
+        int cyphertext_len = BALIGN_UP_N((plaintext_len + 1), o->enc_block_size);
         // write padding
         plaintext[plaintext_len] = 1;
         for (int i = plaintext_len + 1; i < cyphertext_len; i++) {
             plaintext[i] = 0;
         }
         // generate IV
-        BRandom_randomize(o->out, o->group->enc_block_size);
+        BRandom_randomize(o->out, o->enc_block_size);
         // copy IV because BEncryption_Encrypt changes the IV
-        uint8_t iv[o->group->enc_block_size];
-        memcpy(iv, o->out, o->group->enc_block_size);
+        uint8_t iv[o->enc_block_size];
+        memcpy(iv, o->out, o->enc_block_size);
         // encrypt
-        BEncryption_Encrypt(&o->group->encryptor, plaintext, o->out + o->group->enc_block_size, cyphertext_len, iv);
-        out_len = o->group->enc_block_size + cyphertext_len;
+        BEncryption_Encrypt(&o->encryptor, plaintext, o->out + o->enc_block_size, cyphertext_len, iv);
+        out_len = o->enc_block_size + cyphertext_len;
     } else {
         out_len = plaintext_len;
     }
     
+    return out_len;
+}
+
+static void do_encode (SPProtoEncoder *o)
+{
+    ASSERT(o->in_len >= 0)
+    ASSERT(o->in_len <= o->input_mtu)
+    ASSERT(o->out_have)
+    ASSERT(can_encode(o))
+    
+    // encode
+    int out_len = encode_packet(o);
+    
+    // finish packet
+    PacketRecvInterface_Done(&o->output, out_len);
     o->in_len = -1;
     o->out_have = 0;
-    
-    return out_len;
 }
 
-static int output_handler_recv (SPProtoEncoder *o, uint8_t *data, int *data_len)
+static void maybe_encode (SPProtoEncoder *o)
+{
+    if (o->in_len >= 0 && o->out_have && can_encode(o)) {
+        do_encode(o);
+    }
+}
+
+static void output_handler_recv (SPProtoEncoder *o, uint8_t *data)
 {
     ASSERT(o->in_len == -1)
     ASSERT(!o->out_have)
+    DebugObject_Access(&o->d_obj);
     
     // remember output packet
     o->out_have = 1;
     o->out = data;
     
     // determine plaintext location
-    uint8_t *plaintext = (!SPPROTO_HAVE_ENCRYPTION(o->group->sp_params) ? o->out : o->buf);
-    
-    // try to receive input packet
-    int in_len;
-    DEAD_ENTER(o->dead)
-    int res = PacketRecvInterface_Receiver_Recv(o->input, plaintext + SPPROTO_HEADER_LEN(o->group->sp_params), &in_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
+    uint8_t *plaintext = (!SPPROTO_HAVE_ENCRYPTION(o->sp_params) ? o->out : o->buf);
     
-    ASSERT(res == 0 || res == 1)
-    
-    if (!res) {
-        return 0;
-    }
-    
-    ASSERT(in_len >= 0 && in_len <= o->input_mtu)
-    
-    // remember input packet
-    o->in_len = in_len;
-    
-    // check if we can encode
-    if (!can_encode(o)) {
-        return 0;
-    }
-    
-    // encode
-    *data_len = encode_packet(o);
-    
-    return 1;
+    // schedule receive
+    PacketRecvInterface_Receiver_Recv(o->input, plaintext + SPPROTO_HEADER_LEN(o->sp_params));
 }
 
 static void input_handler_done (SPProtoEncoder *o, int in_len)
 {
+    ASSERT(in_len >= 0)
+    ASSERT(in_len <= o->input_mtu)
     ASSERT(o->in_len == -1)
     ASSERT(o->out_have)
-    ASSERT(in_len >= 0 && in_len <= o->input_mtu)
+    DebugObject_Access(&o->d_obj);
     
     // remember input packet
     o->in_len = in_len;
     
-    // check if we can encode
-    if (!can_encode(o)) {
-        return;
-    }
-    
-    // encode
-    int out_len = encode_packet(o);
-    
-    // inform output
-    PacketRecvInterface_Done(&o->output, out_len);
-    return;
-}
-
-static void job_handler (SPProtoEncoder *o)
-{
-    if (o->in_len >= 0 && o->out_have && can_encode(o)) {
-        // encode
-        int out_len = encode_packet(o);
-        
-        // inform output
-        PacketRecvInterface_Done(&o->output, out_len);
-        return;
+    // encode if possible
+    if (can_encode(o)) {
+        do_encode(o);
     }
 }
 
-static void schedule_jobs (SPProtoEncoderGroup *o)
-{
-    LinkedList2Iterator it;
-    LinkedList2Iterator_InitForward(&it, &o->encoders_list);
-    LinkedList2Node *node;
-    while (node = LinkedList2Iterator_Next(&it)) {
-        SPProtoEncoder *enc = UPPER_OBJECT(node, SPProtoEncoder, group_list_node);
-        BPending_Set(&enc->continue_job);
-    }
-}
-
-int SPProtoEncoderGroup_Init (SPProtoEncoderGroup *o, struct spproto_security_params sp_params)
+int SPProtoEncoder_Init (SPProtoEncoder *o, struct spproto_security_params sp_params, PacketRecvInterface *input, BPendingGroup *pg)
 {
     ASSERT(spproto_validate_security_params(sp_params))
     
     // init parameters
     o->sp_params = sp_params;
+    o->input = input;
     
     // calculate hash size
     if (SPPROTO_HAVE_HASH(o->sp_params)) {
@@ -213,7 +181,7 @@ int SPProtoEncoderGroup_Init (SPProtoEncoderGroup *o, struct spproto_security_pa
     // init otp generator
     if (SPPROTO_HAVE_OTP(o->sp_params)) {
         if (!OTPGenerator_Init(&o->otpgen, o->sp_params.otp_num, o->sp_params.otp_mode)) {
-            goto fail1;
+            goto fail0;
         }
     }
     
@@ -222,23 +190,57 @@ int SPProtoEncoderGroup_Init (SPProtoEncoderGroup *o, struct spproto_security_pa
         o->have_encryption_key = 0;
     }
     
-    // init encoders list
-    LinkedList2_Init(&o->encoders_list);
+    // remember input MTU
+    o->input_mtu = PacketRecvInterface_GetMTU(o->input);
+    
+    // calculate output MTU
+    o->output_mtu = spproto_carrier_mtu_for_payload_mtu(o->sp_params, o->input_mtu);
+    
+    // init input
+    PacketRecvInterface_Receiver_Init(o->input, (PacketRecvInterface_handler_done)input_handler_done, o);
+    
+    // have no input in buffer
+    o->in_len = -1;
+    
+    // init output
+    PacketRecvInterface_Init(&o->output, o->output_mtu, (PacketRecvInterface_handler_recv)output_handler_recv, o, pg);
+    
+    // have no output available
+    o->out_have = 0;
+    
+    // allocate plaintext buffer
+    if (SPPROTO_HAVE_ENCRYPTION(o->sp_params)) {
+        int buf_size = BALIGN_UP_N((SPPROTO_HEADER_LEN(o->sp_params) + o->input_mtu + 1), o->enc_block_size);
+        if (!(o->buf = malloc(buf_size))) {
+            goto fail1;
+        }
+    }
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
     
     return 1;
     
 fail1:
+    PacketRecvInterface_Free(&o->output);
+    if (SPPROTO_HAVE_OTP(o->sp_params)) {
+        OTPGenerator_Free(&o->otpgen);
+    }
+fail0:
     return 0;
 }
 
-void SPProtoEncoderGroup_Free (SPProtoEncoderGroup *o)
+void SPProtoEncoder_Free (SPProtoEncoder *o)
 {
-    ASSERT(LinkedList2_IsEmpty(&o->encoders_list))
     DebugObject_Free(&o->d_obj);
     
+    // free plaintext buffer
+    if (SPPROTO_HAVE_ENCRYPTION(o->sp_params)) {
+        free(o->buf);
+    }
+    
+    // free output
+    PacketRecvInterface_Free(&o->output);
+    
     // free encryptor
     if (SPPROTO_HAVE_ENCRYPTION(o->sp_params) && o->have_encryption_key) {
         BEncryption_Free(&o->encryptor);
@@ -250,9 +252,17 @@ void SPProtoEncoderGroup_Free (SPProtoEncoderGroup *o)
     }
 }
 
-void SPProtoEncoderGroup_SetEncryptionKey (SPProtoEncoderGroup *o, uint8_t *encryption_key)
+PacketRecvInterface * SPProtoEncoder_GetOutput (SPProtoEncoder *o)
+{
+    DebugObject_Access(&o->d_obj);
+    
+    return &o->output;
+}
+
+void SPProtoEncoder_SetEncryptionKey (SPProtoEncoder *o, uint8_t *encryption_key)
 {
     ASSERT(SPPROTO_HAVE_ENCRYPTION(o->sp_params))
+    DebugObject_Access(&o->d_obj);
     
     // free encryptor
     if (o->have_encryption_key) {
@@ -265,13 +275,14 @@ void SPProtoEncoderGroup_SetEncryptionKey (SPProtoEncoderGroup *o, uint8_t *encr
     // have encryption key
     o->have_encryption_key = 1;
     
-    // set jobs
-    schedule_jobs(o);
+    // possibly continue I/O
+    maybe_encode(o);
 }
 
-void SPProtoEncoderGroup_RemoveEncryptionKey (SPProtoEncoderGroup *o)
+void SPProtoEncoder_RemoveEncryptionKey (SPProtoEncoder *o)
 {
     ASSERT(SPPROTO_HAVE_ENCRYPTION(o->sp_params))
+    DebugObject_Access(&o->d_obj);
     
     if (o->have_encryption_key) {
         // free encryptor
@@ -282,9 +293,10 @@ void SPProtoEncoderGroup_RemoveEncryptionKey (SPProtoEncoderGroup *o)
     }
 }
 
-void SPProtoEncoderGroup_SetOTPSeed (SPProtoEncoderGroup *o, uint16_t seed_id, uint8_t *key, uint8_t *iv)
+void SPProtoEncoder_SetOTPSeed (SPProtoEncoder *o, uint16_t seed_id, uint8_t *key, uint8_t *iv)
 {
     ASSERT(SPPROTO_HAVE_OTP(o->sp_params))
+    DebugObject_Access(&o->d_obj);
     
     // give seed to OTP generator
     OTPGenerator_SetSeed(&o->otpgen, key, iv);
@@ -292,100 +304,23 @@ void SPProtoEncoderGroup_SetOTPSeed (SPProtoEncoderGroup *o, uint16_t seed_id, u
     // remember seed ID
     o->otpgen_seed_id = seed_id;
     
-    // set jobs
-    schedule_jobs(o);
+    // possibly continue I/O
+    maybe_encode(o);
 }
 
-void SPProtoEncoderGroup_RemoveOTPSeed (SPProtoEncoderGroup *o)
+void SPProtoEncoder_RemoveOTPSeed (SPProtoEncoder *o)
 {
     ASSERT(SPPROTO_HAVE_OTP(o->sp_params))
+    DebugObject_Access(&o->d_obj);
     
     // reset OTP generator
     OTPGenerator_Reset(&o->otpgen);
 }
 
-int SPProtoEncoderGroup_GetOTPPosition (SPProtoEncoderGroup *o)
+int SPProtoEncoder_GetOTPPosition (SPProtoEncoder *o)
 {
     ASSERT(SPPROTO_HAVE_OTP(o->sp_params))
+    DebugObject_Access(&o->d_obj);
     
     return OTPGenerator_GetPosition(&o->otpgen);
 }
-
-int SPProtoEncoder_Init (SPProtoEncoder *o, SPProtoEncoderGroup *group, PacketRecvInterface *input, BPendingGroup *pg)
-{
-    // init parameters
-    o->group = group;
-    o->input = input;
-    
-    // init dead var
-    DEAD_INIT(o->dead);
-    
-    // remember input MTU
-    o->input_mtu = PacketRecvInterface_GetMTU(o->input);
-    
-    // calculate output MTU
-    o->output_mtu = spproto_carrier_mtu_for_payload_mtu(o->group->sp_params, o->input_mtu);
-    
-    // init input
-    PacketRecvInterface_Receiver_Init(o->input, (PacketRecvInterface_handler_done)input_handler_done, o);
-    
-    // have no input in buffer
-    o->in_len = -1;
-    
-    // init output
-    PacketRecvInterface_Init(&o->output, o->output_mtu, (PacketRecvInterface_handler_recv)output_handler_recv, o);
-    
-    // have no output available
-    o->out_have = 0;
-    
-    // allocate plaintext buffer
-    if (SPPROTO_HAVE_ENCRYPTION(o->group->sp_params)) {
-        int buf_size = BALIGN_UP_N((SPPROTO_HEADER_LEN(o->group->sp_params) + o->input_mtu + 1), o->group->enc_block_size);
-        if (!(o->buf = malloc(buf_size))) {
-            goto fail1;
-        }
-    }
-    
-    // insert to group list
-    LinkedList2_Append(&o->group->encoders_list, &o->group_list_node);
-    
-    // init pending job
-    BPending_Init(&o->continue_job, pg, (BPending_handler)job_handler, o);
-    
-    // init debug object
-    DebugObject_Init(&o->d_obj);
-    
-    return 1;
-    
-fail1:
-    PacketRecvInterface_Free(&o->output);
-    return 0;
-}
-
-void SPProtoEncoder_Free (SPProtoEncoder *o)
-{
-    // free debug object
-    DebugObject_Free(&o->d_obj);
-    
-    // free pending job
-    BPending_Free(&o->continue_job);
-    
-    // remove from group list
-    LinkedList2_Remove(&o->group->encoders_list, &o->group_list_node);
-    
-    // free plaintext buffer
-    if (SPPROTO_HAVE_ENCRYPTION(o->group->sp_params)) {
-        free(o->buf);
-    }
-    
-    // free output
-    PacketRecvInterface_Free(&o->output);
-    
-    // kill dead var
-    DEAD_KILL(o->dead);
-}
-
-PacketRecvInterface * SPProtoEncoder_GetOutput (SPProtoEncoder *o)
-{
-    return &o->output;
-}

flow/SPProtoEncoder.h

@@ -29,21 +29,20 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <misc/debug.h>
 #include <protocol/spproto.h>
-#include <structure/LinkedList2.h>
 #include <system/DebugObject.h>
-#include <system/BPending.h>
 #include <security/BEncryption.h>
 #include <security/OTPGenerator.h>
 #include <flow/PacketRecvInterface.h>
 
 /**
- * Object shared between {@link SPProtoEncoder} objects using the same security parameters and resources.
+ * Object which encodes packets according to SPProto.
+ *
+ * Input is with {@link PacketRecvInterface}.
+ * Output is with {@link PacketRecvInterface}.
  */
 typedef struct {
-    DebugObject d_obj;
     struct spproto_security_params sp_params;
     int hash_size;
     int enc_block_size;
@@ -52,25 +51,48 @@ typedef struct {
     uint16_t otpgen_seed_id;
     int have_encryption_key;
     BEncryption encryptor;
-    LinkedList2 encoders_list;
-} SPProtoEncoderGroup;
+    
+    int input_mtu;
+    int output_mtu;
+    PacketRecvInterface *input;
+    int in_len;
+    PacketRecvInterface output;
+    int out_have;
+    uint8_t *out;
+    uint8_t *buf;
+    LinkedList2Node group_list_node;
+    BPending continue_job;
+    DebugObject d_obj;
+} SPProtoEncoder;
 
 /**
  * Initializes the object.
+ * The object is initialized in blocked state.
  *
  * @param o the object
  * @param sp_params SPProto security parameters. Must be valid according to {@link spproto_validate_security_params}.
+ * @param input input interface
+ * @param pg pending group
  * @return 1 on success, 0 on failure
  */
-int SPProtoEncoderGroup_Init (SPProtoEncoderGroup *o, struct spproto_security_params sp_params) WARN_UNUSED;
+int SPProtoEncoder_Init (SPProtoEncoder *o, struct spproto_security_params sp_params, PacketRecvInterface *input, BPendingGroup *pg) WARN_UNUSED;
 
 /**
  * Frees the object.
- * There must be no encoders using this group.
  *
  * @param o the object
  */
-void SPProtoEncoderGroup_Free (SPProtoEncoderGroup *o);
+void SPProtoEncoder_Free (SPProtoEncoder *o);
+
+/**
+ * Returns the output interface.
+ * The MTU of the output interface will depend on the input MTU and security parameters,
+ * that is spproto_carrier_mtu_for_payload_mtu(sp_params, input MTU).
+ *
+ * @param o the object
+ * @return output interface
+ */
+PacketRecvInterface * SPProtoEncoder_GetOutput (SPProtoEncoder *o);
 
 /**
  * Sets an encryption key to use.
@@ -79,7 +101,7 @@ void SPProtoEncoderGroup_Free (SPProtoEncoderGroup *o);
  * @param o the object
  * @param encryption_key key to use
  */
-void SPProtoEncoderGroup_SetEncryptionKey (SPProtoEncoderGroup *o, uint8_t *encryption_key);
+void SPProtoEncoder_SetEncryptionKey (SPProtoEncoder *o, uint8_t *encryption_key);
 
 /**
  * Removes an encryption key if one is configured.
@@ -87,7 +109,7 @@ void SPProtoEncoderGroup_SetEncryptionKey (SPProtoEncoderGroup *o, uint8_t *encr
  *
  * @param o the object
  */
-void SPProtoEncoderGroup_RemoveEncryptionKey (SPProtoEncoderGroup *o);
+void SPProtoEncoder_RemoveEncryptionKey (SPProtoEncoder *o);
 
 /**
  * Sets an OTP seed to use.
@@ -98,7 +120,7 @@ void SPProtoEncoderGroup_RemoveEncryptionKey (SPProtoEncoderGroup *o);
  * @param key OTP encryption key
  * @param iv OTP initialization vector
  */
-void SPProtoEncoderGroup_SetOTPSeed (SPProtoEncoderGroup *o, uint16_t seed_id, uint8_t *key, uint8_t *iv);
+void SPProtoEncoder_SetOTPSeed (SPProtoEncoder *o, uint16_t seed_id, uint8_t *key, uint8_t *iv);
 
 /**
  * Removes the OTP seed if one is configured.
@@ -106,7 +128,7 @@ void SPProtoEncoderGroup_SetOTPSeed (SPProtoEncoderGroup *o, uint16_t seed_id, u
  *
  * @param o the object
  */
-void SPProtoEncoderGroup_RemoveOTPSeed (SPProtoEncoderGroup *o);
+void SPProtoEncoder_RemoveOTPSeed (SPProtoEncoder *o);
 
 /**
  * Returns the number of OTPs used so far, or total number if
@@ -116,57 +138,6 @@ void SPProtoEncoderGroup_RemoveOTPSeed (SPProtoEncoderGroup *o);
  * @param o the object
  * @return OTP position
  */
-int SPProtoEncoderGroup_GetOTPPosition (SPProtoEncoderGroup *o);
-
-/**
- * Object which encodes packets according to SPProto.
- *
- * Input is with {@link PacketRecvInterface}.
- * Output is with {@link PacketRecvInterface}.
- */
-typedef struct {
-    DebugObject d_obj;
-    dead_t dead;
-    SPProtoEncoderGroup *group;
-    int input_mtu;
-    int output_mtu;
-    PacketRecvInterface *input;
-    int in_len;
-    PacketRecvInterface output;
-    int out_have;
-    uint8_t *out;
-    uint8_t *buf;
-    LinkedList2Node group_list_node;
-    BPending continue_job;
-} SPProtoEncoder;
-
-/**
- * Initializes the object.
- * The object is initialized in blocked state.
- *
- * @param o the object
- * @param group {@link SPProtoEncoderGroup} object to use for security parameters and resources
- * @param input input interface
- * @param pg pending group
- * @return 1 on success, 0 on failure
- */
-int SPProtoEncoder_Init (SPProtoEncoder *o, SPProtoEncoderGroup *group, PacketRecvInterface *input, BPendingGroup *pg) WARN_UNUSED;
-
-/**
- * Frees the object.
- *
- * @param o the object
- */
-void SPProtoEncoder_Free (SPProtoEncoder *o);
-
-/**
- * Returns the output interface.
- * The MTU of the output interface will depend on the input MTU and security parameters,
- * that is spproto_carrier_mtu_for_payload_mtu(sp_params, input MTU).
- *
- * @param o the object
- * @return output interface
- */
-PacketRecvInterface * SPProtoEncoder_GetOutput (SPProtoEncoder *o);
+int SPProtoEncoder_GetOTPPosition (SPProtoEncoder *o);
 
 #endif

flow/SinglePacketBuffer.c

@@ -26,73 +26,18 @@
 
 #include <flow/SinglePacketBuffer.h>
 
-static int io_loop (SinglePacketBuffer *o)
-{
-    DEAD_DECLARE
-    int res;
-    
-    while (1) {
-        // receive packet
-        int in_len;
-        DEAD_ENTER2(o->dead)
-        res = PacketRecvInterface_Receiver_Recv(o->input, o->buf, &in_len);
-        if (DEAD_LEAVE(o->dead)) {
-            return -1;
-        }
-        
-        ASSERT(res == 0 || res == 1)
-        
-        if (!res) {
-            // input blocking, continue in input_handler_done
-            return 0;
-        }
-        
-        // send packet
-        DEAD_ENTER2(o->dead)
-        res = PacketPassInterface_Sender_Send(o->output, o->buf, in_len);
-        if (DEAD_LEAVE(o->dead)) {
-            return -1;
-        }
-        
-        ASSERT(res == 0 || res == 1)
-        
-        if (!res) {
-            // output blocking, continue in output_handler_done
-            return 0;
-        }
-    }
-}
-
 static void input_handler_done (SinglePacketBuffer *o, int in_len)
 {
-    // send packet
-    DEAD_ENTER(o->dead)
-    int res = PacketPassInterface_Sender_Send(o->output, o->buf, in_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    if (!res) {
-        // output blocking, continue in output_handler_done
-        return;
-    }
+    DebugObject_Access(&o->d_obj);
     
-    io_loop(o);
-    return;
+    PacketPassInterface_Sender_Send(o->output, o->buf, in_len);
 }
 
 static void output_handler_done (SinglePacketBuffer *o)
 {
-    io_loop(o);
-    return;
-}
-
-static void job_handler (SinglePacketBuffer *o)
-{
-    io_loop(o);
-    return;
+    DebugObject_Access(&o->d_obj);
+    
+    PacketRecvInterface_Receiver_Recv(o->input, o->buf);
 }
 
 int SinglePacketBuffer_Init (SinglePacketBuffer *o, PacketRecvInterface *input, PacketPassInterface *output, BPendingGroup *pg) 
@@ -103,9 +48,6 @@ int SinglePacketBuffer_Init (SinglePacketBuffer *o, PacketRecvInterface *input,
     o->input = input;
     o->output = output;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init input
     PacketRecvInterface_Receiver_Init(o->input, (PacketRecvInterface_handler_done)input_handler_done, o);
     
@@ -117,11 +59,9 @@ int SinglePacketBuffer_Init (SinglePacketBuffer *o, PacketRecvInterface *input,
         goto fail1;
     }
     
-    // init start job
-    BPending_Init(&o->start_job, pg, (BPending_handler)job_handler, o);
-    BPending_Set(&o->start_job);
+    // schedule receive
+    PacketRecvInterface_Receiver_Recv(o->input, o->buf);
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
     
     return 1;
@@ -132,15 +72,8 @@ fail1:
 
 void SinglePacketBuffer_Free (SinglePacketBuffer *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
     
-    // free start job
-    BPending_Free(&o->start_job);
-    
     // free buffer
     free(o->buf);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }

flow/SinglePacketBuffer.h

@@ -30,10 +30,8 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <misc/debug.h>
 #include <system/DebugObject.h>
-#include <system/BPending.h>
 #include <flow/PacketRecvInterface.h>
 #include <flow/PacketPassInterface.h>
 
@@ -43,11 +41,9 @@
  */
 typedef struct {
     DebugObject d_obj;
-    dead_t dead;
     PacketRecvInterface *input;
     PacketPassInterface *output;
     uint8_t *buf;
-    BPending start_job;
 } SinglePacketBuffer;
 
 /**

flow/SinglePacketSender.c

@@ -27,14 +27,14 @@
 static void call_handler (SinglePacketSender *o)
 {
     #ifndef NDEBUG
-    DEAD_ENTER(o->dead)
+    DEAD_ENTER(o->d_dead)
     #endif
     
     o->handler(o->user);
     
     #ifndef NDEBUG
     ASSERT(DEAD_KILLED)
-    DEAD_LEAVE(o->dead);
+    DEAD_LEAVE(o->d_dead);
     #endif
 }
 
@@ -47,59 +47,32 @@ static void output_handler_done (SinglePacketSender *o)
     return;
 }
 
-static void job_handler (SinglePacketSender *o)
-{
-    DebugObject_Access(&o->d_obj);
-    
-    DEAD_ENTER(o->dead)
-    int res = PacketPassInterface_Sender_Send(o->output, o->packet, o->packet_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    if (!res) {
-        return;
-    }
-    
-    // notify user
-    call_handler(o);
-    return;
-}
-
 void SinglePacketSender_Init (SinglePacketSender *o, uint8_t *packet, int packet_len, PacketPassInterface *output, SinglePacketSender_handler handler, void *user, BPendingGroup *pg)
 {
     ASSERT(packet_len >= 0)
     ASSERT(packet_len <= PacketPassInterface_GetMTU(output))
     
     // init arguments
-    o->packet = packet;
-    o->packet_len = packet_len;
     o->output = output;
     o->handler = handler;
     o->user = user;
     
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init output
     PacketPassInterface_Sender_Init(o->output, (PacketPassInterface_handler_done)output_handler_done, o);
     
-    // init start job
-    BPending_Init(&o->start_job, pg, (BPending_handler)job_handler, o);
-    BPending_Set(&o->start_job);
+    // schedule send
+    PacketPassInterface_Sender_Send(o->output, packet, packet_len);
     
     DebugObject_Init(&o->d_obj);
+    #ifndef NDEBUG
+    DEAD_INIT(o->d_dead);
+    #endif
 }
 
 void SinglePacketSender_Free (SinglePacketSender *o)
 {
+    #ifndef NDEBUG
+    DEAD_KILL(o->d_dead);
+    #endif
     DebugObject_Free(&o->d_obj);
-    
-    // free start job
-    BPending_Free(&o->start_job);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }

flow/SinglePacketSender.h

@@ -30,7 +30,6 @@
 #include <stdint.h>
 
 #include <misc/dead.h>
-#include <system/BPending.h>
 #include <system/DebugObject.h>
 #include <flow/PacketPassInterface.h>
 
@@ -46,14 +45,13 @@ typedef void (*SinglePacketSender_handler) (void *user);
  * A {@link PacketPassInterface} source which sends a single packet.
  */
 typedef struct {
-    dead_t dead;
-    uint8_t *packet;
-    int packet_len;
     PacketPassInterface *output;
     SinglePacketSender_handler handler;
     void *user;
-    BPending start_job;
     DebugObject d_obj;
+    #ifndef NDEBUG
+    dead_t d_dead;
+    #endif
 } SinglePacketSender;
 
 /**

flow/StreamPassInterface.c

@@ -0,0 +1,79 @@
+/**
+ * @file StreamPassInterface.c
+ * @author Ambroz Bizjak <[email protected]>
+ * 
+ * @section LICENSE
+ * 
+ * This file is part of BadVPN.
+ * 
+ * BadVPN is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ * 
+ * BadVPN is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <flow/StreamPassInterface.h>
+
+void _StreamPassInterface_job_operation (StreamPassInterface *i)
+{
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len > 0)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_done);
+    #endif
+    
+    // call operation handler
+    #ifndef NDEBUG
+    DebugIn_GoIn(&i->d_in_operation);
+    DEAD_ENTER(i->d_dead)
+    #endif
+    i->handler_operation(i->user_provider, i->buf, i->buf_len);
+    #ifndef NDEBUG
+    if (DEAD_LEAVE(i->d_dead)) {
+        return;
+    }
+    DebugIn_GoOut(&i->d_in_operation);
+    #endif
+}
+
+void _StreamPassInterface_job_done (StreamPassInterface *i)
+{
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len > 0)
+    ASSERT(i->done_len > 0)
+    ASSERT(i->done_len <= i->buf_len)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_done);
+    #endif
+    
+    #ifndef NDEBUG
+    i->d_user_busy = 0;
+    #endif
+    
+    // call done handler
+    #ifndef NDEBUG
+    DebugIn_GoIn(&i->d_in_done);
+    DEAD_ENTER(i->d_dead)
+    #endif
+    i->handler_done(i->user_user, i->done_len);
+    #ifndef NDEBUG
+    if (DEAD_LEAVE(i->d_dead)) {
+        return;
+    }
+    DebugIn_GoOut(&i->d_in_done);
+    #endif
+}

flow/StreamPassInterface.h

@@ -37,249 +37,161 @@
 
 #include <misc/dead.h>
 #include <misc/debug.h>
+#include <misc/debugin.h>
 #include <system/DebugObject.h>
+#include <system/BPending.h>
 
-/**
- * Handler called at the receiver when {@link StreamPassInterface_Sender_Send} is called
- * from the sender.
- * It is guaranteed that the interface is in not sending state.
- * It is guaranteed that the handler is not being called from within the Send handler.
- *
- * @param user value supplied to {@link StreamPassInterface_Init}
- * @param data pointer to data being sent
- * @param data_len amount of data being sent. Will be >0.
- * @return - >0 if the receiver accepts some data immediately, indicating how much of the
- *           data was accepted. The interface remains in not sending state. The receiver
- *           may not use the provided data after the handler returns.
- *         - 0 if the receiver cannot accept any data immediately. The interface enters
- *           sending state as the handler returns. The receiver may use the provided data
- *           as long as it needs to. When it's done processing some data and doesn't need
- *           the data any more, it must call {@link StreamPassInterface_Done}.
- */
-typedef int (*StreamPassInterface_handler_send) (void *user, uint8_t *data, int data_len);
+typedef void (*StreamPassInterface_handler_send) (void *user, uint8_t *data, int data_len);
 
-/**
- * Handler called at the sender when {@link StreamPassInterface_Done} is called from the receiver.
- * The receiver will no longer use the data it was provided with.
- * It is guaranteed that the interface was in sending state.
- * The interface enters not sending state before the handler is called.
- * It is guaranteed that the handler is not being called from within Send or Done handlers.
- *
- * @param user value supplied to {@link StreamPassInterface_Sender_Init}
- * @param data_len amount of data the receiver processed. Will be >0 and not exceed the amount
- *                 of data submitted in {@link StreamPassInterface_Sender_Send}.
- */
 typedef void (*StreamPassInterface_handler_done) (void *user, int data_len);
 
-/**
- * Interface allowing a stream sender to pass stream data to a stream receiver.
- * The sender passes some data by providing the receiver with a pointer
- * to the data. The receiver may then either accept some of the data immediately,
- * or tell the sender to wait for some data to be processed and inform it
- * when it's done.
- */
-typedef struct
-{
-    DebugObject d_obj;
-    
-    // receiver data
-    StreamPassInterface_handler_send handler_send;
-    void *user_receiver;
+typedef struct {
+    // provider data
+    StreamPassInterface_handler_send handler_operation;
+    void *user_provider;
     
-    // sender data
+    // user data
     StreamPassInterface_handler_done handler_done;
-    void *user_sender;
+    void *user_user;
+    
+    // jobs
+    BPending job_operation;
+    BPending job_done;
+    
+    // packet supplied by user
+    uint8_t *buf;
+    int buf_len;
+    
+    // length supplied by done
+    int done_len;
     
-    // debug vars
+    DebugObject d_obj;
     #ifndef NDEBUG
-    dead_t debug_dead;
-    int debug_busy_len;
-    int debug_in_send;
-    int debug_in_done;
+    DebugIn d_in_operation;
+    DebugIn d_in_done;
+    dead_t d_dead;
+    int d_user_busy;
     #endif
 } StreamPassInterface;
 
-/**
- * Initializes the interface. The sender portion must also be initialized
- * with {@link StreamPassInterface_Sender_Init} before I/O can start.
- * The interface is initialized in not sending state.
- *
- * @param i the object
- * @param handler_send handler called when the sender wants to send some data
- * @param user arbitrary value that will be passed to receiver callback functions
- */
-static void StreamPassInterface_Init (StreamPassInterface *i, StreamPassInterface_handler_send handler_send, void *user);
+static void StreamPassInterface_Init (StreamPassInterface *i, StreamPassInterface_handler_send handler_operation, void *user, BPendingGroup *pg);
 
-/**
- * Frees the interface.
- *
- * @param i the object
- */
 static void StreamPassInterface_Free (StreamPassInterface *i);
 
-/**
- * Notifies the sender that the receiver has finished processing some of the data being sent.
- * The receiver must not use the data it was provided any more.
- * The interface must be in sending state.
- * The interface enters not sending state before notifying the sender.
- * Must not be called from within Send or Done handlers.
- *
- * Be aware that the sender may attempt to send data from within this function.
- *
- * @param i the object
- * @param data_len amount of data processed. Must be >0 and not exceed the amount of data
- *                 the receiver was provided with in {@link StreamPassInterface_handler_send}.
- */
 static void StreamPassInterface_Done (StreamPassInterface *i, int data_len);
 
-/**
- * Initializes the sender portion of the interface.
- *
- * @param i the object
- * @param handler_done handler called when the receiver has finished processing some data
- * @param user arbitrary value that will be passed to sender callback functions
- */
 static void StreamPassInterface_Sender_Init (StreamPassInterface *i, StreamPassInterface_handler_done handler_done, void *user);
 
-/**
- * Attempts to send some data.
- * The interface must be in not sending state.
- * Must not be called from within the Send handler.
- *
- * @param i the object
- * @param data pointer to data to send
- * @param data_len amount of data to send. Must be >0.
- * @return - >0 if some data was accepted by the receiver immediately, indicating how much of
- *           the data was accepted. The data is no longer needed.
- *           The interface remains in not sending state.
- *         - 0 if no data could not be accepted immediately and is being processed.
- *           The interface enters sending state, and the data must stay accessible while the
- *           receiver is processing it. When the receiver is done processing it, the
- *           {@link StreamPassInterface_handler_done} handler will be called.
- */
-static int StreamPassInterface_Sender_Send (StreamPassInterface *i, uint8_t *data, int data_len);
+static void StreamPassInterface_Sender_Send (StreamPassInterface *i, uint8_t *data, int data_len);
 
 #ifndef NDEBUG
 
-/**
- * Determines if we are in a Send call.
- * Only available if NDEBUG is not defined.
- * 
- * @param i the object
- * @return 1 if in a Send call, 0 if not
- */
 static int StreamPassInterface_InClient (StreamPassInterface *i);
 
-/**
- * Determines if we are in a Done call.
- * Only available if NDEBUG is not defined.
- * 
- * @param i the object
- * @return 1 if in a Done call, 0 if not
- */
 static int StreamPassInterface_InDone (StreamPassInterface *i);
 
 #endif
 
-void StreamPassInterface_Init (StreamPassInterface *i, StreamPassInterface_handler_send handler_send, void *user)
+void _StreamPassInterface_job_operation (StreamPassInterface *i);
+void _StreamPassInterface_job_done (StreamPassInterface *i);
+
+void StreamPassInterface_Init (StreamPassInterface *i, StreamPassInterface_handler_send handler_operation, void *user, BPendingGroup *pg)
 {
-    i->handler_send = handler_send;
-    i->user_receiver = user;
+    // init arguments
+    i->handler_operation = handler_operation;
+    i->user_provider = user;
+    
+    // set no user
     i->handler_done = NULL;
-    i->user_sender = NULL;
     
-    // init debugging
-    #ifndef NDEBUG
-    DEAD_INIT(i->debug_dead);
-    i->debug_busy_len = -1;
-    i->debug_in_send = 0;
-    i->debug_in_done = 0;
-    #endif
+    // init jobs
+    BPending_Init(&i->job_operation, pg, (BPending_handler)_StreamPassInterface_job_operation, i);
+    BPending_Init(&i->job_done, pg, (BPending_handler)_StreamPassInterface_job_done, i);
     
-    // init debug object
     DebugObject_Init(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_Init(&i->d_in_operation);
+    DebugIn_Init(&i->d_in_done);
+    DEAD_INIT(i->d_dead);
+    i->d_user_busy = 0;
+    #endif
 }
 
 void StreamPassInterface_Free (StreamPassInterface *i)
 {
-    // free debug object
-    DebugObject_Free(&i->d_obj);
-    
-    // free debugging
     #ifndef NDEBUG
-    DEAD_KILL(i->debug_dead);
+    DEAD_KILL(i->d_dead);
     #endif
+    DebugObject_Free(&i->d_obj);
+    
+    // free jobs
+    BPending_Free(&i->job_done);
+    BPending_Free(&i->job_operation);
 }
 
 void StreamPassInterface_Done (StreamPassInterface *i, int data_len)
 {
-    ASSERT(i->debug_busy_len > 0)
-    ASSERT(!i->debug_in_send)
-    ASSERT(!i->debug_in_done)
     ASSERT(data_len > 0)
-    ASSERT(data_len <= i->debug_busy_len)
-    
+    ASSERT(data_len <= i->buf_len)
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len > 0)
+    ASSERT(i->handler_done)
+    ASSERT(!BPending_IsSet(&i->job_operation))
+    DebugObject_Access(&i->d_obj);
     #ifndef NDEBUG
-    i->debug_busy_len = -1;
-    i->debug_in_done = 1;
-    DEAD_ENTER(i->debug_dead)
+    DebugIn_AmOut(&i->d_in_done);
     #endif
     
-    i->handler_done(i->user_sender, data_len);
+    i->done_len = data_len;
     
-    #ifndef NDEBUG
-    if (DEAD_LEAVE(i->debug_dead)) {
-        return;
-    }
-    i->debug_in_done = 0;
-    #endif
+    BPending_Set(&i->job_done);
 }
 
 void StreamPassInterface_Sender_Init (StreamPassInterface *i, StreamPassInterface_handler_done handler_done, void *user)
 {
+    ASSERT(handler_done)
+    ASSERT(!i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    
     i->handler_done = handler_done;
-    i->user_sender = user;
+    i->user_user = user;
 }
 
-int StreamPassInterface_Sender_Send (StreamPassInterface *i, uint8_t *data, int data_len)
+void StreamPassInterface_Sender_Send (StreamPassInterface *i, uint8_t *data, int data_len)
 {
-    ASSERT(i->debug_busy_len == -1)
-    ASSERT(!i->debug_in_send)
     ASSERT(data_len > 0)
     ASSERT(data)
-    
+    ASSERT(!i->d_user_busy)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
     #ifndef NDEBUG
-    i->debug_in_send = 1;
-    DEAD_ENTER(i->debug_dead)
+    DebugIn_AmOut(&i->d_in_operation);
     #endif
     
-    int res = i->handler_send(i->user_receiver, data, data_len);
+    i->buf = data;
+    i->buf_len = data_len;
     
     #ifndef NDEBUG
-    if (DEAD_LEAVE(i->debug_dead)) {
-        return -1;
-    }
-    i->debug_in_send = 0;
-    ASSERT(res >= 0)
-    ASSERT(res <= data_len)
-    if (res == 0) {
-        i->debug_busy_len = data_len;
-    }
+    i->d_user_busy = 1;
     #endif
-
-    return res;
+    
+    BPending_Set(&i->job_operation);
 }
 
 #ifndef NDEBUG
 
 int StreamPassInterface_InClient (StreamPassInterface *i)
 {
-    return i->debug_in_send;
+    DebugObject_Access(&i->d_obj);
+    
+    return DebugIn_In(&i->d_in_operation);
 }
 
 int StreamPassInterface_InDone (StreamPassInterface *i)
 {
-    return i->debug_in_done;
+    DebugObject_Access(&i->d_obj);
+    
+    return DebugIn_In(&i->d_in_done);
 }
 
 #endif

flow/StreamRecvConnector.c

@@ -26,121 +26,45 @@
 
 #include <flow/StreamRecvConnector.h>
 
-static int output_handler_recv (StreamRecvConnector *o, uint8_t *data, int data_avail)
+static void output_handler_recv (StreamRecvConnector *o, uint8_t *data, int data_avail)
 {
+    ASSERT(data_avail > 0)
     ASSERT(o->out_avail == -1)
     ASSERT(!o->input || !o->in_blocking)
-    ASSERT(data_avail > 0)
-    
-    // if we have no input, remember output data
-    if (!o->input) {
-        o->out_avail = data_avail;
-        o->out = data;
-        return 0;
-    }
-    
-    // try to receive data
-    int res;
-    while (1) {
-        DEAD_ENTER_N(obj, o->dead)
-        DEAD_ENTER_N(inp, o->input_dead)
-        res = StreamRecvInterface_Receiver_Recv(o->input, data, data_avail);
-        DEAD_LEAVE_N(obj, o->dead);
-        DEAD_LEAVE_N(inp, o->input_dead);
-        if (DEAD_KILLED_N(obj)) {
-            return -1;
-        }
-        if (DEAD_KILLED_N(inp)) {
-            if (!o->input) {
-                // lost input
-                o->out_avail = data_avail;
-                o->out = data;
-                return 0;
-            }
-            // got a new input, retry
-            continue;
-        }
-        break;
-    };
     
-    ASSERT(res >= 0)
-    ASSERT(res <= data_avail)
+    // remember output packet
+    o->out_avail = data_avail;
+    o->out = data;
     
-    if (res == 0) {
-        // input blocking
-        o->out_avail = data_avail;
-        o->out = data;
+    if (o->input) {
+        // schedule receive
+        StreamRecvInterface_Receiver_Recv(o->input, o->out, o->out_avail);
         o->in_blocking = 1;
-        return 0;
     }
-    
-    return res;
 }
 
 static void input_handler_done (StreamRecvConnector *o, int data_len)
 {
+    ASSERT(data_len > 0)
+    ASSERT(data_len <= o->out_avail)
     ASSERT(o->out_avail > 0)
     ASSERT(o->input)
     ASSERT(o->in_blocking)
-    ASSERT(data_len > 0)
-    ASSERT(data_len <= o->out_avail)
-    
-    // have no output packet
-    o->out_avail = -1;
     
     // input not blocking any more
     o->in_blocking = 0;
     
-    // allow output to receive more packets
-    StreamRecvInterface_Done(&o->output, data_len);
-    return;
-}
-
-static void job_handler (StreamRecvConnector *o)
-{
-    ASSERT(o->input)
-    ASSERT(!o->in_blocking)
-    ASSERT(o->out_avail > 0)
-    
-    // try to receive data
-    DEAD_ENTER_N(obj, o->dead)
-    DEAD_ENTER_N(inp, o->input_dead)
-    int res = StreamRecvInterface_Receiver_Recv(o->input, o->out, o->out_avail);
-    DEAD_LEAVE_N(obj, o->dead);
-    DEAD_LEAVE_N(inp, o->input_dead);
-    if (DEAD_KILLED_N(obj)) {
-        return;
-    }
-    if (DEAD_KILLED_N(inp)) {
-        // lost current input. Do nothing here.
-        // If we gained a new one, its own job is responsible for it.
-        return;
-    }
-    
-    ASSERT(res >= 0)
-    ASSERT(res <= o->out_avail)
-    
-    if (res == 0) {
-        // input blocking
-        o->in_blocking = 1;
-        return;
-    }
-    
     // have no output packet
     o->out_avail = -1;
     
-    // allow output to receive more data
-    StreamRecvInterface_Done(&o->output, res);
-    return;
+    // allow output to receive more packets
+    StreamRecvInterface_Done(&o->output, data_len);
 }
 
 void StreamRecvConnector_Init (StreamRecvConnector *o, BPendingGroup *pg)
 {
-    // init dead var
-    DEAD_INIT(o->dead);
-    
     // init output
-    StreamRecvInterface_Init(&o->output, (StreamRecvInterface_handler_recv)output_handler_recv, o);
+    StreamRecvInterface_Init(&o->output, (StreamRecvInterface_handler_recv)output_handler_recv, o, pg);
     
     // have no output packet
     o->out_avail = -1;
@@ -148,41 +72,28 @@ void StreamRecvConnector_Init (StreamRecvConnector *o, BPendingGroup *pg)
     // have no input
     o->input = NULL;
     
-    // init continue job
-    BPending_Init(&o->continue_job, pg, (BPending_handler)job_handler, o);
-    
-    // init debug object
     DebugObject_Init(&o->d_obj);
 }
 
 void StreamRecvConnector_Free (StreamRecvConnector *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
     
-    // free continue job
-    BPending_Free(&o->continue_job);
-    
-    // free input dead var
-    if (o->input) {
-        DEAD_KILL(o->input_dead);
-    }
-    
     // free output
     StreamRecvInterface_Free(&o->output);
-    
-    // free dead var
-    DEAD_KILL(o->dead);
 }
 
 StreamRecvInterface * StreamRecvConnector_GetOutput (StreamRecvConnector *o)
 {
+    DebugObject_Access(&o->d_obj);
+    
     return &o->output;
 }
 
 void StreamRecvConnector_ConnectInput (StreamRecvConnector *o, StreamRecvInterface *input)
 {
     ASSERT(!o->input)
+    DebugObject_Access(&o->d_obj);
     
     // set input
     o->input = input;
@@ -190,27 +101,20 @@ void StreamRecvConnector_ConnectInput (StreamRecvConnector *o, StreamRecvInterfa
     // init input
     StreamRecvInterface_Receiver_Init(o->input, (StreamRecvInterface_handler_done)input_handler_done, o);
     
-    // init input dead var
-    DEAD_INIT(o->input_dead);
-    
     // set input not blocking
     o->in_blocking = 0;
     
-    // if we have an input packet, set continue job
+    // if we have an output packet, schedule receive
     if (o->out_avail > 0) {
-        BPending_Set(&o->continue_job);
+        StreamRecvInterface_Receiver_Recv(o->input, o->out, o->out_avail);
+        o->in_blocking = 1;
     }
 }
 
 void StreamRecvConnector_DisconnectInput (StreamRecvConnector *o)
 {
     ASSERT(o->input)
-    
-    // unset continue job (in case it wasn't called yet)
-    BPending_Unset(&o->continue_job);
-    
-    // free dead var
-    DEAD_KILL(o->input_dead);
+    DebugObject_Access(&o->d_obj);
     
     // set no input
     o->input = NULL;

flow/StreamRecvConnector.h

@@ -30,9 +30,7 @@
 
 #include <stdint.h>
 
-#include <misc/dead.h>
 #include <system/DebugObject.h>
-#include <system/BPending.h>
 #include <flow/StreamRecvInterface.h>
 
 /**
@@ -40,15 +38,13 @@
  * connected and disconnected on the fly.
  */
 typedef struct {
-    DebugObject d_obj;
-    dead_t dead;
     StreamRecvInterface output;
     int out_avail;
     uint8_t *out;
     StreamRecvInterface *input;
     dead_t input_dead;
     int in_blocking;
-    BPending continue_job;
+    DebugObject d_obj;
 } StreamRecvConnector;
 
 /**

flow/StreamRecvInterface.c

@@ -0,0 +1,79 @@
+/**
+ * @file StreamRecvInterface.c
+ * @author Ambroz Bizjak <[email protected]>
+ * 
+ * @section LICENSE
+ * 
+ * This file is part of BadVPN.
+ * 
+ * BadVPN is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ * 
+ * BadVPN is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <flow/StreamRecvInterface.h>
+
+void _StreamRecvInterface_job_operation (StreamRecvInterface *i)
+{
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len > 0)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_done);
+    #endif
+    
+    // call operation handler
+    #ifndef NDEBUG
+    DebugIn_GoIn(&i->d_in_operation);
+    DEAD_ENTER(i->d_dead)
+    #endif
+    i->handler_operation(i->user_provider, i->buf, i->buf_len);
+    #ifndef NDEBUG
+    if (DEAD_LEAVE(i->d_dead)) {
+        return;
+    }
+    DebugIn_GoOut(&i->d_in_operation);
+    #endif
+}
+
+void _StreamRecvInterface_job_done (StreamRecvInterface *i)
+{
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len > 0)
+    ASSERT(i->done_len > 0)
+    ASSERT(i->done_len <= i->buf_len)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_AmOut(&i->d_in_operation);
+    DebugIn_AmOut(&i->d_in_done);
+    #endif
+    
+    #ifndef NDEBUG
+    i->d_user_busy = 0;
+    #endif
+    
+    // call done handler
+    #ifndef NDEBUG
+    DebugIn_GoIn(&i->d_in_done);
+    DEAD_ENTER(i->d_dead)
+    #endif
+    i->handler_done(i->user_user, i->done_len);
+    #ifndef NDEBUG
+    if (DEAD_LEAVE(i->d_dead)) {
+        return;
+    }
+    DebugIn_GoOut(&i->d_in_done);
+    #endif
+}

flow/StreamRecvInterface.h

@@ -24,8 +24,8 @@
  * Interface allowing a stream receiver to receive stream data from a stream sender.
  * 
  * Note that this interface behaves exactly the same and has the same code as
- * {@link StreamPassInterface} if names and its external semantics are disregarded.
- * If you modify this file, you should probably modify {@link StreamPassInterface}
+ * {@link StreamRecvInterface} if names and its external semantics are disregarded.
+ * If you modify this file, you should probably modify {@link StreamRecvInterface}
  * too.
  */
 
@@ -37,248 +37,161 @@
 
 #include <misc/dead.h>
 #include <misc/debug.h>
+#include <misc/debugin.h>
 #include <system/DebugObject.h>
+#include <system/BPending.h>
 
-/**
- * Handler called at the sender when {@link StreamRecvInterface_Receiver_Recv} is called
- * from the receiver.
- * It is guaranteed that the interface is in not receiving state.
- * It is guaranteed that the handler is not being called from within Recv or Cancel handlers.
- *
- * @param user value supplied to {@link StreamRecvInterface_Init}
- * @param data pointer to the buffer where data is to be written
- * @param data_avail size of the buffer. Will be >0.
- * @return - >0 if the sender provides some data immediately, indicating how much data was
- *           written to the buffer. The interface remains in not receiving state.
- *           The sender may not use the provided buffer after the handler returns.
- *         - 0 if the sender cannot provide any data immediately. The interface enters
- *           receiving state as the handler returns. The sender must write some data to the
- *           provided buffer and call {@link StreamRecvInterface_Done} when it's done.
- */
-typedef int (*StreamRecvInterface_handler_recv) (void *user, uint8_t *data, int data_avail);
+typedef void (*StreamRecvInterface_handler_recv) (void *user, uint8_t *data, int data_len);
 
-/**
- * Handler called at the receiver when {@link StreamRecvInterface_Done} is called from the sender.
- * The sender will no longer use the buffer it was provided with.
- * It is guaranteed that the interface was in receiving state.
- * The interface enters not receiving state before the handler is called.
- * It is guaranteed that the handler is not being called from within Recv or Done handlers.
- *
- * @param user value supplied to {@link StreamRecvInterface_Receiver_Init}
- * @param data_len number of bytes written. Will be >0 and <= the size of the buffer
- *                 that was provided in the previous {@link StreamRecvInterface_Receiver_Recv}
- *                 call.
- */
 typedef void (*StreamRecvInterface_handler_done) (void *user, int data_len);
 
-/**
- * Interface allowing a stream receiver to receive stream data from a stream sender.
- * The receiver receives data by providing the sender with a buffer. The sender
- * may then either provide some data immediately, or tell the receiver to wait for
- * some data to be available and inform it when it's done.
- */
 typedef struct {
-    DebugObject d_obj;
+    // provider data
+    StreamRecvInterface_handler_recv handler_operation;
+    void *user_provider;
     
-    // sender data
-    StreamRecvInterface_handler_recv handler_recv;
-    void *user_sender;
-
-    // receiver data
+    // user data
     StreamRecvInterface_handler_done handler_done;
-    void *user_receiver;
+    void *user_user;
+    
+    // jobs
+    BPending job_operation;
+    BPending job_done;
+    
+    // packet supplied by user
+    uint8_t *buf;
+    int buf_len;
+    
+    // length supplied by done
+    int done_len;
     
-    // debug vars
+    DebugObject d_obj;
     #ifndef NDEBUG
-    dead_t debug_dead;
-    int debug_busy_len;
-    int debug_in_recv;
-    int debug_in_done;
+    DebugIn d_in_operation;
+    DebugIn d_in_done;
+    dead_t d_dead;
+    int d_user_busy;
     #endif
 } StreamRecvInterface;
 
-/**
- * Initializes the interface. The receiver portion must also be initialized
- * with {@link StreamRecvInterface_Receiver_Init} before I/O can start.
- * The interface is initialized in not receiving state.
- *
- * @param i the object
- * @param handler_recv handler called when the receiver wants to receive data
- * @param user arbitrary value that will be passed to sender callback functions
- */
-static void StreamRecvInterface_Init (StreamRecvInterface *i, StreamRecvInterface_handler_recv handler_recv, void *user);
+static void StreamRecvInterface_Init (StreamRecvInterface *i, StreamRecvInterface_handler_recv handler_operation, void *user, BPendingGroup *pg);
 
-/**
- * Frees the interface.
- *
- * @param i the object
- */
 static void StreamRecvInterface_Free (StreamRecvInterface *i);
 
-/**
- * Notifies the receiver that the sender has finished providing some data.
- * The sender must not use the buffer it was provided any more.
- * The interface must be in receiving state.
- * The interface enters not receiving state before notifying the receiver.
- * Must not be called from within Recv or Done handlers.
- *
- * Be aware that the receiver may attempt to receive data from within this function.
- *
- * @param i the object
- * @param data_len number of bytes written. Must be >0 and <= the size of the buffer
- *                 that was provided in the previous {@link StreamRecvInterface_handler_recv}
- *                 call.
- */
 static void StreamRecvInterface_Done (StreamRecvInterface *i, int data_len);
 
-/**
- * Initializes the receiver portion of the interface.
- *
- * @param i the object
- * @param handler_done handler called when the sender has finished providing a packet
- * @param user arbitrary value that will be passed to receiver callback functions
- */
 static void StreamRecvInterface_Receiver_Init (StreamRecvInterface *i, StreamRecvInterface_handler_done handler_done, void *user);
 
-/**
- * Attempts to receive some data.
- * The interface must be in not receiving state.
- * Must not be called from within the Recv handler.
- *
- * @param i the object
- * @param data pointer to the buffer where data is to be written
- * @param data_avail size of the buffer. Must be >0.
- * @return - >0 if some data was provided by the sender imediately, indicating how much data was
- *           provided. The buffer is no longer needed.
- *           The interface remains in not receiving state.
- *         - 0 if no data could not be provided immediately.
- *           The interface enters receiving state, and the buffer must stay accessible while the
- *           sender is providing the data. When the sender is done providing it, the
- *           {@link StreamRecvInterface_handler_done} handler will be called.
- */
-static int StreamRecvInterface_Receiver_Recv (StreamRecvInterface *i, uint8_t *data, int data_avail);
+static void StreamRecvInterface_Receiver_Recv (StreamRecvInterface *i, uint8_t *data, int data_len);
 
 #ifndef NDEBUG
 
-/**
- * Determines if we are in a Recv call.
- * Only available if NDEBUG is not defined.
- * 
- * @param i the object
- * @return 1 if in a Recv call, 0 if not
- */
 static int StreamRecvInterface_InClient (StreamRecvInterface *i);
 
-/**
- * Determines if we are in a Done call.
- * Only available if NDEBUG is not defined.
- * 
- * @param i the object
- * @return 1 if in a Done call, 0 if not
- */
 static int StreamRecvInterface_InDone (StreamRecvInterface *i);
 
 #endif
 
-void StreamRecvInterface_Init (StreamRecvInterface *i, StreamRecvInterface_handler_recv handler_recv, void *user)
+void _StreamRecvInterface_job_operation (StreamRecvInterface *i);
+void _StreamRecvInterface_job_done (StreamRecvInterface *i);
+
+void StreamRecvInterface_Init (StreamRecvInterface *i, StreamRecvInterface_handler_recv handler_operation, void *user, BPendingGroup *pg)
 {
-    i->handler_recv = handler_recv;
-    i->user_sender = user;
+    // init arguments
+    i->handler_operation = handler_operation;
+    i->user_provider = user;
+    
+    // set no user
     i->handler_done = NULL;
-    i->user_receiver = NULL;
     
-    // init debugging
-    #ifndef NDEBUG
-    DEAD_INIT(i->debug_dead);
-    i->debug_busy_len = -1;
-    i->debug_in_recv = 0;
-    i->debug_in_done = 0;
-    #endif
+    // init jobs
+    BPending_Init(&i->job_operation, pg, (BPending_handler)_StreamRecvInterface_job_operation, i);
+    BPending_Init(&i->job_done, pg, (BPending_handler)_StreamRecvInterface_job_done, i);
     
-    // init debug object
     DebugObject_Init(&i->d_obj);
+    #ifndef NDEBUG
+    DebugIn_Init(&i->d_in_operation);
+    DebugIn_Init(&i->d_in_done);
+    DEAD_INIT(i->d_dead);
+    i->d_user_busy = 0;
+    #endif
 }
 
 void StreamRecvInterface_Free (StreamRecvInterface *i)
 {
-    // free debug object
-    DebugObject_Free(&i->d_obj);
-    
-    // free debugging
     #ifndef NDEBUG
-    DEAD_KILL(i->debug_dead);
+    DEAD_KILL(i->d_dead);
     #endif
+    DebugObject_Free(&i->d_obj);
+    
+    // free jobs
+    BPending_Free(&i->job_done);
+    BPending_Free(&i->job_operation);
 }
 
 void StreamRecvInterface_Done (StreamRecvInterface *i, int data_len)
 {
-    ASSERT(i->debug_busy_len > 0)
-    ASSERT(!i->debug_in_recv)
-    ASSERT(!i->debug_in_done)
     ASSERT(data_len > 0)
-    ASSERT(data_len <= i->debug_busy_len)
-    
+    ASSERT(data_len <= i->buf_len)
+    ASSERT(i->d_user_busy)
+    ASSERT(i->buf_len > 0)
+    ASSERT(i->handler_done)
+    ASSERT(!BPending_IsSet(&i->job_operation))
+    DebugObject_Access(&i->d_obj);
     #ifndef NDEBUG
-    i->debug_busy_len = -1;
-    i->debug_in_done = 1;
-    DEAD_ENTER(i->debug_dead)
+    DebugIn_AmOut(&i->d_in_done);
     #endif
     
-    i->handler_done(i->user_receiver, data_len);
+    i->done_len = data_len;
     
-    #ifndef NDEBUG
-    if (DEAD_LEAVE(i->debug_dead)) {
-        return;
-    }
-    i->debug_in_done = 0;
-    #endif
+    BPending_Set(&i->job_done);
 }
 
 void StreamRecvInterface_Receiver_Init (StreamRecvInterface *i, StreamRecvInterface_handler_done handler_done, void *user)
 {
+    ASSERT(handler_done)
+    ASSERT(!i->handler_done)
+    DebugObject_Access(&i->d_obj);
+    
     i->handler_done = handler_done;
-    i->user_receiver = user;
+    i->user_user = user;
 }
 
-int StreamRecvInterface_Receiver_Recv (StreamRecvInterface *i, uint8_t *data, int data_avail)
+void StreamRecvInterface_Receiver_Recv (StreamRecvInterface *i, uint8_t *data, int data_len)
 {
-    ASSERT(i->debug_busy_len == -1)
-    ASSERT(!i->debug_in_recv)
-    ASSERT(data_avail > 0)
+    ASSERT(data_len > 0)
     ASSERT(data)
-    
+    ASSERT(!i->d_user_busy)
+    ASSERT(i->handler_done)
+    DebugObject_Access(&i->d_obj);
     #ifndef NDEBUG
-    i->debug_in_recv = 1;
-    DEAD_ENTER(i->debug_dead)
+    DebugIn_AmOut(&i->d_in_operation);
     #endif
     
-    int res = i->handler_recv(i->user_sender, data, data_avail);
+    i->buf = data;
+    i->buf_len = data_len;
     
     #ifndef NDEBUG
-    if (DEAD_LEAVE(i->debug_dead)) {
-        return -1;
-    }
-    i->debug_in_recv = 0;
-    ASSERT(res >= 0)
-    ASSERT(res <= data_avail)
-    if (res == 0) {
-        i->debug_busy_len = data_avail;
-    }
+    i->d_user_busy = 1;
     #endif
     
-    return res;
+    BPending_Set(&i->job_operation);
 }
 
 #ifndef NDEBUG
 
 int StreamRecvInterface_InClient (StreamRecvInterface *i)
 {
-    return i->debug_in_recv;
+    DebugObject_Access(&i->d_obj);
+    
+    return DebugIn_In(&i->d_in_operation);
 }
 
 int StreamRecvInterface_InDone (StreamRecvInterface *i)
 {
-    return i->debug_in_done;
+    DebugObject_Access(&i->d_obj);
+    
+    return DebugIn_In(&i->d_in_done);
 }
 
 #endif

flow/StreamSocketSink.c

@@ -26,69 +26,63 @@
 
 static void report_error (StreamSocketSink *s, int error)
 {
-    #ifndef NDEBUG
-    s->in_error = 1;
     DEAD_ENTER(s->dead)
-    #endif
-    
     FlowErrorReporter_ReportError(&s->rep, &error);
-    
-    #ifndef NDEBUG
     ASSERT(DEAD_KILLED)
     DEAD_LEAVE(s->dead);
-    #endif
 }
 
-static int input_handler_send (StreamSocketSink *s, uint8_t *data, int data_len)
+static void try_send (StreamSocketSink *s)
 {
-    ASSERT(s->in_len == -1)
-    ASSERT(data_len > 0)
-    ASSERT(!s->in_error)
+    ASSERT(s->in_len > 0)
+    
+    int res = BSocket_Send(s->bsock, s->in, s->in_len);
+    if (res < 0 && BSocket_GetError(s->bsock) == BSOCKET_ERROR_LATER) {
+        // wait for socket in socket_handler
+        BSocket_EnableEvent(s->bsock, BSOCKET_WRITE);
+        return;
+    }
     
-    int res = BSocket_Send(s->bsock, data, data_len);
     if (res < 0) {
-        int error = BSocket_GetError(s->bsock);
-        if (error == BSOCKET_ERROR_LATER) {
-            s->in_len = data_len;
-            s->in = data;
-            BSocket_EnableEvent(s->bsock, BSOCKET_WRITE);
-            return 0;
-        }
         report_error(s, STREAMSOCKETSINK_ERROR_BSOCKET);
-        return -1;
+        return;
     }
     
     ASSERT(res > 0)
+    ASSERT(res <= s->in_len)
+    
+    // finish packet
+    s->in_len = -1;
+    StreamPassInterface_Done(&s->input, res);
+}
+
+static void input_handler_send (StreamSocketSink *s, uint8_t *data, int data_len)
+{
+    ASSERT(data_len > 0)
+    ASSERT(s->in_len == -1)
+    DebugObject_Access(&s->d_obj);
     
-    return res;
+    // set packet
+    s->in_len = data_len;
+    s->in = data;
+    
+    try_send(s);
+    return;
 }
 
 static void socket_handler (StreamSocketSink *s, int event)
 {
     ASSERT(s->in_len > 0)
     ASSERT(event == BSOCKET_WRITE)
-    ASSERT(!s->in_error)
-    
-    int res = BSocket_Send(s->bsock, s->in, s->in_len);
-    if (res < 0) {
-        int error = BSocket_GetError(s->bsock);
-        if (error == BSOCKET_ERROR_LATER) {
-            return;
-        }
-        report_error(s, STREAMSOCKETSINK_ERROR_BSOCKET);
-        return;
-    }
-    
-    ASSERT(res > 0)
+    DebugObject_Access(&s->d_obj);
     
     BSocket_DisableEvent(s->bsock, BSOCKET_WRITE);
-    s->in_len = -1;
     
-    StreamPassInterface_Done(&s->input, res);
+    try_send(s);
     return;
 }
 
-void StreamSocketSink_Init (StreamSocketSink *s, FlowErrorReporter rep, BSocket *bsock)
+void StreamSocketSink_Init (StreamSocketSink *s, FlowErrorReporter rep, BSocket *bsock, BPendingGroup *pg)
 {
     // init arguments
     s->rep = rep;
@@ -101,23 +95,16 @@ void StreamSocketSink_Init (StreamSocketSink *s, FlowErrorReporter rep, BSocket
     BSocket_AddEventHandler(s->bsock, BSOCKET_WRITE, (BSocket_handler)socket_handler, s);
     
     // init input
-    StreamPassInterface_Init(&s->input, (StreamPassInterface_handler_send)input_handler_send, s);
+    StreamPassInterface_Init(&s->input, (StreamPassInterface_handler_send)input_handler_send, s, pg);
     
     // have no input packet
     s->in_len = -1;
     
-    // init debugging
-    #ifndef NDEBUG
-    s->in_error = 0;
-    #endif
-    
-    // init debug object
     DebugObject_Init(&s->d_obj);
 }
 
 void StreamSocketSink_Free (StreamSocketSink *s)
 {
-    // free debug object
     DebugObject_Free(&s->d_obj);
     
     // free input
@@ -132,5 +119,7 @@ void StreamSocketSink_Free (StreamSocketSink *s)
 
 StreamPassInterface * StreamSocketSink_GetInput (StreamSocketSink *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     return &s->input;
 }

flow/StreamSocketSink.h

@@ -41,16 +41,13 @@
  * A {@link StreamPassInterface} sink which sends data to a stream socket.
  */
 typedef struct {
-    DebugObject d_obj;
     dead_t dead;
     FlowErrorReporter rep;
     BSocket *bsock;
     StreamPassInterface input;
     int in_len;
     uint8_t *in;
-    #ifndef NDEBUG
-    int in_error;
-    #endif
+    DebugObject d_obj;
 } StreamSocketSink;
 
 /**
@@ -63,8 +60,9 @@ typedef struct {
  *            The object must be freed from the error handler.
  * @param bsock stream socket to write data to. Registers a BSOCKET_WRITE handler which
  *              must not be registered.
+ * @param pg pending group
  */
-void StreamSocketSink_Init (StreamSocketSink *s, FlowErrorReporter rep, BSocket *bsock);
+void StreamSocketSink_Init (StreamSocketSink *s, FlowErrorReporter rep, BSocket *bsock, BPendingGroup *pg);
 
 /**
  * Frees the sink.

flow/StreamSocketSource.c

@@ -20,83 +20,74 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
 
-#include <stdlib.h>
-
 #include <misc/debug.h>
 
 #include <flow/StreamSocketSource.h>
 
 static void report_error (StreamSocketSource *s, int error)
 {
-    #ifndef NDEBUG
-    s->in_error = 1;
     DEAD_ENTER(s->dead)
-    #endif
-    
     FlowErrorReporter_ReportError(&s->rep, &error);
-    
-    #ifndef NDEBUG
     ASSERT(DEAD_KILLED)
     DEAD_LEAVE(s->dead);
-    #endif
 }
 
-static int output_handler_recv (StreamSocketSource *s, uint8_t *data, int data_avail)
+static void try_recv (StreamSocketSource *s)
 {
-    ASSERT(s->out_avail == -1)
-    ASSERT(data_avail > 0)
-    ASSERT(!s->in_error)
+    ASSERT(s->out_avail > 0)
+    
+    int res = BSocket_Recv(s->bsock, s->out, s->out_avail);
+    if (res < 0 && BSocket_GetError(s->bsock) == BSOCKET_ERROR_LATER) {
+        // wait for socket in socket_handler
+        BSocket_EnableEvent(s->bsock, BSOCKET_READ);
+        return;
+    }
     
-    int res = BSocket_Recv(s->bsock, data, data_avail);
     if (res < 0) {
-        int error = BSocket_GetError(s->bsock);
-        if (error == BSOCKET_ERROR_LATER) {
-            s->out_avail = data_avail;
-            s->out = data;
-            BSocket_EnableEvent(s->bsock, BSOCKET_READ);
-            return 0;
-        }
         report_error(s, STREAMSOCKETSOURCE_ERROR_BSOCKET);
-        return -1;
+        return;
     }
     
     if (res == 0) {
         report_error(s, STREAMSOCKETSOURCE_ERROR_CLOSED);
-        return -1;
+        return;
     }
     
-    return res;
+    ASSERT(res > 0)
+    ASSERT(res <= s->out_avail)
+    
+    // finish packet
+    s->out_avail = -1;
+    StreamRecvInterface_Done(&s->output, res);
+}
+
+static void output_handler_recv (StreamSocketSource *s, uint8_t *data, int data_avail)
+{
+    ASSERT(data_avail > 0)
+    ASSERT(s->out_avail == -1)
+    DebugObject_Access(&s->d_obj);
+    
+    // set packet
+    s->out_avail = data_avail;
+    s->out = data;
+    
+    try_recv(s);
+    return;
 }
 
 static void socket_handler (StreamSocketSource *s, int event)
 {
     ASSERT(s->out_avail > 0)
     ASSERT(event == BSOCKET_READ)
-    ASSERT(!s->in_error)
-    
-    int res = BSocket_Recv(s->bsock, s->out, s->out_avail);
-    if (res < 0) {
-        int error = BSocket_GetError(s->bsock);
-        if (error == BSOCKET_ERROR_LATER) {
-            return;
-        }
-        report_error(s, STREAMSOCKETSOURCE_ERROR_BSOCKET);
-        return;
-    }
-    
-    if (res == 0) {
-        report_error(s, STREAMSOCKETSOURCE_ERROR_CLOSED);
-        return;
-    }
+    DebugObject_Access(&s->d_obj);
     
     BSocket_DisableEvent(s->bsock, BSOCKET_READ);
-    s->out_avail = -1;
     
-    StreamRecvInterface_Done(&s->output, res);
+    try_recv(s);
     return;
 }
 
-void StreamSocketSource_Init (StreamSocketSource *s, FlowErrorReporter rep, BSocket *bsock)
+void StreamSocketSource_Init (StreamSocketSource *s, FlowErrorReporter rep, BSocket *bsock, BPendingGroup *pg)
 {
     // init arguments
     s->rep = rep;
@@ -109,23 +100,16 @@ void StreamSocketSource_Init (StreamSocketSource *s, FlowErrorReporter rep, BSoc
     BSocket_AddEventHandler(s->bsock, BSOCKET_READ, (BSocket_handler)socket_handler, s);
     
     // init output
-    StreamRecvInterface_Init(&s->output, (StreamRecvInterface_handler_recv)output_handler_recv, s);
+    StreamRecvInterface_Init(&s->output, (StreamRecvInterface_handler_recv)output_handler_recv, s, pg);
     
     // have no output packet
     s->out_avail = -1;
     
-    // init debugging
-    #ifndef NDEBUG
-    s->in_error = 0;
-    #endif
-    
-    // init debug object
     DebugObject_Init(&s->d_obj);
 }
 
 void StreamSocketSource_Free (StreamSocketSource *s)
 {
-    // free debug object
     DebugObject_Free(&s->d_obj);
     
     // free output
@@ -140,5 +124,7 @@ void StreamSocketSource_Free (StreamSocketSource *s)
 
 StreamRecvInterface * StreamSocketSource_GetOutput (StreamSocketSource *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     return &s->output;
 }

flow/StreamSocketSource.h

@@ -27,11 +27,13 @@
 #ifndef BADVPN_FLOW_STREAMSOCKETSOURCE_H
 #define BADVPN_FLOW_STREAMSOCKETSOURCE_H
 
+#include <stdint.h>
+
 #include <misc/dead.h>
 #include <system/DebugObject.h>
 #include <system/BSocket.h>
-#include <flow/error.h>
 #include <flow/StreamRecvInterface.h>
+#include <flow/error.h>
 
 #define STREAMSOCKETSOURCE_ERROR_CLOSED 0
 #define STREAMSOCKETSOURCE_ERROR_BSOCKET 1
@@ -40,16 +42,13 @@
  * A {@link StreamRecvInterface} source which receives data from a stream socket.
  */
 typedef struct {
-    DebugObject d_obj;
     dead_t dead;
     FlowErrorReporter rep;
     BSocket *bsock;
     StreamRecvInterface output;
     int out_avail;
     uint8_t *out;
-    #ifndef NDEBUG
-    int in_error;
-    #endif
+    DebugObject d_obj;
 } StreamSocketSource;
 
 /**
@@ -63,8 +62,9 @@ typedef struct {
  *            The object must be freed from the error handler.
  * @param bsock stream socket to read data from. Registers a BSOCKET_READ handler which
  *              must not be registered.
+ * @param pg pending group
  */
-void StreamSocketSource_Init (StreamSocketSource *s, FlowErrorReporter rep, BSocket *bsock);
+void StreamSocketSource_Init (StreamSocketSource *s, FlowErrorReporter rep, BSocket *bsock, BPendingGroup *pg);
 
 /**
  * Frees the source.

ipc/BIPC.c

@@ -48,16 +48,16 @@ static int init_io (BIPC *o, int send_mtu, PacketPassInterface *recv_if, BReacto
     FlowErrorDomain_Init(&o->domain, (FlowErrorDomain_handler)error_handler, o);
     
     // init sending
-    StreamSocketSink_Init(&o->send_sink, FlowErrorReporter_Create(&o->domain, COMPONENT_SINK), &o->sock);
-    PacketStreamSender_Init(&o->send_pss, StreamSocketSink_GetInput(&o->send_sink), PACKETPROTO_ENCLEN(send_mtu));
-    PacketCopier_Init(&o->send_copier, send_mtu);
-    PacketProtoEncoder_Init(&o->send_encoder, PacketCopier_GetOutput(&o->send_copier));
+    StreamSocketSink_Init(&o->send_sink, FlowErrorReporter_Create(&o->domain, COMPONENT_SINK), &o->sock, BReactor_PendingGroup(reactor));
+    PacketStreamSender_Init(&o->send_pss, StreamSocketSink_GetInput(&o->send_sink), PACKETPROTO_ENCLEN(send_mtu), BReactor_PendingGroup(reactor));
+    PacketCopier_Init(&o->send_copier, send_mtu,  BReactor_PendingGroup(reactor));
+    PacketProtoEncoder_Init(&o->send_encoder, PacketCopier_GetOutput(&o->send_copier), BReactor_PendingGroup(reactor));
     if (!SinglePacketBuffer_Init(&o->send_buf, PacketProtoEncoder_GetOutput(&o->send_encoder), PacketStreamSender_GetInput(&o->send_pss), BReactor_PendingGroup(reactor))) {
         goto fail1;
     }
     
     // init receiving
-    StreamSocketSource_Init(&o->recv_source, FlowErrorReporter_Create(&o->domain, COMPONENT_SOURCE), &o->sock);
+    StreamSocketSource_Init(&o->recv_source, FlowErrorReporter_Create(&o->domain, COMPONENT_SOURCE), &o->sock, BReactor_PendingGroup(reactor));
     if (!PacketProtoDecoder_Init(&o->recv_decoder, FlowErrorReporter_Create(&o->domain, COMPONENT_DECODER), StreamSocketSource_GetOutput(&o->recv_source), recv_if, BReactor_PendingGroup(reactor))) {
         goto fail2;
     }

misc/version.h

@@ -28,7 +28,7 @@
 #define BADVPN_MISC_VERSION_H
 
 #define GLOBAL_PRODUCT_NAME "BadVPN"
-#define GLOBAL_VERSION "1.999.95"
+#define GLOBAL_VERSION "1.999.96rc7"
 #define GLOBAL_COPYRIGHT_NOTICE "Copyright (C) 2010 Ambroz Bizjak <[email protected]>"
 
 #endif

nspr_support/BSocketPRFileDesc.c

@@ -162,8 +162,6 @@ static PRStatus method_getpeername (PRFileDesc *fd, PRNetAddr *addr)
 
 static PRStatus method_getsocketoption (PRFileDesc *fd, PRSocketOptionData *data)
 {
-    BSocket *bsock = (BSocket *)fd->secret;
-    
     switch (data->option) {
         case PR_SockOpt_Nonblocking:
             data->value.non_blocking = PR_TRUE;

nspr_support/PRStreamSink.c

@@ -28,69 +28,61 @@
 
 static void report_error (PRStreamSink *s, int error)
 {
-    #ifndef NDEBUG
-    s->in_error = 1;
     DEAD_ENTER(s->dead)
-    #endif
-    
     FlowErrorReporter_ReportError(&s->rep, &error);
-    
-    #ifndef NDEBUG
     ASSERT(DEAD_KILLED)
     DEAD_LEAVE(s->dead);
-    #endif
 }
 
-static int input_handler_send (PRStreamSink *s, uint8_t *data, int data_len)
+static void try_send (PRStreamSink *s)
 {
-    ASSERT(s->in_len == -1)
-    ASSERT(data_len > 0)
-    ASSERT(!s->in_error)
+    ASSERT(s->in_len > 0)
+    
+    int res = PR_Write(s->bprfd->prfd, s->in, s->in_len);
+    if (res < 0 && PR_GetError() == PR_WOULD_BLOCK_ERROR) {
+        // wait for socket in prfd_handler
+        BPRFileDesc_EnableEvent(s->bprfd, PR_POLL_WRITE);
+        return;
+    }
     
-    int res = PR_Write(s->bprfd->prfd, data, data_len);
     if (res < 0) {
-        PRErrorCode error = PR_GetError();
-        if (error == PR_WOULD_BLOCK_ERROR) {
-            s->in_len = data_len;
-            s->in = data;
-            BPRFileDesc_EnableEvent(s->bprfd, PR_POLL_WRITE);
-            return 0;
-        }
         report_error(s, PRSTREAMSINK_ERROR_NSPR);
-        return -1;
+        return;
     }
     
     ASSERT(res > 0)
+    ASSERT(res <= s->in_len)
     
-    return res;
+    // finish packet
+    s->in_len = -1;
+    StreamPassInterface_Done(&s->input, res);
+}
+
+static void input_handler_send (PRStreamSink *s, uint8_t *data, int data_len)
+{
+    ASSERT(data_len > 0)
+    ASSERT(s->in_len == -1)
+    DebugObject_Access(&s->d_obj);
+    
+    // set packet
+    s->in_len = data_len;
+    s->in = data;
+    
+    try_send(s);
+    return;
 }
 
 static void prfd_handler (PRStreamSink *s, PRInt16 event)
 {
     ASSERT(s->in_len > 0)
     ASSERT(event == PR_POLL_WRITE)
-    ASSERT(!s->in_error)
-    
-    int res = PR_Write(s->bprfd->prfd, s->in, s->in_len);
-    if (res < 0) {
-        PRErrorCode error = PR_GetError();
-        if (error == PR_WOULD_BLOCK_ERROR) {
-            BPRFileDesc_EnableEvent(s->bprfd, PR_POLL_WRITE);
-            return;
-        }
-        report_error(s, PRSTREAMSINK_ERROR_NSPR);
-        return;
-    }
-    
-    ASSERT(res > 0)
-    
-    s->in_len = -1;
+    DebugObject_Access(&s->d_obj);
     
-    StreamPassInterface_Done(&s->input, res);
+    try_send(s);
     return;
 }
 
-void PRStreamSink_Init (PRStreamSink *s, FlowErrorReporter rep, BPRFileDesc *bprfd)
+void PRStreamSink_Init (PRStreamSink *s, FlowErrorReporter rep, BPRFileDesc *bprfd, BPendingGroup *pg)
 {
     // init arguments
     s->rep = rep;
@@ -103,23 +95,16 @@ void PRStreamSink_Init (PRStreamSink *s, FlowErrorReporter rep, BPRFileDesc *bpr
     BPRFileDesc_AddEventHandler(s->bprfd, PR_POLL_WRITE, (BPRFileDesc_handler)prfd_handler, s);
     
     // init input
-    StreamPassInterface_Init(&s->input, (StreamPassInterface_handler_send)input_handler_send, s);
+    StreamPassInterface_Init(&s->input, (StreamPassInterface_handler_send)input_handler_send, s, pg);
     
     // have no input packet
     s->in_len = -1;
     
-    // init debugging
-    #ifndef NDEBUG
-    s->in_error = 0;
-    #endif
-    
-    // init debug object
     DebugObject_Init(&s->d_obj);
 }
 
 void PRStreamSink_Free (PRStreamSink *s)
 {
-    // free debug object
     DebugObject_Free(&s->d_obj);
     
     // free input
@@ -134,5 +119,7 @@ void PRStreamSink_Free (PRStreamSink *s)
 
 StreamPassInterface * PRStreamSink_GetInput (PRStreamSink *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     return &s->input;
 }

nspr_support/PRStreamSink.h

@@ -31,8 +31,8 @@
 
 #include <misc/dead.h>
 #include <system/DebugObject.h>
-#include <flow/error.h>
 #include <flow/StreamPassInterface.h>
+#include <flow/error.h>
 #include <nspr_support/BPRFileDesc.h>
 
 #define PRSTREAMSINK_ERROR_NSPR 1
@@ -41,16 +41,13 @@
  * A {@link StreamPassInterface} sink for a NSPR file descriptor (PRFileDesc) via {@link BPRFileDesc}.
  */
 typedef struct {
-    DebugObject d_obj;
     dead_t dead;
     FlowErrorReporter rep;
     BPRFileDesc *bprfd;
     StreamPassInterface input;
     int in_len;
     uint8_t *in;
-    #ifndef NDEBUG
-    int in_error;
-    #endif
+    DebugObject d_obj;
 } PRStreamSink;
 
 /**
@@ -63,8 +60,9 @@ typedef struct {
  *            The object must be freed from the error handler.
  * @param bprfd the {@link BPRFileDesc} object to write data to. Registers a
  *              PR_POLL_WRITE handler which must not be registered.
+ * @param pg pending group
  */
-void PRStreamSink_Init (PRStreamSink *s, FlowErrorReporter rep, BPRFileDesc *bprfd);
+void PRStreamSink_Init (PRStreamSink *s, FlowErrorReporter rep, BPRFileDesc *bprfd, BPendingGroup *pg);
 
 /**
  * Frees the object.

nspr_support/PRStreamSource.c

@@ -28,75 +28,66 @@
 
 static void report_error (PRStreamSource *s, int error)
 {
-    #ifndef NDEBUG
-    s->in_error = 1;
     DEAD_ENTER(s->dead)
-    #endif
-    
     FlowErrorReporter_ReportError(&s->rep, &error);
-    
-    #ifndef NDEBUG
     ASSERT(DEAD_KILLED)
     DEAD_LEAVE(s->dead);
-    #endif
 }
 
-static int output_handler_recv (PRStreamSource *s, uint8_t *data, int data_avail)
+static void try_recv (PRStreamSource *s)
 {
-    ASSERT(s->out_avail == -1)
-    ASSERT(data_avail > 0)
-    ASSERT(!s->in_error)
+    ASSERT(s->out_avail > 0)
+    
+    int res = PR_Read(s->bprfd->prfd, s->out, s->out_avail);
+    if (res < 0 && PR_GetError() == PR_WOULD_BLOCK_ERROR) {
+        // wait for socket in prfd_handler
+        BPRFileDesc_EnableEvent(s->bprfd, PR_POLL_READ);
+        return;
+    }
     
-    PRInt32 res = PR_Read(s->bprfd->prfd, data, data_avail);
     if (res < 0) {
-        PRErrorCode error = PR_GetError();
-        if (error == PR_WOULD_BLOCK_ERROR) {
-            s->out_avail = data_avail;
-            s->out = data;
-            BPRFileDesc_EnableEvent(s->bprfd, PR_POLL_READ);
-            return 0;
-        }
         report_error(s, PRSTREAMSOURCE_ERROR_NSPR);
-        return -1;
+        return;
     }
     
     if (res == 0) {
         report_error(s, PRSTREAMSOURCE_ERROR_CLOSED);
-        return -1;
+        return;
     }
     
-    return res;
+    ASSERT(res > 0)
+    ASSERT(res <= s->out_avail)
+    
+    // finish packet
+    s->out_avail = -1;
+    StreamRecvInterface_Done(&s->output, res);
+}
+
+static void output_handler_recv (PRStreamSource *s, uint8_t *data, int data_avail)
+{
+    ASSERT(data_avail > 0)
+    ASSERT(s->out_avail == -1)
+    DebugObject_Access(&s->d_obj);
+    
+    // set packet
+    s->out_avail = data_avail;
+    s->out = data;
+    
+    try_recv(s);
+    return;
 }
 
 static void prfd_handler (PRStreamSource *s, PRInt16 event)
 {
     ASSERT(s->out_avail > 0)
     ASSERT(event == PR_POLL_READ)
-    ASSERT(!s->in_error)
-    
-    PRInt32 res = PR_Read(s->bprfd->prfd, s->out, s->out_avail);
-    if (res < 0) {
-        PRErrorCode error = PR_GetError();
-        if (error == PR_WOULD_BLOCK_ERROR) {
-            BPRFileDesc_EnableEvent(s->bprfd, PR_POLL_READ);
-            return;
-        }
-        report_error(s, PRSTREAMSOURCE_ERROR_NSPR);
-        return;
-    }
-    
-    if (res == 0) {
-        report_error(s, PRSTREAMSOURCE_ERROR_CLOSED);
-        return;
-    }
+    DebugObject_Access(&s->d_obj);
     
-    s->out_avail = -1;
-    
-    StreamRecvInterface_Done(&s->output, res);
+    try_recv(s);
     return;
 }
 
-void PRStreamSource_Init (PRStreamSource *s, FlowErrorReporter rep, BPRFileDesc *bprfd)
+void PRStreamSource_Init (PRStreamSource *s, FlowErrorReporter rep, BPRFileDesc *bprfd, BPendingGroup *pg)
 {
     // init arguments
     s->rep = rep;
@@ -109,23 +100,16 @@ void PRStreamSource_Init (PRStreamSource *s, FlowErrorReporter rep, BPRFileDesc
     BPRFileDesc_AddEventHandler(s->bprfd, PR_POLL_READ, (BPRFileDesc_handler)prfd_handler, s);
     
     // init output
-    StreamRecvInterface_Init(&s->output, (StreamRecvInterface_handler_recv)output_handler_recv, s);
+    StreamRecvInterface_Init(&s->output, (StreamRecvInterface_handler_recv)output_handler_recv, s, pg);
     
     // have no output packet
     s->out_avail = -1;
     
-    // init debugging
-    #ifndef NDEBUG
-    s->in_error = 0;
-    #endif
-    
-    // init debug object
     DebugObject_Init(&s->d_obj);
 }
 
 void PRStreamSource_Free (PRStreamSource *s)
 {
-    // free debug object
     DebugObject_Free(&s->d_obj);
     
     // free output
@@ -140,5 +124,7 @@ void PRStreamSource_Free (PRStreamSource *s)
 
 StreamRecvInterface * PRStreamSource_GetOutput (PRStreamSource *s)
 {
+    DebugObject_Access(&s->d_obj);
+    
     return &s->output;
 }

nspr_support/PRStreamSource.h

@@ -31,8 +31,8 @@
 
 #include <misc/dead.h>
 #include <system/DebugObject.h>
-#include <flow/error.h>
 #include <flow/StreamRecvInterface.h>
+#include <flow/error.h>
 #include <nspr_support/BPRFileDesc.h>
 
 #define PRSTREAMSOURCE_ERROR_CLOSED 0
@@ -42,16 +42,13 @@
  * A {@link StreamRecvInterface} source for a NSPR file descriptor (PRFileDesc) via {@link BPRFileDesc}.
  */
 typedef struct {
-    DebugObject d_obj;
     dead_t dead;
     FlowErrorReporter rep;
     BPRFileDesc *bprfd;
     StreamRecvInterface output;
     int out_avail;
     uint8_t *out;
-    #ifndef NDEBUG
-    int in_error;
-    #endif
+    DebugObject d_obj;
 } PRStreamSource;
 
 /**
@@ -65,8 +62,9 @@ typedef struct {
  *            The object must be freed from the error handler.
  * @param bprfd the {@link BPRFileDesc} object to read data from. Registers a
  *              PR_POLL_READ handler which must not be registered.
+ * @param pg pending group
  */
-void PRStreamSource_Init (PRStreamSource *s, FlowErrorReporter rep, BPRFileDesc *bprfd);
+void PRStreamSource_Init (PRStreamSource *s, FlowErrorReporter rep, BPRFileDesc *bprfd, BPendingGroup *pg);
 
 /**
  * Frees the object.

server/server.h

@@ -23,7 +23,6 @@
 #include <stdint.h>
 
 #include <protocol/scproto.h>
-#include <misc/dead.h>
 #include <structure/LinkedList2.h>
 #include <structure/HashTable.h>
 #include <structure/BAVL.h>
@@ -69,8 +68,6 @@
 struct client_data;
 
 struct peer_flow {
-    // dead variable
-    dead_t dead;
     // source client
     struct client_data *src_client;
     // destination client
@@ -85,15 +82,19 @@ struct peer_flow {
     // output chain
     PacketPassFairQueueFlow qflow;
     PacketProtoFlow oflow;
-    BestEffortPacketWriteInterface *bepwi;
+    BufferWriter *input;
     int packet_len;
     uint8_t *packet;
 };
 
+struct peer_know {
+    struct client_data *from;
+    struct client_data *to;
+    LinkedList2Node from_node;
+    LinkedList2Node to_node;
+};
+
 struct client_data {
-    // dead variable
-    dead_t dead;
-    
     // socket
     BSocket sock;
     BAddr addr;
@@ -122,12 +123,21 @@ struct client_data {
     // node in clients-by-id hash table
     HashTableNode table_node_id;
     
+    // knowledge lists
+    LinkedList2 know_out_list;
+    LinkedList2 know_in_list;
+    
     // flows from us
     LinkedList2 peer_out_flows_list;
     BAVL peer_out_flows_tree;
     
     // whether it's being removed
     int dying;
+    BPending dying_job;
+    
+    // publish job
+    BPending publish_job;
+    LinkedList2Iterator publish_it;
     
     // error domain
     FlowErrorDomain domain;
@@ -151,7 +161,7 @@ struct client_data {
     // output control flow
     PacketPassPriorityQueueFlow output_control_qflow;
     PacketProtoFlow output_control_oflow;
-    BestEffortPacketWriteInterface *output_control_input;
+    BufferWriter *output_control_input;
     int output_control_packet_len;
     uint8_t *output_control_packet;
     

server_connection/ServerConnection.c

@@ -42,13 +42,13 @@ static void connect_handler (ServerConnection *o, int event);
 static void pending_handler (ServerConnection *o);
 static SECStatus client_auth_data_callback (ServerConnection *o, PRFileDesc *fd, CERTDistNames *caNames, CERTCertificate **pRetCert, SECKEYPrivateKey **pRetKey);
 static void error_handler (ServerConnection *o, int component, const void *data);
-static int input_handler_send (ServerConnection *o, uint8_t *data, int data_len);
+static void input_handler_send (ServerConnection *o, uint8_t *data, int data_len);
 static void packet_hello (ServerConnection *o, uint8_t *data, int data_len);
 static void packet_newclient (ServerConnection *o, uint8_t *data, int data_len);
 static void packet_endclient (ServerConnection *o, uint8_t *data, int data_len);
 static void packet_inmsg (ServerConnection *o, uint8_t *data, int data_len);
 static int start_packet (ServerConnection *o, void **data, int len);
-static int end_packet (ServerConnection *o, uint8_t type);
+static void end_packet (ServerConnection *o, uint8_t type);
 
 void report_error (ServerConnection *o)
 {
@@ -72,6 +72,7 @@ void connect_handler (ServerConnection *o, int event)
 {
     ASSERT(o->state == STATE_CONNECTING)
     ASSERT(event == BSOCKET_CONNECT)
+    DebugObject_Access(&o->d_obj);
     
     // remove connect event handler
     BSocket_RemoveEventHandler(&o->sock, BSOCKET_CONNECT);
@@ -121,29 +122,56 @@ void connect_handler (ServerConnection *o, int event)
     // init error domain
     FlowErrorDomain_Init(&o->ioerrdomain, (FlowErrorDomain_handler)error_handler, o);
     
+    // init input chain
+    StreamRecvInterface *source_interface;
+    if (o->have_ssl) {
+        PRStreamSource_Init(&o->input_source.ssl, FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_SOURCE), &o->ssl_bprfd, BReactor_PendingGroup(o->reactor));
+        source_interface = PRStreamSource_GetOutput(&o->input_source.ssl);
+    } else {
+        StreamSocketSource_Init(&o->input_source.plain, FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_SOURCE), &o->sock, BReactor_PendingGroup(o->reactor));
+        source_interface = StreamSocketSource_GetOutput(&o->input_source.plain);
+    }
+    PacketPassInterface_Init(&o->input_interface, SC_MAX_ENC, (PacketPassInterface_handler_send)input_handler_send, o, BReactor_PendingGroup(o->reactor));
+    if (!PacketProtoDecoder_Init(
+        &o->input_decoder,
+        FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_DECODER),
+        source_interface,
+        &o->input_interface,
+        BReactor_PendingGroup(o->reactor)
+    )) {
+        BLog(BLOG_ERROR, "PacketProtoDecoder_Init failed");
+        goto fail2;
+    }
+    
+    // set job to send hello
+    // this needs to be in here because hello sending must be done after sending started (so we can write into the send buffer),
+    // but before receiving started (so we don't get into conflict with the user sending packets)
+    BPending_Init(&o->start_job, BReactor_PendingGroup(o->reactor), (BPending_handler)pending_handler, o);
+    BPending_Set(&o->start_job);
+    
     // init keepalive output branch
-    SCKeepaliveSource_Init(&o->output_ka_zero);
-    PacketProtoEncoder_Init(&o->output_ka_encoder, SCKeepaliveSource_GetOutput(&o->output_ka_zero));
+    SCKeepaliveSource_Init(&o->output_ka_zero, BReactor_PendingGroup(o->reactor));
+    PacketProtoEncoder_Init(&o->output_ka_encoder, SCKeepaliveSource_GetOutput(&o->output_ka_zero), BReactor_PendingGroup(o->reactor));
     
     // init output common
     
     // init sink
     StreamPassInterface *sink_interface;
     if (o->have_ssl) {
-        PRStreamSink_Init(&o->output_sink.ssl, FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_SINK), &o->ssl_bprfd);
+        PRStreamSink_Init(&o->output_sink.ssl, FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_SINK), &o->ssl_bprfd, BReactor_PendingGroup(o->reactor));
         sink_interface = PRStreamSink_GetInput(&o->output_sink.ssl);
     } else {
-        StreamSocketSink_Init(&o->output_sink.plain, FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_SINK), &o->sock);
+        StreamSocketSink_Init(&o->output_sink.plain, FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_SINK), &o->sock, BReactor_PendingGroup(o->reactor));
         sink_interface = StreamSocketSink_GetInput(&o->output_sink.plain);
     }
     
     // init sender
-    PacketStreamSender_Init(&o->output_sender, sink_interface, PACKETPROTO_ENCLEN(SC_MAX_ENC));
+    PacketStreamSender_Init(&o->output_sender, sink_interface, PACKETPROTO_ENCLEN(SC_MAX_ENC), BReactor_PendingGroup(o->reactor));
     
     // init keepalives
     if (!KeepaliveIO_Init(&o->output_keepaliveio, o->reactor, PacketStreamSender_GetInput(&o->output_sender), PacketProtoEncoder_GetOutput(&o->output_ka_encoder), o->keepalive_interval)) {
         BLog(BLOG_ERROR, "KeepaliveIO_Init failed");
-        goto fail1a;
+        goto fail3;
     }
     
     // init queue
@@ -157,7 +185,7 @@ void connect_handler (ServerConnection *o, int event)
     // init PacketProtoFlow
     if (!PacketProtoFlow_Init(&o->output_local_oflow, SC_MAX_ENC, o->buffer_size, PacketPassPriorityQueueFlow_GetInput(&o->output_local_qflow), BReactor_PendingGroup(o->reactor))) {
         BLog(BLOG_ERROR, "PacketProtoFlow_Init failed");
-        goto fail2;
+        goto fail4;
     }
     o->output_local_if = PacketProtoFlow_GetInput(&o->output_local_oflow);
     
@@ -167,54 +195,17 @@ void connect_handler (ServerConnection *o, int event)
     // init output user flow
     PacketPassPriorityQueueFlow_Init(&o->output_user_qflow, &o->output_queue, 1);
     
-    // init input chain
-    StreamRecvInterface *source_interface;
-    if (o->have_ssl) {
-        PRStreamSource_Init(&o->input_source.ssl, FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_SOURCE), &o->ssl_bprfd);
-        source_interface = PRStreamSource_GetOutput(&o->input_source.ssl);
-    } else {
-        StreamSocketSource_Init(&o->input_source.plain, FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_SOURCE), &o->sock);
-        source_interface = StreamSocketSource_GetOutput(&o->input_source.plain);
-    }
-    PacketPassInterface_Init(&o->input_interface, SC_MAX_ENC, (PacketPassInterface_handler_send)input_handler_send, o);
-    if (!PacketProtoDecoder_Init(
-        &o->input_decoder,
-        FlowErrorReporter_Create(&o->ioerrdomain, COMPONENT_DECODER),
-        source_interface,
-        &o->input_interface,
-        BReactor_PendingGroup(o->reactor)
-    )) {
-        BLog(BLOG_ERROR, "PacketProtoDecoder_Init failed");
-        goto fail3;
-    }
-    
-    // set job to start I/O
-    BPending_Init(&o->start_job, BReactor_PendingGroup(o->reactor), (BPending_handler)pending_handler, o);
-    BPending_Set(&o->start_job);
-    
     // update state
     o->state = STATE_WAITINIT;
     
     return;
     
-    // free input
-fail3:
-    PacketPassInterface_Free(&o->input_interface);
-    if (o->have_ssl) {
-        PRStreamSource_Free(&o->input_source.ssl);
-    } else {
-        StreamSocketSource_Free(&o->input_source.plain);
-    }
-    // free output user flow
-    PacketPassPriorityQueueFlow_Free(&o->output_user_qflow);
-    // free output local flow
-    PacketProtoFlow_Free(&o->output_local_oflow);
-fail2:
+fail4:
     PacketPassPriorityQueueFlow_Free(&o->output_local_qflow);
     // free output common
     PacketPassPriorityQueue_Free(&o->output_queue);
     KeepaliveIO_Free(&o->output_keepaliveio);
-fail1a:
+fail3:
     PacketStreamSender_Free(&o->output_sender);
     if (o->have_ssl) {
         PRStreamSink_Free(&o->output_sink.ssl);
@@ -224,6 +215,17 @@ fail1a:
     // free output keep-alive branch
     PacketProtoEncoder_Free(&o->output_ka_encoder);
     SCKeepaliveSource_Free(&o->output_ka_zero);
+    // free job
+    BPending_Free(&o->start_job);
+    // free input
+    PacketProtoDecoder_Free(&o->input_decoder);
+fail2:
+    PacketPassInterface_Free(&o->input_interface);
+    if (o->have_ssl) {
+        PRStreamSource_Free(&o->input_source.ssl);
+    } else {
+        StreamSocketSource_Free(&o->input_source.plain);
+    }
     // free SSL
     if (o->have_ssl) {
         BPRFileDesc_Free(&o->ssl_bprfd);
@@ -238,27 +240,23 @@ fail0:
 void pending_handler (ServerConnection *o)
 {
     ASSERT(o->state == STATE_WAITINIT)
+    DebugObject_Access(&o->d_obj);
     
     // send hello
-    int res;
     struct sc_client_hello *packet;
-    if ((res = start_packet(o, (void **)&packet, sizeof(struct sc_client_hello))) < 0) {
-        return;
-    }
-    if (!res) {
+    if (!start_packet(o, (void **)&packet, sizeof(struct sc_client_hello))) {
         BLog(BLOG_ERROR, "no buffer for hello");
         report_error(o);
         return;
     }
     packet->version = htol16(SC_VERSION);
-    if (end_packet(o, SCID_CLIENTHELLO) < 0) {
-        return;
-    }
+    end_packet(o, SCID_CLIENTHELLO);
 }
 
 SECStatus client_auth_data_callback (ServerConnection *o, PRFileDesc *fd, CERTDistNames *caNames, CERTCertificate **pRetCert, SECKEYPrivateKey **pRetKey)
 {
     ASSERT(o->have_ssl)
+    DebugObject_Access(&o->d_obj);
     
     CERTCertificate *newcert;
     if (!(newcert = CERT_DupCertificate(o->client_cert))) {
@@ -279,6 +277,7 @@ SECStatus client_auth_data_callback (ServerConnection *o, PRFileDesc *fd, CERTDi
 void error_handler (ServerConnection *o, int component, const void *data)
 {
     ASSERT(o->state >= STATE_WAITINIT)
+    DebugObject_Access(&o->d_obj);
     
     switch (component) {
         case COMPONENT_SOURCE:
@@ -301,16 +300,17 @@ void error_handler (ServerConnection *o, int component, const void *data)
     return;
 }
 
-int input_handler_send (ServerConnection *o, uint8_t *data, int data_len)
+void input_handler_send (ServerConnection *o, uint8_t *data, int data_len)
 {
     ASSERT(o->state >= STATE_WAITINIT)
     ASSERT(data_len >= 0)
     ASSERT(data_len <= SC_MAX_ENC)
+    DebugObject_Access(&o->d_obj);
     
     if (data_len < sizeof(struct sc_header)) {
         BLog(BLOG_ERROR, "packet too short (no sc header)");
         report_error(o);
-        return -1;
+        return;
     }
     
     struct sc_header *header = (struct sc_header *)data;
@@ -318,36 +318,28 @@ int input_handler_send (ServerConnection *o, uint8_t *data, int data_len)
     uint8_t *sc_data = data + sizeof(struct sc_header);
     int sc_data_len = data_len - sizeof(struct sc_header);
     
-    #ifndef NDEBUG
-    DEAD_ENTER(o->dead)
-    #endif
+    // finish packet
+    PacketPassInterface_Done(&o->input_interface);
     
     // call appropriate handler based on packet type
     switch (header->type) {
         case SCID_SERVERHELLO:
             packet_hello(o, sc_data, sc_data_len);
-            break;
+            return;
         case SCID_NEWCLIENT:
             packet_newclient(o, sc_data, sc_data_len);
-            break;
+            return;
         case SCID_ENDCLIENT:
             packet_endclient(o, sc_data, sc_data_len);
-            break;
+            return;
         case SCID_INMSG:
             packet_inmsg(o, sc_data, sc_data_len);
-            break;
+            return;
         default:
             BLog(BLOG_ERROR, "unknown packet type %d", (int)header->type);
             report_error(o);
+            return;
     }
-    
-    #ifndef NDEBUG
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
-    #endif
-    
-    return 1;
 }
 
 void packet_hello (ServerConnection *o, uint8_t *data, int data_len)
@@ -460,16 +452,8 @@ int start_packet (ServerConnection *o, void **data, int len)
     ASSERT(len <= SC_MAX_PAYLOAD)
     ASSERT(data || len == 0)
     
-    // obtain location for writing the packet
-    DEAD_ENTER(o->dead)
-    int res = BestEffortPacketWriteInterface_Sender_StartPacket(o->output_local_if, &o->output_local_packet);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
-    
-    ASSERT(res == 0 || res == 1)
-    
-    if (!res) {
+    // obtain memory location
+    if (!BufferWriter_StartPacket(o->output_local_if, &o->output_local_packet)) {
         BLog(BLOG_ERROR, "out of buffer");
         return 0;
     }
@@ -483,7 +467,7 @@ int start_packet (ServerConnection *o, void **data, int len)
     return 1;
 }
 
-int end_packet (ServerConnection *o, uint8_t type)
+void end_packet (ServerConnection *o, uint8_t type)
 {
     ASSERT(o->state >= STATE_WAITINIT)
     ASSERT(o->output_local_packet_len >= 0)
@@ -494,15 +478,9 @@ int end_packet (ServerConnection *o, uint8_t type)
     header->type = type;
     
     // finish writing packet
-    DEAD_ENTER(o->dead)
-    BestEffortPacketWriteInterface_Sender_EndPacket(o->output_local_if, sizeof(struct sc_header) + o->output_local_packet_len);
-    if (DEAD_LEAVE(o->dead)) {
-        return -1;
-    }
+    BufferWriter_EndPacket(o->output_local_if, sizeof(struct sc_header) + o->output_local_packet_len);
     
     o->output_local_packet_len = -1;
-    
-    return 0;
 }
 
 int ServerConnection_Init (
@@ -570,7 +548,6 @@ int ServerConnection_Init (
     // set no error
     o->error = 0;
     
-    // init debug object
     DebugObject_Init(&o->d_obj);
     
     return 1;
@@ -583,22 +560,9 @@ fail0:
 
 void ServerConnection_Free (ServerConnection *o)
 {
-    // free debug object
     DebugObject_Free(&o->d_obj);
     
     if (o->state > STATE_CONNECTING) {
-        // free job
-        BPending_Free(&o->start_job);
-        
-        // free input chain
-        PacketProtoDecoder_Free(&o->input_decoder);
-        PacketPassInterface_Free(&o->input_interface);
-        if (o->have_ssl) {
-            PRStreamSource_Free(&o->input_source.ssl);
-        } else {
-            StreamSocketSource_Free(&o->input_source.plain);
-        }
-        
         // allow freeing queue flows
         PacketPassPriorityQueue_PrepareFree(&o->output_queue);
         
@@ -623,6 +587,18 @@ void ServerConnection_Free (ServerConnection *o)
         PacketProtoEncoder_Free(&o->output_ka_encoder);
         SCKeepaliveSource_Free(&o->output_ka_zero);
         
+        // free job
+        BPending_Free(&o->start_job);
+        
+        // free input chain
+        PacketProtoDecoder_Free(&o->input_decoder);
+        PacketPassInterface_Free(&o->input_interface);
+        if (o->have_ssl) {
+            PRStreamSource_Free(&o->input_source.ssl);
+        } else {
+            StreamSocketSource_Free(&o->input_source.plain);
+        }
+        
         // free SSL
         if (o->have_ssl) {
             BPRFileDesc_Free(&o->ssl_bprfd);
@@ -645,13 +621,10 @@ int ServerConnection_StartMessage (ServerConnection *o, void **data, peerid_t pe
     ASSERT(len >= 0)
     ASSERT(len <= SC_MAX_MSGLEN)
     ASSERT(data || len == 0)
+    DebugObject_Access(&o->d_obj);
     
     uint8_t *packet;
-    int res;
-    if ((res = start_packet(o, (void **)&packet, sizeof(struct sc_client_outmsg) + len)) < 0) {
-        return -1;
-    }
-    if (!res) {
+    if (!start_packet(o, (void **)&packet, sizeof(struct sc_client_outmsg) + len)) {
         return 0;
     }
     
@@ -670,15 +643,16 @@ void ServerConnection_EndMessage (ServerConnection *o)
     ASSERT(!o->error)
     ASSERT(o->state == STATE_COMPLETE)
     ASSERT(o->output_local_packet_len >= 0)
+    DebugObject_Access(&o->d_obj);
     
     end_packet(o, SCID_OUTMSG);
-    return;
 }
 
 PacketPassInterface * ServerConnection_GetSendInterface (ServerConnection *o)
 {
     ASSERT(!o->error)
     ASSERT(o->state == STATE_COMPLETE)
+    DebugObject_Access(&o->d_obj);
     
     return PacketPassPriorityQueueFlow_GetInput(&o->output_user_qflow);
 }

server_connection/ServerConnection.h

@@ -115,9 +115,6 @@ typedef void (*ServerConnection_handler_message) (void *user, peerid_t peer_id,
  * Object used to communicate with a VPN chat server.
  */
 typedef struct {
-    // debug object
-    DebugObject d_obj;
-    
     // dead var
     dead_t dead;
     
@@ -169,6 +166,14 @@ typedef struct {
     // I/O error domain
     FlowErrorDomain ioerrdomain;
     
+    // input
+    union {
+        StreamSocketSource plain;
+        PRStreamSource ssl;
+    } input_source;
+    PacketProtoDecoder input_decoder;
+    PacketPassInterface input_interface;
+    
     // keepalive output branch
     SCKeepaliveSource output_ka_zero;
     PacketProtoEncoder output_ka_encoder;
@@ -185,23 +190,17 @@ typedef struct {
     // output local flow
     int output_local_packet_len;
     uint8_t *output_local_packet;
-    BestEffortPacketWriteInterface *output_local_if;
+    BufferWriter *output_local_if;
     PacketProtoFlow output_local_oflow;
     PacketPassPriorityQueueFlow output_local_qflow;
     
     // output user flow
     PacketPassPriorityQueueFlow output_user_qflow;
     
-    // input
-    union {
-        StreamSocketSource plain;
-        PRStreamSource ssl;
-    } input_source;
-    PacketProtoDecoder input_decoder;
-    PacketPassInterface input_interface;
-    
     // job to start client I/O
     BPending start_job;
+    
+    DebugObject d_obj;
 } ServerConnection;
 
 /**

system/BPending.c

@@ -58,7 +58,7 @@ void BPendingGroup_ExecuteJob (BPendingGroup *g)
     DebugObject_Access(&g->d_obj);
     
     // get a job
-    LinkedList2Node *node = LinkedList2_GetFirst(&g->jobs);
+    LinkedList2Node *node = LinkedList2_GetLast(&g->jobs);
     BPending *p = UPPER_OBJECT(node, BPending, pending_node);
     ASSERT(p->pending)
     
@@ -129,3 +129,10 @@ void BPending_Unset (BPending *o)
         o->pending = 0;
     }
 }
+
+int BPending_IsSet (BPending *o)
+{
+    DebugObject_Access(&o->d_obj);
+    
+    return o->pending;
+}

system/BPending.h

@@ -43,7 +43,7 @@
 typedef void (*BPending_handler) (void *user);
 
 /**
- * Object that contains a queue of jobs pending execution.
+ * Object that contains a list of jobs pending execution.
  */
 typedef struct {
     LinkedList2 jobs;
@@ -87,8 +87,10 @@ void BPendingGroup_Free (BPendingGroup *g);
 int BPendingGroup_HasJobs (BPendingGroup *g);
 
 /**
- * Executes a job.
- * There must be at least one job in the queue.
+ * Executes the top job on the job list.
+ * The job is removed from the list and enters
+ * not set state before being executed.
+ * There must be at least one job in job list.
  * 
  * @param g the object
  */
@@ -115,9 +117,9 @@ void BPending_Init (BPending *o, BPendingGroup *g, BPending_handler handler, voi
 void BPending_Free (BPending *o);
 
 /**
- * Enables the job, appending it to the end of the group's queue.
+ * Enables the job, pushing it to the top of the job list.
  * If the object was already in set state, the job is removed from its
- * current position in the queue before being appended.
+ * current position in the list before being pushed.
  * The object enters set state.
  * 
  * @param o the object
@@ -125,7 +127,7 @@ void BPending_Free (BPending *o);
 void BPending_Set (BPending *o);
 
 /**
- * Disables the job, removing it from the group's queue.
+ * Disables the job, removing it from the job list.
  * If the object was not in set state, nothing is done.
  * The object enters not set state.
  * 
@@ -133,4 +135,12 @@ void BPending_Set (BPending *o);
  */
 void BPending_Unset (BPending *o);
 
+/**
+ * Checks if the job is in set state.
+ * 
+ * @param o the object
+ * @return 1 if in set state, 0 if not
+ */
+int BPending_IsSet (BPending *o);
+
 #endif

system/BReactor.c

@@ -498,8 +498,6 @@ int BReactor_Exec (BReactor *bsys)
 {
     BLog(BLOG_DEBUG, "Entering event loop");
 
-    bsys->exiting = 0;
-
     while (1) {
         dispatch_jobs(bsys);
         dispatch_timers(bsys);

system/BReactor.h

@@ -248,7 +248,7 @@ int BReactor_Exec (BReactor *bsys);
 /**
  * Causes the event loop ({@link BReactor_Exec}) to cease
  * dispatching events and return.
- * If the event loop is not running, this function does nothing.
+ * Any further calls of {@link BReactor_Exec} will return immediately.
  *
  * @param bsys the object
  * @param code value {@link BReactor_Exec} should return. If this is

tests/TestPacketSink.h

@@ -1,86 +0,0 @@
-#ifndef _TESTPACKETSINK_H
-#define _TESTPACKETSINK_H
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <misc/debug.h>
-#include <system/DebugObject.h>
-#include <flow/PacketPassInterface.h>
-
-typedef struct {
-    DebugObject d_obj;
-    PacketPassInterface input;
-    int accpeting;
-    int have_packet;
-    const char *expect;
-} TestPacketSink;
-
-static int _TestPacketSink_input_handler_send (TestPacketSink *s, uint8_t *data, int data_len)
-{
-    ASSERT(!s->have_packet)
-    ASSERT(s->expect)
-    ASSERT(strlen(s->expect) == data_len)
-    ASSERT(!memcmp(s->expect, data, data_len))
-    
-    s->expect = NULL;
-    
-    if (s->accpeting) {
-        return 1;
-    }
-    
-    s->have_packet = 1;
-    
-    return 0;
-}
-
-static void _TestPacketSink_input_handler_cancel (TestPacketSink *s)
-{
-    ASSERT(s->have_packet)
-    
-    s->have_packet = 0;
-}
-
-static void TestPacketSink_Init (TestPacketSink *s, int mtu)
-{
-    PacketPassInterface_Init(&s->input, mtu, (PacketPassInterface_handler_send)_TestPacketSink_input_handler_send, s);
-    PacketPassInterface_EnableCancel(&s->input, (PacketPassInterface_handler_cancel)_TestPacketSink_input_handler_cancel);
-    s->accpeting = 1;
-    s->have_packet = 0;
-    s->expect = NULL;
-    DebugObject_Init(&s->d_obj);
-}
-
-static void TestPacketSink_Free (TestPacketSink *s)
-{
-    DebugObject_Free(&s->d_obj);
-    PacketPassInterface_Free(&s->input);
-}
-
-static PacketPassInterface * TestPacketSink_GetInput (TestPacketSink *s)
-{
-    return &s->input;
-}
-
-static void TestPacketSink_SetAccepting (TestPacketSink *s, int accepting)
-{
-    s->accpeting = !!accepting;
-}
-
-static void TestPacketSink_Done (TestPacketSink *s)
-{
-    ASSERT(s->have_packet)
-    
-    s->have_packet = 0;
-    
-    PacketPassInterface_Done(&s->input);
-    return;
-}
-
-static void TestPacketSink_Expect (TestPacketSink *s, const char *str)
-{
-    s->expect = str;
-}
-
-#endif