- TCPIP lock rewrite (part 3 of 3)

- Use a spin lock to protect the signalled connections list
 - Add a debug ASSERT into each function that depends on the TCPLock being held
 - Release TCPLock before completing the IRP
 - Add some locking where it was forgotten
 - Optimize HandleSignalledConnection

svn path=/trunk/; revision=41752

reactos/drivers/network/tcpip/include/tcp.h

@@ -84,7 +84,8 @@ typedef struct _SLEEPING_THREAD {
 #define SRF_FIN   TCP_FIN
 
 extern LONG TCP_IPIdentification;
-extern LIST_ENTRY SignalledConnections;
+extern LIST_ENTRY SignalledConnectionsList;
+extern KSPIN_LOCK SignalledConnectionsLock;
 extern LIST_ENTRY SleepingThreadsList;
 extern FAST_MUTEX SleepingThreadsLock;
 extern RECURSIVE_MUTEX TCPLock;

reactos/drivers/network/tcpip/recmutex/recmutex.h

@@ -24,4 +24,6 @@ extern VOID RecursiveMutexInit( PRECURSIVE_MUTEX RecMutex );
 extern SIZE_T RecursiveMutexEnter( PRECURSIVE_MUTEX RecMutex, BOOLEAN ToRead );
 extern VOID RecursiveMutexLeave( PRECURSIVE_MUTEX RecMutex );
 
+#define ASSERT_LOCKED(x) ASSERT((x)->Locked)
+
 #endif/*_ROSRTL_RECMUTEX_H*/

reactos/drivers/network/tcpip/tcpip/dispatch.c

@@ -106,6 +106,7 @@ VOID DispDoDisconnect( PVOID Data ) {
     PDISCONNECT_TYPE DisType = (PDISCONNECT_TYPE)Data;
 
     TI_DbgPrint(DEBUG_IRP, ("PostCancel: DoDisconnect\n"));
+    TcpipRecursiveMutexEnter(&TCPLock, TRUE);
     TCPDisconnect
 	( DisType->Context,
 	  DisType->Type,
@@ -113,6 +114,7 @@ VOID DispDoDisconnect( PVOID Data ) {
 	  NULL,
 	  DispDataRequestComplete,
 	  DisType->Irp );
+    TcpipRecursiveMutexLeave(&TCPLock);
     TI_DbgPrint(DEBUG_IRP, ("PostCancel: DoDisconnect done\n"));
 
     DispDataRequestComplete(DisType->Irp, STATUS_CANCELLED, 0);
@@ -405,13 +407,13 @@ NTSTATUS DispTdiConnect(
       Irp );
 
 done:
+  TcpipRecursiveMutexLeave( &TCPLock );
+
   if (Status != STATUS_PENDING) {
       DispDataRequestComplete(Irp, Status, 0);
   } else
       IoMarkIrpPending(Irp);
 
-  TcpipRecursiveMutexLeave( &TCPLock );
-
   TI_DbgPrint(MAX_TRACE, ("TCP Connect returned %08x\n", Status));
 
   return Status;
@@ -513,13 +515,13 @@ NTSTATUS DispTdiDisconnect(
       Irp );
 
 done:
+   TcpipRecursiveMutexLeave( &TCPLock );
+
    if (Status != STATUS_PENDING) {
        DispDataRequestComplete(Irp, Status, 0);
    } else
        IoMarkIrpPending(Irp);
 
-  TcpipRecursiveMutexLeave( &TCPLock );
-
   TI_DbgPrint(MAX_TRACE, ("TCP Disconnect returned %08x\n", Status));
 
   return Status;
@@ -618,13 +620,13 @@ NTSTATUS DispTdiListen(
   }
 
 done:
+  TcpipRecursiveMutexLeave( &TCPLock );
+
   if (Status != STATUS_PENDING) {
       DispDataRequestComplete(Irp, Status, 0);
   } else
       IoMarkIrpPending(Irp);
 
-  TcpipRecursiveMutexLeave( &TCPLock );
-
   TI_DbgPrint(MID_TRACE,("Leaving %x\n", Status));
 
   return Status;
@@ -834,13 +836,13 @@ NTSTATUS DispTdiReceive(
     }
 
 done:
+  TcpipRecursiveMutexLeave( &TCPLock );
+
   if (Status != STATUS_PENDING) {
       DispDataRequestComplete(Irp, Status, BytesReceived);
   } else
       IoMarkIrpPending(Irp);
 
-  TcpipRecursiveMutexLeave( &TCPLock );
-
   TI_DbgPrint(DEBUG_IRP, ("Leaving. Status is (0x%X)\n", Status));
 
   return Status;
@@ -912,13 +914,13 @@ NTSTATUS DispTdiReceiveDatagram(
     }
 
 done:
+   TcpipRecursiveMutexLeave( &TCPLock );
+
    if (Status != STATUS_PENDING) {
        DispDataRequestComplete(Irp, Status, BytesReceived);
    } else
        IoMarkIrpPending(Irp);
 
-  TcpipRecursiveMutexLeave( &TCPLock );
-
   TI_DbgPrint(DEBUG_IRP, ("Leaving. Status is (0x%X)\n", Status));
 
   return Status;
@@ -988,13 +990,13 @@ NTSTATUS DispTdiSend(
     }
 
 done:
+   TcpipRecursiveMutexLeave( &TCPLock );
+
    if (Status != STATUS_PENDING) {
        DispDataRequestComplete(Irp, Status, BytesSent);
    } else
        IoMarkIrpPending(Irp);
 
-  TcpipRecursiveMutexLeave( &TCPLock );
-
   TI_DbgPrint(DEBUG_IRP, ("Leaving. Status is (0x%X)\n", Status));
 
   return Status;
@@ -1070,13 +1072,13 @@ NTSTATUS DispTdiSendDatagram(
     }
 
 done:
+    TcpipRecursiveMutexLeave( &TCPLock );
+
     if (Status != STATUS_PENDING) {
         DispDataRequestComplete(Irp, Status, Irp->IoStatus.Information);
     } else
         IoMarkIrpPending(Irp);
 
-    TcpipRecursiveMutexLeave( &TCPLock );
-
     TI_DbgPrint(DEBUG_IRP, ("Leaving.\n"));
 
     return Status;

reactos/drivers/network/tcpip/tcpip/fileobjs.c

@@ -363,7 +363,9 @@ NTSTATUS FileCloseAddress(
   case IPPROTO_TCP:
     TCPFreePort( AddrFile->Port );
     if( AddrFile->Listener ) {
+            TcpipRecursiveMutexEnter(&TCPLock, TRUE);
 	    TCPClose( AddrFile->Listener );
+            TcpipRecursiveMutexLeave(&TCPLock);
 	    exFreePool( AddrFile->Listener );
     }
     break;
@@ -425,7 +427,9 @@ NTSTATUS FileOpenConnection(
 
   if( !Connection ) return STATUS_NO_MEMORY;
 
+  TcpipRecursiveMutexEnter(&TCPLock, TRUE);
   Status = TCPSocket( Connection, AF_INET, SOCK_STREAM, IPPROTO_TCP );
+  TcpipRecursiveMutexLeave(&TCPLock);
 
   if( !NT_SUCCESS(Status) ) {
       TCPFreeConnectionEndpoint( Connection );

reactos/lib/drivers/ip/transport/tcp/accept.c

@@ -19,6 +19,8 @@ NTSTATUS TCPServiceListeningSocket( PCONNECTION_ENDPOINT Listener,
     PTA_IP_ADDRESS RequestAddressReturn;
     PTDI_CONNECTION_INFORMATION WhoIsConnecting;
 
+    ASSERT_LOCKED(&TCPLock);
+
     /* Unpack TDI info -- We need the return connection information
      * struct to return the address so it can be filtered if needed
      * by WSAAccept -- The returned address will be passed on to

reactos/lib/drivers/ip/transport/tcp/event.c

@@ -16,6 +16,8 @@ int TCPSocketState(void *ClientData,
            OSK_UINT NewState ) {
     PCONNECTION_ENDPOINT Connection = WhichConnection;
 
+    ASSERT_LOCKED(&TCPLock);
+
     TI_DbgPrint(MID_TRACE,("Flags: %c%c%c%c\n",
                NewState & SEL_CONNECT ? 'C' : 'c',
                NewState & SEL_READ    ? 'R' : 'r',
@@ -42,7 +44,7 @@ int TCPSocketState(void *ClientData,
     Connection->SignalState |= NewState;
     if( !Connection->Signalled ) {
     Connection->Signalled = TRUE;
-    InsertTailList( &SignalledConnections, &Connection->SignalList );
+    ExInterlockedInsertTailList( &SignalledConnectionsList, &Connection->SignalList, &SignalledConnectionsLock );
     }
 
     return 0;
@@ -52,6 +54,8 @@ void TCPPacketSendComplete( PVOID Context,
                 PNDIS_PACKET NdisPacket,
                 NDIS_STATUS NdisStatus ) {
     TI_DbgPrint(DEBUG_TCP,("called %x\n", NdisPacket));
+    ASSERT_LOCKED(&TCPLock);
+
     FreeNdisPacket(NdisPacket);
     TI_DbgPrint(DEBUG_TCP,("done\n"));
 }
@@ -65,6 +69,8 @@ int TCPPacketSend(void *ClientData, OSK_PCHAR data, OSK_UINT len ) {
     IP_ADDRESS RemoteAddress, LocalAddress;
     PIPv4_HEADER Header;
 
+    ASSERT_LOCKED(&TCPLock);
+
     if( *data == 0x45 ) { /* IPv4 */
     Header = (PIPv4_HEADER)data;
     LocalAddress.Type = IP_ADDRESS_V4;
@@ -114,6 +120,8 @@ int TCPSleep( void *ClientData, void *token, int priority, char *msg,
     PSLEEPING_THREAD SleepingThread;
     LARGE_INTEGER Timeout;
 
+    ASSERT_LOCKED(&TCPLock);
+
     TI_DbgPrint(DEBUG_TCP,
         ("Called TSLEEP: tok = %x, pri = %d, wmesg = %s, tmio = %x\n",
          token, priority, msg, tmio));
@@ -159,6 +167,8 @@ void TCPWakeup( void *ClientData, void *token ) {
     PLIST_ENTRY Entry;
     PSLEEPING_THREAD SleepingThread;
 
+    ASSERT_LOCKED(&TCPLock);
+
     TcpipAcquireFastMutex( &SleepingThreadsLock );
     Entry = SleepingThreadsList.Flink;
     while( Entry != &SleepingThreadsList ) {
@@ -223,6 +233,8 @@ void *TCPMalloc( void *ClientData,
     void *v;
     ULONG Signature;
 
+    ASSERT_LOCKED(&TCPLock);
+
 #if 0 != MEM_PROFILE
     static OSK_UINT *Sizes = NULL, *Counts = NULL, ArrayAllocated = 0;
     static OSK_UINT ArrayUsed = 0, AllocationCount = 0;
@@ -298,6 +310,8 @@ void TCPFree( void *ClientData,
           void *data, OSK_PCHAR File, OSK_UINT Line ) {
     ULONG Signature;
 
+    ASSERT_LOCKED(&TCPLock);
+
     UntrackFL( (PCHAR)File, Line, data, FOURCC('f','b','s','d') );
     data = (void *)((char *) data - sizeof(ULONG));
     Signature = *((ULONG *) data);

reactos/lib/drivers/ip/transport/tcp/if.c

@@ -35,6 +35,8 @@ POSK_IFADDR TCPGetInterfaceData( PIP_INTERFACE IF ) {
     POSK_IFADDR ifaddr = IF->TCPContext;
     ASSERT(ifaddr);
 
+    ASSERT_LOCKED(&TCPLock);
+
     RtlZeroMemory(ifaddr, sizeof(OSK_IFADDR) + 2 * sizeof( struct sockaddr_in ));
 
     struct sockaddr_in *addr_in = (struct sockaddr_in *)&ifaddr[1];
@@ -77,6 +79,8 @@ POSK_IFADDR TCPFindInterface( void *ClientData,
     struct sockaddr_in *addr_in = (struct sockaddr_in *)ReqAddr;
     POSK_IFADDR InterfaceData;
 
+    ASSERT_LOCKED(&TCPLock);
+
     TI_DbgPrint(DEBUG_TCPIF,("called for type %d\n", FindType));
 
     if( !ReqAddr ) {

reactos/lib/drivers/ip/transport/tcp/tcp.c

@@ -15,14 +15,14 @@
 LONG TCP_IPIdentification = 0;
 static BOOLEAN TCPInitialized = FALSE;
 static NPAGED_LOOKASIDE_LIST TCPSegmentList;
-LIST_ENTRY SignalledConnections;
+LIST_ENTRY SignalledConnectionsList;
+KSPIN_LOCK SignalledConnectionsLock;
 LIST_ENTRY SleepingThreadsList;
 FAST_MUTEX SleepingThreadsLock;
 RECURSIVE_MUTEX TCPLock;
 PORT_SET TCPPorts;
 
-static VOID HandleSignalledConnection( PCONNECTION_ENDPOINT Connection,
+static VOID HandleSignalledConnection( PCONNECTION_ENDPOINT Connection ) {
-                                       ULONG NewState ) {
     NTSTATUS Status = STATUS_SUCCESS;
     PTCP_COMPLETION_ROUTINE Complete;
     PTDI_BUCKET Bucket;
@@ -30,194 +30,12 @@ static VOID HandleSignalledConnection( PCONNECTION_ENDPOINT Connection,
     PIRP Irp;
     PMDL Mdl;
 
+    ASSERT_LOCKED(&TCPLock);
+
     TI_DbgPrint(MID_TRACE,("Handling signalled state on %x (%x)\n",
                            Connection, Connection->SocketContext));
 
-    /* Things that can happen when we try the initial connection */
+    if( Connection->SignalState & SEL_FIN ) {
-    if( NewState & SEL_CONNECT ) {
-        while( (Entry = ExInterlockedRemoveHeadList( &Connection->ConnectRequest,
-                                                     &Connection->Lock )) != NULL ) {
-            
-            TI_DbgPrint(DEBUG_TCP, ("Connect Event\n"));
-
-            Bucket = CONTAINING_RECORD( Entry, TDI_BUCKET, Entry );
-            Complete = Bucket->Request.RequestNotifyObject;
-            TI_DbgPrint(DEBUG_TCP,
-                        ("Completing Request %x\n", Bucket->Request.RequestContext));
-
-            if( (NewState & (SEL_CONNECT | SEL_FIN)) ==
-                (SEL_CONNECT | SEL_FIN) )
-                Status = STATUS_CONNECTION_REFUSED;
-            else
-                Status = STATUS_SUCCESS;
-
-            Complete( Bucket->Request.RequestContext, Status, 0 );
-
-            /* Frees the bucket allocated in TCPConnect */
-            exFreePool( Bucket );
-        }
-    }
-
-    if( NewState & SEL_ACCEPT ) {
-        /* Handle readable on a listening socket --
-         * TODO: Implement filtering
-         */
-
-        TI_DbgPrint(DEBUG_TCP,("Accepting new connection on %x (Queue: %s)\n",
-                               Connection,
-                               IsListEmpty(&Connection->ListenRequest) ?
-                               "empty" : "nonempty"));
-
-        while( (Entry = ExInterlockedRemoveHeadList( &Connection->ListenRequest,
-                                                     &Connection->Lock )) != NULL ) {
-            PIO_STACK_LOCATION IrpSp;
-
-            Bucket = CONTAINING_RECORD( Entry, TDI_BUCKET, Entry );
-            Complete = Bucket->Request.RequestNotifyObject;
-
-            Irp = Bucket->Request.RequestContext;
-            IrpSp = IoGetCurrentIrpStackLocation( Irp );
-
-            TI_DbgPrint(DEBUG_TCP,("Getting the socket\n"));
-            Status = TCPServiceListeningSocket
-                ( Connection->AddressFile->Listener,
-                  Bucket->AssociatedEndpoint,
-                  (PTDI_REQUEST_KERNEL)&IrpSp->Parameters );
-
-            TI_DbgPrint(DEBUG_TCP,("Socket: Status: %x\n"));
-
-            if( Status == STATUS_PENDING ) {
-                ExInterlockedInsertHeadList( &Connection->ListenRequest, &Bucket->Entry, &Connection->Lock );
-                break;
-            } else {
-                Complete( Bucket->Request.RequestContext, Status, 0 );
-                exFreePool( Bucket );
-            }
-        }
-    }
-
-    /* Things that happen after we're connected */
-    if( NewState & SEL_READ ) {
-        TI_DbgPrint(DEBUG_TCP,("Readable: irp list %s\n",
-                               IsListEmpty(&Connection->ReceiveRequest) ?
-                               "empty" : "nonempty"));
-
-        while( (Entry = ExInterlockedRemoveHeadList( &Connection->ReceiveRequest,
-                                                     &Connection->Lock )) != NULL ) {
-            OSK_UINT RecvLen = 0, Received = 0;
-            PVOID RecvBuffer = 0;
-
-            Bucket = CONTAINING_RECORD( Entry, TDI_BUCKET, Entry );
-            Complete = Bucket->Request.RequestNotifyObject;
-
-            Irp = Bucket->Request.RequestContext;
-            Mdl = Irp->MdlAddress;
-
-            TI_DbgPrint(DEBUG_TCP,
-                        ("Getting the user buffer from %x\n", Mdl));
-
-            NdisQueryBuffer( Mdl, &RecvBuffer, &RecvLen );
-
-            TI_DbgPrint(DEBUG_TCP,
-                        ("Reading %d bytes to %x\n", RecvLen, RecvBuffer));
-
-            TI_DbgPrint(DEBUG_TCP, ("Connection: %x\n", Connection));
-            TI_DbgPrint
-                (DEBUG_TCP,
-                 ("Connection->SocketContext: %x\n",
-                  Connection->SocketContext));
-            TI_DbgPrint(DEBUG_TCP, ("RecvBuffer: %x\n", RecvBuffer));
-
-            Status = TCPTranslateError
-                ( OskitTCPRecv( Connection->SocketContext,
-                                RecvBuffer,
-                                RecvLen,
-                                &Received,
-                                0 ) );
-
-            TI_DbgPrint(DEBUG_TCP,("TCP Bytes: %d\n", Received));
-
-            if( Status == STATUS_SUCCESS ) {
-                TI_DbgPrint(DEBUG_TCP,("Received %d bytes with status %x\n",
-                                       Received, Status));
-
-                Complete( Bucket->Request.RequestContext,
-                          STATUS_SUCCESS, Received );
-                exFreePool( Bucket );
-            } else if( Status == STATUS_PENDING ) {
-                ExInterlockedInsertHeadList
-                    ( &Connection->ReceiveRequest, &Bucket->Entry, &Connection->Lock );
-                break;
-            } else {
-                TI_DbgPrint(DEBUG_TCP,
-                            ("Completing Receive request: %x %x\n",
-                             Bucket->Request, Status));
-                Complete( Bucket->Request.RequestContext, Status, 0 );
-                exFreePool( Bucket );
-            }
-        }
-    }
-    if( NewState & SEL_WRITE ) {
-        TI_DbgPrint(DEBUG_TCP,("Writeable: irp list %s\n",
-                               IsListEmpty(&Connection->SendRequest) ?
-                               "empty" : "nonempty"));
-
-        while( (Entry = ExInterlockedRemoveHeadList( &Connection->SendRequest,
-                                                     &Connection->Lock )) != NULL ) {
-            OSK_UINT SendLen = 0, Sent = 0;
-            PVOID SendBuffer = 0;
-
-            Bucket = CONTAINING_RECORD( Entry, TDI_BUCKET, Entry );
-            Complete = Bucket->Request.RequestNotifyObject;
-
-            Irp = Bucket->Request.RequestContext;
-            Mdl = Irp->MdlAddress;
-
-            TI_DbgPrint(DEBUG_TCP,
-                        ("Getting the user buffer from %x\n", Mdl));
-
-            NdisQueryBuffer( Mdl, &SendBuffer, &SendLen );
-
-            TI_DbgPrint(DEBUG_TCP,
-                        ("Writing %d bytes to %x\n", SendLen, SendBuffer));
-
-            TI_DbgPrint(DEBUG_TCP, ("Connection: %x\n", Connection));
-            TI_DbgPrint
-                (DEBUG_TCP,
-                 ("Connection->SocketContext: %x\n",
-                  Connection->SocketContext));
-
-            Status = TCPTranslateError
-                ( OskitTCPSend( Connection->SocketContext,
-                                SendBuffer,
-                                SendLen,
-                                &Sent,
-                                0 ) );
-
-            TI_DbgPrint(DEBUG_TCP,("TCP Bytes: %d\n", Sent));
-
-            if( Status == STATUS_SUCCESS ) {
-                TI_DbgPrint(DEBUG_TCP,("Sent %d bytes with status %x\n",
-                                       Sent, Status));
-
-                Complete( Bucket->Request.RequestContext,
-                          STATUS_SUCCESS, Sent );
-                exFreePool( Bucket );
-            } else if( Status == STATUS_PENDING ) {
-                ExInterlockedInsertHeadList
-                    ( &Connection->SendRequest, &Bucket->Entry, &Connection->Lock );
-                break;
-            } else {
-                TI_DbgPrint(DEBUG_TCP,
-                            ("Completing Send request: %x %x\n",
-                             Bucket->Request, Status));
-                Complete( Bucket->Request.RequestContext, Status, 0 );
-                exFreePool( Bucket );
-            }
-        }
-    }
-
-    if( NewState & SEL_FIN ) {
         TI_DbgPrint(DEBUG_TCP, ("EOF From socket\n"));
 
         while ((Entry = ExInterlockedRemoveHeadList( &Connection->ReceiveRequest,
@@ -279,8 +97,189 @@ static VOID HandleSignalledConnection( PCONNECTION_ENDPOINT Connection,
 
            Complete( Bucket->Request.RequestContext, STATUS_CANCELLED, 0 );
         }
+
+        Connection->SignalState = 0;
     }
 
+    /* Things that can happen when we try the initial connection */
+    if( Connection->SignalState & SEL_CONNECT ) {
+        while( (Entry = ExInterlockedRemoveHeadList( &Connection->ConnectRequest,
+                                                     &Connection->Lock )) != NULL ) {
+            
+            TI_DbgPrint(DEBUG_TCP, ("Connect Event\n"));
+
+            Bucket = CONTAINING_RECORD( Entry, TDI_BUCKET, Entry );
+            Complete = Bucket->Request.RequestNotifyObject;
+            TI_DbgPrint(DEBUG_TCP,
+                        ("Completing Request %x\n", Bucket->Request.RequestContext));
+
+            Complete( Bucket->Request.RequestContext, STATUS_SUCCESS, 0 );
+
+            /* Frees the bucket allocated in TCPConnect */
+            exFreePool( Bucket );
+        }
+    }
+
+    if( Connection->SignalState & SEL_ACCEPT ) {
+        /* Handle readable on a listening socket --
+         * TODO: Implement filtering
+         */
+
+        TI_DbgPrint(DEBUG_TCP,("Accepting new connection on %x (Queue: %s)\n",
+                               Connection,
+                               IsListEmpty(&Connection->ListenRequest) ?
+                               "empty" : "nonempty"));
+
+        while( (Entry = ExInterlockedRemoveHeadList( &Connection->ListenRequest,
+                                                     &Connection->Lock )) != NULL ) {
+            PIO_STACK_LOCATION IrpSp;
+
+            Bucket = CONTAINING_RECORD( Entry, TDI_BUCKET, Entry );
+            Complete = Bucket->Request.RequestNotifyObject;
+
+            Irp = Bucket->Request.RequestContext;
+            IrpSp = IoGetCurrentIrpStackLocation( Irp );
+
+            TI_DbgPrint(DEBUG_TCP,("Getting the socket\n"));
+            Status = TCPServiceListeningSocket
+                ( Connection->AddressFile->Listener,
+                  Bucket->AssociatedEndpoint,
+                  (PTDI_REQUEST_KERNEL)&IrpSp->Parameters );
+
+            TI_DbgPrint(DEBUG_TCP,("Socket: Status: %x\n"));
+
+            if( Status == STATUS_PENDING ) {
+                ExInterlockedInsertHeadList( &Connection->ListenRequest, &Bucket->Entry, &Connection->Lock );
+                break;
+            } else {
+                Complete( Bucket->Request.RequestContext, Status, 0 );
+                exFreePool( Bucket );
+            }
+        }
+    }
+
+    /* Things that happen after we're connected */
+    if( Connection->SignalState & SEL_READ ) {
+        TI_DbgPrint(DEBUG_TCP,("Readable: irp list %s\n",
+                               IsListEmpty(&Connection->ReceiveRequest) ?
+                               "empty" : "nonempty"));
+
+        while( (Entry = ExInterlockedRemoveHeadList( &Connection->ReceiveRequest,
+                                                     &Connection->Lock )) != NULL ) {
+            OSK_UINT RecvLen = 0, Received = 0;
+            PVOID RecvBuffer = 0;
+
+            Bucket = CONTAINING_RECORD( Entry, TDI_BUCKET, Entry );
+            Complete = Bucket->Request.RequestNotifyObject;
+
+            Irp = Bucket->Request.RequestContext;
+            Mdl = Irp->MdlAddress;
+
+            TI_DbgPrint(DEBUG_TCP,
+                        ("Getting the user buffer from %x\n", Mdl));
+
+            NdisQueryBuffer( Mdl, &RecvBuffer, &RecvLen );
+
+            TI_DbgPrint(DEBUG_TCP,
+                        ("Reading %d bytes to %x\n", RecvLen, RecvBuffer));
+
+            TI_DbgPrint(DEBUG_TCP, ("Connection: %x\n", Connection));
+            TI_DbgPrint
+                (DEBUG_TCP,
+                 ("Connection->SocketContext: %x\n",
+                  Connection->SocketContext));
+            TI_DbgPrint(DEBUG_TCP, ("RecvBuffer: %x\n", RecvBuffer));
+
+            Status = TCPTranslateError
+                ( OskitTCPRecv( Connection->SocketContext,
+                                RecvBuffer,
+                                RecvLen,
+                                &Received,
+                                0 ) );
+
+            TI_DbgPrint(DEBUG_TCP,("TCP Bytes: %d\n", Received));
+
+            if( Status == STATUS_SUCCESS ) {
+                TI_DbgPrint(DEBUG_TCP,("Received %d bytes with status %x\n",
+                                       Received, Status));
+
+                Complete( Bucket->Request.RequestContext,
+                          STATUS_SUCCESS, Received );
+                exFreePool( Bucket );
+            } else if( Status == STATUS_PENDING ) {
+                ExInterlockedInsertHeadList
+                    ( &Connection->ReceiveRequest, &Bucket->Entry, &Connection->Lock );
+                break;
+            } else {
+                TI_DbgPrint(DEBUG_TCP,
+                            ("Completing Receive request: %x %x\n",
+                             Bucket->Request, Status));
+                Complete( Bucket->Request.RequestContext, Status, 0 );
+                exFreePool( Bucket );
+            }
+        }
+    }
+    if( Connection->SignalState & SEL_WRITE ) {
+        TI_DbgPrint(DEBUG_TCP,("Writeable: irp list %s\n",
+                               IsListEmpty(&Connection->SendRequest) ?
+                               "empty" : "nonempty"));
+
+        while( (Entry = ExInterlockedRemoveHeadList( &Connection->SendRequest,
+                                                     &Connection->Lock )) != NULL ) {
+            OSK_UINT SendLen = 0, Sent = 0;
+            PVOID SendBuffer = 0;
+
+            Bucket = CONTAINING_RECORD( Entry, TDI_BUCKET, Entry );
+            Complete = Bucket->Request.RequestNotifyObject;
+
+            Irp = Bucket->Request.RequestContext;
+            Mdl = Irp->MdlAddress;
+
+            TI_DbgPrint(DEBUG_TCP,
+                        ("Getting the user buffer from %x\n", Mdl));
+
+            NdisQueryBuffer( Mdl, &SendBuffer, &SendLen );
+
+            TI_DbgPrint(DEBUG_TCP,
+                        ("Writing %d bytes to %x\n", SendLen, SendBuffer));
+
+            TI_DbgPrint(DEBUG_TCP, ("Connection: %x\n", Connection));
+            TI_DbgPrint
+                (DEBUG_TCP,
+                 ("Connection->SocketContext: %x\n",
+                  Connection->SocketContext));
+
+            Status = TCPTranslateError
+                ( OskitTCPSend( Connection->SocketContext,
+                                SendBuffer,
+                                SendLen,
+                                &Sent,
+                                0 ) );
+
+            TI_DbgPrint(DEBUG_TCP,("TCP Bytes: %d\n", Sent));
+
+            if( Status == STATUS_SUCCESS ) {
+                TI_DbgPrint(DEBUG_TCP,("Sent %d bytes with status %x\n",
+                                       Sent, Status));
+
+                Complete( Bucket->Request.RequestContext,
+                          STATUS_SUCCESS, Sent );
+                exFreePool( Bucket );
+            } else if( Status == STATUS_PENDING ) {
+                ExInterlockedInsertHeadList
+                    ( &Connection->SendRequest, &Bucket->Entry, &Connection->Lock );
+                break;
+            } else {
+                TI_DbgPrint(DEBUG_TCP,
+                            ("Completing Send request: %x %x\n",
+                             Bucket->Request, Status));
+                Complete( Bucket->Request.RequestContext, Status, 0 );
+                exFreePool( Bucket );
+            }
+        }
+    }
+
+    Connection->SignalState = 0;
     Connection->Signalled = FALSE;
 }
 
@@ -288,11 +287,11 @@ VOID DrainSignals() {
     PCONNECTION_ENDPOINT Connection;
     PLIST_ENTRY ListEntry;
 
-    while( !IsListEmpty( &SignalledConnections ) ) {
+    while( (ListEntry = ExInterlockedRemoveHeadList(&SignalledConnectionsList,
-        ListEntry = RemoveHeadList( &SignalledConnections );
+                                                    &SignalledConnectionsLock)) != NULL) {
         Connection = CONTAINING_RECORD( ListEntry, CONNECTION_ENDPOINT,
                                         SignalList );
-        HandleSignalledConnection( Connection, Connection->SignalState );
+        HandleSignalledConnection( Connection );
     }
 }
 
@@ -328,6 +327,8 @@ NTSTATUS TCPSocket( PCONNECTION_ENDPOINT Connection,
                     UINT Family, UINT Type, UINT Proto ) {
     NTSTATUS Status;
 
+    ASSERT_LOCKED(&TCPLock);
+
     TI_DbgPrint(DEBUG_TCP,("Called: Connection %x, Family %d, Type %d, "
                            "Proto %d\n",
                            Connection, Family, Type, Proto));
@@ -480,8 +481,9 @@ NTSTATUS TCPStartup(VOID)
 
     TcpipRecursiveMutexInit( &TCPLock );
     ExInitializeFastMutex( &SleepingThreadsLock );
+    KeInitializeSpinLock( &SignalledConnectionsLock );
     InitializeListHead( &SleepingThreadsList );
-    InitializeListHead( &SignalledConnections );
+    InitializeListHead( &SignalledConnectionsList );
     Status = TCPMemStartup();
     if ( ! NT_SUCCESS(Status) ) {
         return Status;
@@ -493,8 +495,10 @@ NTSTATUS TCPStartup(VOID)
         return Status;
     }
 
+    TcpipRecursiveMutexEnter(&TCPLock, TRUE);
     RegisterOskitTCPEventHandlers( &EventHandlers );
     InitOskitTCP();
+    TcpipRecursiveMutexLeave(&TCPLock);
 
     /* Register this protocol with IP layer */
     IPRegisterProtocol(IPPROTO_TCP, TCPReceive);
@@ -581,6 +585,8 @@ NTSTATUS TCPConnect
 
     TI_DbgPrint(DEBUG_TCP,("TCPConnect: Called\n"));
 
+    ASSERT_LOCKED(&TCPLock);
+
     Status = AddrBuildAddress
         ((PTRANSPORT_ADDRESS)ConnInfo->RemoteAddress,
          &RemoteAddress,
@@ -655,6 +661,8 @@ NTSTATUS TCPDisconnect
   PVOID Context ) {
     NTSTATUS Status;
 
+    ASSERT_LOCKED(&TCPLock);
+
     TI_DbgPrint(DEBUG_TCP,("started\n"));
 
     switch( Flags & (TDI_DISCONNECT_ABORT | TDI_DISCONNECT_RELEASE) ) {
@@ -686,6 +694,8 @@ NTSTATUS TCPClose
 
     TI_DbgPrint(DEBUG_TCP,("TCPClose started\n"));
 
+    ASSERT_LOCKED(&TCPLock);
+
     /* Make our code remove all pending IRPs */
     Connection->State |= SEL_FIN;
     DrainSignals();
@@ -713,6 +723,8 @@ NTSTATUS TCPReceiveData
     TI_DbgPrint(DEBUG_TCP,("Called for %d bytes (on socket %x)\n",
                            ReceiveLength, Connection->SocketContext));
 
+    ASSERT_LOCKED(&TCPLock);
+
     ASSERT_KM_POINTER(Connection->SocketContext);
 
     /* Closing */
@@ -775,6 +787,8 @@ NTSTATUS TCPSendData
     NTSTATUS Status;
     PTDI_BUCKET Bucket;
 
+    ASSERT_LOCKED(&TCPLock);
+
     TI_DbgPrint(DEBUG_TCP,("Called for %d bytes (on socket %x)\n",
                            SendLength, Connection->SocketContext));
 
@@ -852,6 +866,8 @@ NTSTATUS TCPGetSockAddress
     OSK_UI16 LocalPort, RemotePort;
     PTA_IP_ADDRESS AddressIP = (PTA_IP_ADDRESS)Address;
 
+    ASSERT_LOCKED(&TCPLock);
+
     OskitTCPGetAddress
         ( Connection->SocketContext,
           &LocalAddress, &LocalPort,