6 The Completion Port Model

6.1.1 APIs
1) HANDLE CreateIoCompletionPort(
     HANDLE FileHandle,
     HANDLE ExistingCompletionPort,
     DWORD CompletionKey,
     DWORD NumberOfConcurrentThreads);
Before examining the parameters in detail, be aware that this function is actually used for two distinct purposes: 1.To create a completion port object; 2.To associate a handle with a completion port.
2) typedef struct _PER_HANDLE_DATA
{
 SOCKET   Socket;
 SOCKADDR_STORAGE  ClientAddr;
 // Other information useful to be associated with the handle
} PER_HANDLE_DATA, * LPPER_HANDLE_DATA;
3) typedef struct
{
    OVERLAPPED Overlapped;
    char       Buffer[DATA_BUFSIZE];
    int  BufferLen;
    int        OperationType;
} PER_IO_DATA;
4) BOOL GetQueuedCompletionStatus(
    HANDLE CompletionPort,
    LPDWORD lpNumberOfBytesTransferred,
    PULONG_PTR lpCompletionKey,
    LPOVERLAPPED * lpOverlapped,
    DWORD dwMilliseconds);

6.1.2 Using steps
The following example demonstrates how to start developing an echo server application using the completion port model. In this code, we take the following preparation steps:
1. Create a completion port. The fourth parameter is left as 0, specifying that only one worker thread per processor will be allowed to execute at a time on the completion port.
2. Determine how many processors exist on the system.
3. Create worker threads to service completed I/O requests on the completion port using processor information in step 2. In the case of this simple example, we create one worker thread per processor because we do not expect our threads to ever get in a suspended condition in which there would not be enough threads to execute for each processor. When the CreateThread function is called, you must supply a worker routine that the thread executes upon creation. We will discuss the worker thread's responsibilities later in this section.
4. Prepare a listening socket to listen for connections on port 5150.
5. Accept inbound connections using the accept function.
6. Create a data structure to represent per-handle data and save the accepted socket handle in the structure.
7. Associate the new socket handle returned from accept with the completion port by calling CreateIoCompletionPort. Pass the per-handle data structure to CreateIoCompletionPort via the completion key parameter.
8. Start processing I/O on the accepted connection. Essentially, you want to post one or more asynchronous WSARecv or WSASend requests on the new socket using the overlapped I/O mechanism. When these I/O requests complete, a worker thread services the I/O requests and continues processing future I/O requests, as we will see later in the worker routine specified in step 3.
9. Repeat steps 5–8 until server terminates.
6.1.3 Samples
HANDLE CompletionPort;
WSADATA wsd;
SYSTEM_INFO SystemInfo;
SOCKADDR_IN InternetAddr;
SOCKET Listen;
int i;

// Load Winsock
StartWinsock(MAKEWORD(2,2), &wsd);

// Step 1:
// Create an I/O completion port

CompletionPort = CreateIoCompletionPort(
    INVALID_HANDLE_VALUE, NULL, 0, 0);

// Step 2:
// Determine how many processors are on the system

GetSystemInfo(&SystemInfo);

// Step 3:
// Create worker threads based on the number of
// processors available on the system. For this
// simple case, we create one worker thread for each
// processor.

for(i = 0; i < SystemInfo.dwNumberOfProcessors; i++)
{
    HANDLE ThreadHandle;

    // Create a server worker thread, and pass the
    // completion port to the thread. NOTE: the
    // ServerWorkerThread procedure is not defined
    // in this listing.

    ThreadHandle = CreateThread(NULL, 0,
        ServerWorkerThread, CompletionPort,
        0, NULL);

    // Close the thread handle
    CloseHandle(ThreadHandle);
}

// Step 4:
// Create a listening socket

Listen = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0,
    WSA_FLAG_OVERLAPPED);

InternetAddr.sin_family = AF_INET;
InternetAddr.sin_addr.s_addr = htonl(INADDR_ANY);
InternetAddr.sin_port = htons(5150);
bind(Listen, (PSOCKADDR) &InternetAddr,
    sizeof(InternetAddr));

// Prepare socket for listening

listen(Listen, 5);

while(TRUE)
{
    PER_HANDLE_DATA *PerHandleData=NULL;
    SOCKADDR_IN saRemote;
    SOCKET Accept;
    int RemoteLen;
    // Step 5:
    // Accept connections and assign to the completion
    // port

    RemoteLen = sizeof(saRemote);
    Accept = WSAAccept(Listen, (SOCKADDR *)&saRemote,
    &RemoteLen);

    // Step 6:
    // Create per-handle data information structure to
    // associate with the socket
    PerHandleData = (LPPER_HANDLE_DATA)
        GlobalAlloc(GPTR, sizeof(PER_HANDLE_DATA));

    printf("Socket number %d connected\n", Accept);
    PerHandleData->Socket = Accept;
    memcpy(&PerHandleData->ClientAddr, &saRemote, RemoteLen);

    // Step 7:
    // Associate the accepted socket with the
    // completion port

    CreateIoCompletionPort((HANDLE) Accept,
        CompletionPort, (DWORD) PerHandleData, 0);

    // Step 8:
    //  Start processing I/O on the accepted socket.
    //  Post one or more WSASend() or WSARecv() calls
    //  on the socket using overlapped I/O.
    WSARecv(...);
}

DWORD WINAPI ServerWorkerThread(LPVOID lpParam)
{
    // The requirements for the worker thread will be
    // discussed later.
return 0;
}
DWORD WINAPI ServerWorkerThread(
    LPVOID CompletionPortID)
{
    HANDLE CompletionPort = (HANDLE) CompletionPortID;
    DWORD BytesTransferred;
    LPOVERLAPPED Overlapped;
    LPPER_HANDLE_DATA PerHandleData;
    LPPER_IO_DATA PerIoData;
    DWORD SendBytes, RecvBytes;
    DWORD Flags;
   
    while(TRUE)
    {
        // Wait for I/O to complete on any socket
        // associated with the completion port
   
        ret = GetQueuedCompletionStatus(CompletionPort,
            &BytesTransferred,(LPDWORD)&PerHandleData,
            (LPOVERLAPPED *) &PerIoData, INFINITE);

        // First check to see if an error has occurred
        // on the socket; if so, close the
        // socket and clean up the per-handle data
        // and per-I/O operation data associated with
        // the socket

        if (BytesTransferred == 0 &&
            (PerIoData->OperationType == RECV_POSTED ││
             PerIoData->OperationType == SEND_POSTED))
        {
            // A zero BytesTransferred indicates that the
            // socket has been closed by the peer, so
            // you should close the socket. Note:
            // Per-handle data was used to reference the
            // socket associated with the I/O operation.
 
            closesocket(PerHandleData->Socket);

            GlobalFree(PerHandleData);
            GlobalFree(PerIoData);
            continue;
        }

 PerIoData = CONTAINING_RECORD(lpOverlapped, PER_IO_DATA, Overlapped);

        // Service the completed I/O request. You can
        // determine which I/O request has just
        // completed by looking at the OperationType
        // field contained in the per-I/O operation data.
         if (PerIoData->OperationType == RECV_POSTED)
        {
            // Do something with the received data
            // in PerIoData->Buffer
        }

        // Post another WSASend or WSARecv operation.
        // As an example, we will post another WSARecv()
        // I/O operation.

        Flags = 0;

        // Set up the per-I/O operation data for the next
        // overlapped call
        ZeroMemory(&(PerIoData->Overlapped),
            sizeof(OVERLAPPED));

        PerIoData->DataBuf.len = DATA_BUFSIZE;
        PerIoData->DataBuf.buf = PerIoData->Buffer;
        PerIoData->OperationType = RECV_POSTED;

        WSARecv(PerHandleData->Socket,
            &(PerIoData->DataBuf), 1, &RecvBytes,
            &Flags, &(PerIoData->Overlapped), NULL);
    }
}