Libevent源码分析（七）--- IOCP

关于iocp模型，网上有很多资料，这里不详细分析，下面这篇文章写的非常详细：
完成端口(CompletionPort)详解 - 手把手教你玩转网络编程系列之三

event_base中有一个iocp变量，event_base的初始化函数中会调用event_base_start_iocp开启iocp功能，event_base_start_iocp又会调用event_iocp_port_launch来初始化IOCP：

#ifdef WIN32    if (cfg && (cfg->flags & EVENT_BASE_FLAG_STARTUP_IOCP))        event_base_start_iocp(base, cfg->n_cpus_hint);#endifintevent_base_start_iocp(struct event_base *base, int n_cpus){#ifdef WIN32    if (base->iocp)        return 0;    base->iocp = event_iocp_port_launch(n_cpus);    if (!base->iocp) {        event_warnx("%s: Couldn't launch IOCP", __func__);        return -1;    }    return 0;#else    return -1;#endif

iocp指向一个event_iocp_port结构体：

struct event_iocp_port {    /** The port itself */    HANDLE port;    /* A lock to cover internal structures. */    CRITICAL_SECTION lock;    /** Number of threads ever open on the port. */    short n_threads;    /** True iff we're shutting down all the threads on this port */    short shutdown;    /** How often the threads on this port check for shutdown and other     * conditions */    long ms;    /* The threads that are waiting for events. */    HANDLE *threads;    /** Number of threads currently open on this port. */    short n_live_threads;    /** A semaphore to signal when we are done shutting down. */    HANDLE *shutdownSemaphore;};

其中port使iocp的端口，shutdown，lock和shutdownSemaphore用于关闭iocp。ms是GetQueuedCompletionStatus的等待时间，threads是线程句柄，n_threads代表线程数量，n_live_threads代表当前没有关闭的线程。
event_iocp_port的初始化在event_iocp_port_launch函数中进行：

struct event_iocp_port *event_iocp_port_launch(int n_cpus){    struct event_iocp_port *port;    int i;    if (!extension_fns_initialized)        init_extension_functions(&the_extension_fns);    if (!(port = mm_calloc(1, sizeof(struct event_iocp_port))))        return NULL;    if (n_cpus <= 0)        n_cpus = N_CPUS_DEFAULT;    port->n_threads = n_cpus * 2;    port->threads = mm_calloc(port->n_threads, sizeof(HANDLE));    if (!port->threads)        goto err;    port->port = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0,            n_cpus);    port->ms = -1;    if (!port->port)        goto err;    port->shutdownSemaphore = CreateSemaphore(NULL, 0, 1, NULL);    if (!port->shutdownSemaphore)        goto err;    for (i=0; i<port->n_threads; ++i) {        ev_uintptr_t th = _beginthread(loop, 0, port);        if (th == (ev_uintptr_t)-1)            goto err;        port->threads[i] = (HANDLE)th;        ++port->n_live_threads;    }    InitializeCriticalSectionAndSpinCount(&port->lock, 1000);    return port;err:    if (port->port)        CloseHandle(port->port);    if (port->threads)        mm_free(port->threads);    if (port->shutdownSemaphore)        CloseHandle(port->shutdownSemaphore);    mm_free(port);    return NULL;}

event_iocp_port_launch会获取iocp的扩展函数库，然后创建iocp的端口和用于关闭iocp的信号量，iocp还需要一个线程池，线程池大小为cpu数量的2倍。

voidevent_overlapped_init(struct event_overlapped *o, iocp_callback cb){    memset(o, 0, sizeof(struct event_overlapped));    o->cb = cb;}static voidhandle_entry(OVERLAPPED *o, ULONG_PTR completion_key, DWORD nBytes, int ok){    struct event_overlapped *eo =        EVUTIL_UPCAST(o, struct event_overlapped, overlapped);    eo->cb(eo, completion_key, nBytes, ok);}static voidloop(void *_port){    struct event_iocp_port *port = _port;    long ms = port->ms;    HANDLE p = port->port;    if (ms <= 0)        ms = INFINITE;    while (1) {        OVERLAPPED *overlapped=NULL;        ULONG_PTR key=0;        DWORD bytes=0;        int ok = GetQueuedCompletionStatus(p, &bytes, &key,            &overlapped, ms);        EnterCriticalSection(&port->lock);        if (port->shutdown) {            if (--port->n_live_threads == 0)                ReleaseSemaphore(port->shutdownSemaphore, 1,                        NULL);            LeaveCriticalSection(&port->lock);            return;        }        LeaveCriticalSection(&port->lock);        if (key != NOTIFICATION_KEY && overlapped)            handle_entry(overlapped, key, bytes, ok);        else if (!overlapped)            break;    }    event_warnx("GetQueuedCompletionStatus exited with no event.");    EnterCriticalSection(&port->lock);    if (--port->n_live_threads == 0)        ReleaseSemaphore(port->shutdownSemaphore, 1, NULL);    LeaveCriticalSection(&port->lock);}

loop是线程池运行的函数，他通过调用GetQueuedCompletionStatus监听iocp中的事件。ms在初始化中被设置为－1，无限等待。正常轻快下GetQueuedCompletionStatus中的key会返回0，如果返回－1即NOTIFICATION_KEY证明调用了event_iocp_notify_all函数从而调用了PostQueuedCompletionStatus导致GetQueuedCompletionStatus立刻返回。这时需要关闭线程，结束iocp。当loop检测到事件之后会调用handle_entry来调用对应的回调。
event_overlapped的定义如下：

struct event_overlapped {    OVERLAPPED overlapped;    iocp_callback cb;};

该结构体的第一个变量OVERLAPPED就是投入读写事件时传入的OVERLAPPED，根据GetQueuedCompletionStatus的返回可以获得对应的OVERLAPPED，从而可以根据EVUTIL_UPCAST获得event_overlapped结构体，调用对应的回调。

看完iocp的工作流程，接下来看bufferevent_asyn是如何使用iocp的。想要试用bufferevent_asyn，首先要调用bufferevent_async_new：

struct bufferevent *bufferevent_async_new(struct event_base *base,    evutil_socket_t fd, int options){    struct bufferevent_async *bev_a;    struct bufferevent *bev;    struct event_iocp_port *iocp;    options |= BEV_OPT_THREADSAFE;    if (!(iocp = event_base_get_iocp(base)))        return NULL;    if (fd >= 0 && event_iocp_port_associate(iocp, fd, 1)<0) {        int err = GetLastError();        /* We may have alrady associated this fd with a port.         * Let's hope it's this port, and that the error code         * for doing this neer changes. */        if (err != ERROR_INVALID_PARAMETER)            return NULL;    }    if (!(bev_a = mm_calloc(1, sizeof(struct bufferevent_async))))        return NULL;    bev = &bev_a->bev.bev;    if (!(bev->input = evbuffer_overlapped_new(fd))) {        mm_free(bev_a);        return NULL;    }    if (!(bev->output = evbuffer_overlapped_new(fd))) {        evbuffer_free(bev->input);        mm_free(bev_a);        return NULL;    }    if (bufferevent_init_common(&bev_a->bev, base, &bufferevent_ops_async,        options)<0)        goto err;    evbuffer_add_cb(bev->input, be_async_inbuf_callback, bev);    evbuffer_add_cb(bev->output, be_async_outbuf_callback, bev);    event_overlapped_init(&bev_a->connect_overlapped, connect_complete);    event_overlapped_init(&bev_a->read_overlapped, read_complete);    event_overlapped_init(&bev_a->write_overlapped, write_complete);    bev_a->ok = fd >= 0;    if (bev_a->ok)        _bufferevent_init_generic_timeout_cbs(bev);    return bev;err:    bufferevent_free(&bev_a->bev.bev);    return NULL;}

参数中的fd可以是有效的套接字，也可以是－1(之后在be_async_ctrl中指定)，event_iocp_port_associate用于把套接字关联到iocp的端口上。evbuffer_overlapped_new创建一个evbuffer_overlapped结构体并且返回一个evbuffer：

struct evbuffer_overlapped {    struct evbuffer buffer;    /** The socket that we're doing overlapped IO on. */    evutil_socket_t fd;    /** pending I/O type */    unsigned read_in_progress : 1;    unsigned write_in_progress : 1;    /** The first pinned chain in the buffer. */    struct evbuffer_chain *first_pinned;    /** How many chains are pinned; how many of the fields in buffers     * are we using. */    int n_buffers;    WSABUF buffers[MAX_WSABUFS];};struct evbuffer *evbuffer_overlapped_new(evutil_socket_t fd){    struct evbuffer_overlapped *evo;    evo = mm_calloc(1, sizeof(struct evbuffer_overlapped));    if (!evo)        return NULL;    TAILQ_INIT(&evo->buffer.callbacks);    evo->buffer.refcnt = 1;    evo->buffer.last_with_datap = &evo->buffer.first;    evo->buffer.is_overlapped = 1;    evo->fd = fd;    return &evo->buffer;}

evbuffer_overlapped结构体可以由evbuffer通过upcast_evbuffer函数得到。libevent大量使用这种方法隐藏复杂实现，只给开发者简单的通用的结构，内部则通过类型转换获得真正的结构体。evbuffer_overlapped结构体中read_in_progress和write_in_progress用来标记读写事件是否已经投递。buffers是iocp投递读写时用到的数据缓存。
继续看bufferevent_async_new函数，be_async_inbuf_callback和be_async_outbuf_callback函数被设置为读写evbuffer的callback。connect_complete，read_complete和write_complete分别被设置为connect_overlapped，read_overlapped和write_overlapped的回调。接下来就看一下这些回调：

static voidbe_async_outbuf_callback(struct evbuffer *buf,    const struct evbuffer_cb_info *cbinfo,    void *arg){    struct bufferevent *bev = arg;    struct bufferevent_async *bev_async = upcast(bev);    /* If we added data to the outbuf and were not writing before,     * we may want to write now. */    _bufferevent_incref_and_lock(bev);    if (cbinfo->n_added)        bev_async_consider_writing(bev_async);    _bufferevent_decref_and_unlock(bev);}be_async_outbuf_callback是output的回调，当有数据写入output（如果设置阀值需要达到阀值）时会调用此函数。该函数会调用bev_async_consider_writing进行写事件的投递：

static void
bev_async_consider_writing(struct bufferevent_async *beva)
{
size_t at_most;
int limit;
struct bufferevent *bev = &beva->bev.bev;

/* Don't write if there's a write in progress, or we do not * want to write, or when there's nothing left to write. */if (beva->write_in_progress || beva->bev.connecting)    return;if (!beva->ok || !(bev->enabled&EV_WRITE) ||    !evbuffer_get_length(bev->output)) {    bev_async_del_write(beva);    return;}at_most = evbuffer_get_length(bev->output);/* This is safe so long as bufferevent_get_write_max never returns * more than INT_MAX.  That's true for now. XXXX */limit = (int)_bufferevent_get_write_max(&beva->bev);if (at_most >= (size_t)limit && limit >= 0)    at_most = limit;if (beva->bev.write_suspended) {    bev_async_del_write(beva);    return;}/*  XXXX doesn't respect low-water mark very well. */bufferevent_incref(bev);if (evbuffer_launch_write(bev->output, at_most,    &beva->write_overlapped)) {    bufferevent_decref(bev);    beva->ok = 0;    _bufferevent_run_eventcb(bev, BEV_EVENT_ERROR);} else {    beva->write_in_progress = at_most;    _bufferevent_decrement_write_buckets(&beva->bev, at_most);    bev_async_add_write(beva);}

}

bev_async_consider_writing会判断当前是否已经有数据在投递，如果没有则设置一个合理的at_most值然后调用evbuffer_launch_write：

int
evbuffer_launch_write(struct evbuffer *buf, ev_ssize_t at_most,
struct event_overlapped *ol)
{
struct evbuffer_overlapped *buf_o = upcast_evbuffer(buf);
int r = -1;
int i;
struct evbuffer_chain *chain;
DWORD bytesSent;

if (!buf) {    /* No buffer, or it isn't overlapped */    return -1;}EVBUFFER_LOCK(buf);EVUTIL_ASSERT(!buf_o->read_in_progress);if (buf->freeze_start || buf_o->write_in_progress)    goto done;if (!buf->total_len) {    /* Nothing to write */    r = 0;    goto done;} else if (at_most < 0 || (size_t)at_most > buf->total_len) {    at_most = buf->total_len;}evbuffer_freeze(buf, 1);buf_o->first_pinned = NULL;buf_o->n_buffers = 0;memset(buf_o->buffers, 0, sizeof(buf_o->buffers));chain = buf_o->first_pinned = buf->first;for (i=0; i < MAX_WSABUFS && chain; ++i, chain=chain->next) {    WSABUF *b = &buf_o->buffers[i];    b->buf = (char*)( chain->buffer + chain->misalign );    _evbuffer_chain_pin(chain, EVBUFFER_MEM_PINNED_W);    if ((size_t)at_most > chain->off) {        /* XXXX Cast is safe for now, since win32 has no           mmaped chains.  But later, we need to have this           add more WSAbufs if chain->off is greater than           ULONG_MAX */        b->len = (unsigned long)chain->off;        at_most -= chain->off;    } else {        b->len = (unsigned long)at_most;        ++i;        break;    }}buf_o->n_buffers = i;_evbuffer_incref(buf);if (WSASend(buf_o->fd, buf_o->buffers, i, &bytesSent, 0,    &ol->overlapped, NULL)) {    int error = WSAGetLastError();    if (error != WSA_IO_PENDING) {        /* An actual error. */        pin_release(buf_o, EVBUFFER_MEM_PINNED_W);        evbuffer_unfreeze(buf, 1);        evbuffer_free(buf); /* decref */        goto done;    }}buf_o->write_in_progress = 1;r = 0;

done:
EVBUFFER_UNLOCK(buf);
return r;
}

evbuffer_launch_write将evbuffer中的数据和WSABUF做一个映射，之后调用WSASend进行发送，次函数的参数ol是bufferevent_async的write_overlapped结构体，WSASend函数中传入的就是write_overlapped结构体中的overlapped变量：

static void
handle_entry(OVERLAPPED *o, ULONG_PTR completion_key, DWORD nBytes, int ok)
{
struct event_overlapped *eo =
EVUTIL_UPCAST(o, struct event_overlapped, overlapped);
eo->cb(eo, completion_key, nBytes, ok);
}

这样当handle_entry处理回调时就会找到write_overlapped结构体，从而找到write_complete回调：

static void
write_complete(struct event_overlapped *eo, ev_uintptr_t key,
ev_ssize_t nbytes, int ok)
{
struct bufferevent_async *bev_a = upcast_write(eo);
struct bufferevent *bev = &bev_a->bev.bev;
short what = BEV_EVENT_WRITING;
ev_ssize_t amount_unwritten;

BEV_LOCK(bev);EVUTIL_ASSERT(bev_a->write_in_progress);amount_unwritten = bev_a->write_in_progress - nbytes;evbuffer_commit_write(bev->output, nbytes);bev_a->write_in_progress = 0;if (amount_unwritten)    _bufferevent_decrement_write_buckets(&bev_a->bev,                                         -amount_unwritten);if (!ok)    bev_async_set_wsa_error(bev, eo);if (bev_a->ok) {    if (ok && nbytes) {        BEV_RESET_GENERIC_WRITE_TIMEOUT(bev);        if (evbuffer_get_length(bev->output) <=            bev->wm_write.low)            _bufferevent_run_writecb(bev);        bev_async_consider_writing(bev_a);    } else if (!ok) {        what |= BEV_EVENT_ERROR;        bev_a->ok = 0;        _bufferevent_run_eventcb(bev, what);    } else if (!nbytes) {        what |= BEV_EVENT_EOF;        bev_a->ok = 0;        _bufferevent_run_eventcb(bev, what);    }}_bufferevent_decref_and_unlock(bev);

}

write_complete首先调用evbuffer_commit_write来处理evbuffer中的数据，然后判断是否需要调用bufferevent的写事件回调活着事件回调，同时如果evbuffer中还有数据需要再次投递。读事件和连接事件的流程和写事件的流程比较类似，这里不在分析，注意的是当使用IOCP模式时，bufferevent的读写事件不再又event_base管理，而是直接使用iocp。当投递读写事件活着连接事件时会调用event_base_add_virtual，当事件完成时则会调用event_base_del_virtual，这些事件对event_base来说是虚拟的，因为它们是iocp负责管理的。但是当iocp统计事件时同样会将他们计算在内。

static void
bev_async_del_write(struct bufferevent_async *beva)
{
struct bufferevent *bev = &beva->bev.bev;

if (beva->write_added) {    beva->write_added = 0;    event_base_del_virtual(bev->ev_base);}

}

static void
bev_async_del_read(struct bufferevent_async *beva)
{
struct bufferevent *bev = &beva->bev.bev;

if (beva->read_added) {    beva->read_added = 0;    event_base_del_virtual(bev->ev_base);}

}

static void
bev_async_add_write(struct bufferevent_async *beva)
{
struct bufferevent *bev = &beva->bev.bev;

if (!beva->write_added) {    beva->write_added = 1;    event_base_add_virtual(bev->ev_base);}

}

static void
bev_async_add_read(struct bufferevent_async *beva)
{
struct bufferevent *bev = &beva->bev.bev;

if (!beva->read_added) {    beva->read_added = 1;    event_base_add_virtual(bev->ev_base);}

}
“`
libevent的源码分析至此告一段落，另外libevent的http和dns在这里不在分析，它们都是建立在之前分析的源码基础上的应用，如果后面有时间会补上这部分的内容。