《UNIX网络编程 卷2》 笔记： 使用内存映射I/O实现消息队列

本节是《UNIX网络编程 卷2》 笔记 的最后一节，我们使用内存映射I/O实现Posix消息队列。之所以将这节内容放到最后是因为这个实现比较复杂，要完全弄清楚它的原理除需要掌握之前讲述的一些内容外还要有链表这种数据结构的知识储备。这个实现的具体思想和上节实现信号量类似：把消息队列写入一个文件中，然后使用mmap函数映射该文件实现消息队列在多个进程间共享。下图是用这种方法实现的一个消息队列（该消息队列中有4个消息，每个消息的长度是7字节）使用的各种数据结构布局：

从上图我们可以看到一个消息队列包含消息队列头和多个消息。消息队列头（mq_hdr结构）中包含消息队列属性（mymq_attr结构）、互斥锁、条件变量和其他维护队列中消息的信息。每个消息中包含消息头（msg_hdr结构）和数据。每次创建或打开一个消息队列时，我们返回一个mq_info结构，这个结构中包含一个指向消息队列的指针。

初始时，消息队列中所有的消息都是空闲状态，也就是说这个消息中不包含有效数据。当放置一个消息到消息队列中时，我们挑选一个空闲状态的消息，把数据放入，这条消息的状态就变成非空闲的了。上图各个数据结构的定义如下：

#define MQI_MAGIC0x98765432#define MSGSIZE(i) ((((i) + sizeof(long)-1) / sizeof(long)) * sizeof(long))typedef struct mq_info *mymqd_t;/*消息队列属性*/struct mymq_attr {long mq_flags; /*标志，O_NONBLOCK等*/long mq_maxmsg; /*最大消息数*/long mq_msgsize; /*一个消息最大长度*/long mq_curmsgs; /*队列中当前消息的数量*/};/*消息队列头结构*/struct mq_hdr {struct mymq_attr mqh_attr; /*消息队列属性*/long mqh_head; /*第一个非空闲消息的偏移*/long mqh_free; /*第一个空闲消息的偏移*/long mqh_nwait; /*等待获取消息的线程数量*/pid_t mqh_pid; /*注册接收消息队列异步事件通知的进程ID*/struct sigevent mqh_event; pthread_mutex_t mqh_lock;pthread_cond_t mqh_wait;};/*消息头结构*/struct msg_hdr {long msg_next; /*下一个消息的偏移*/ssize_t msg_len; /*长度*/unsigned int msg_prio; /*优先级*/};/*消息队列结构*/struct mq_info {struct mq_hdr *mqi_hdr;long mqi_magic;int mqi_flags;};

MSGSIZE宏保证消息的长度按长字对齐。

创建或打开一个消息队列的实现函数如下：

#define MAX_TRIES 10struct mymq_attr defattr = {0, 128, 1024, 0};mymqd_t mymq_open(const char *pathname, int oflag, ...){int i, fd, nonblock, created, save_errno;long msgsize, filesize, index;va_list ap;mode_t mode;char *mptr;struct stat statbuff;struct mq_hdr *mqhdr;struct msg_hdr *msghdr;struct mymq_attr *attr;struct mq_info *mqinfo;pthread_mutexattr_t mattr;pthread_condattr_t cattr;created = 0;nonblock = oflag & O_NONBLOCK;oflag &= ~O_NONBLOCK;mptr = (char *)MAP_FAILED;mqinfo = NULL;again:if (oflag & O_CREAT) {va_start(ap, oflag);mode = va_arg(ap, mode_t) & ~S_IXUSR;attr = va_arg(ap, struct mymq_attr *);va_end(ap);fd = open(pathname, oflag | O_EXCL | O_RDWR, mode | S_IXUSR);if (fd < 0) {if (errno == EEXIST && (oflag & O_EXCL) == 0)goto exists;elsereturn (mymqd_t)-1;}created = 1;if (attr == NULL)attr = &defattr;else {/*参数错误*/if (attr->mq_maxmsg <= 0 || attr->mq_msgsize <= 0) {errno = EINVAL;goto err;}}/*消息大小对齐*/msgsize = MSGSIZE(attr->mq_msgsize);filesize = sizeof(struct mq_hdr) + (attr->mq_maxmsg * (sizeof(struct msg_hdr) + msgsize));/*设置文件大小*/if (lseek(fd, filesize - 1, SEEK_SET) == -1)goto err;if (write(fd, "", 1) == -1)goto err;/*映射文件到共享内存*/mptr = mmap(NULL, filesize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);if (mptr == MAP_FAILED)goto err;if ((mqinfo = malloc(sizeof(struct mq_info))) == NULL)goto err;/**/mqinfo->mqi_hdr = mqhdr = (struct mq_hdr *)mptr;mqinfo->mqi_magic = MQI_MAGIC;mqinfo->mqi_flags = nonblock;/*初始化消息队列头*/mqhdr->mqh_attr.mq_flags = 0;mqhdr->mqh_attr.mq_maxmsg = attr->mq_maxmsg;mqhdr->mqh_attr.mq_msgsize= attr->mq_msgsize;mqhdr->mqh_attr.mq_curmsgs = 0;mqhdr->mqh_nwait = 0;mqhdr->mqh_pid = 0;/*第一个非空闲消息的偏移*/mqhdr->mqh_head = 0;index = sizeof(struct mq_hdr);/*第一个空闲消息的偏移*/mqhdr->mqh_free = index;/*初始化所有的消息头*/for (i = 0; i < attr->mq_maxmsg - 1; i++) {msghdr = (struct msg_hdr *)&mptr[index];index += sizeof(struct msg_hdr) + msgsize;/*下一个消息头在文件中的偏移*/msghdr->msg_next = index;}/*最后一个消息头*/msghdr = (struct msg_hdr *)&mptr[index];msghdr->msg_next = 0;/*初始化互斥锁*/if ((i = pthread_mutexattr_init(&mattr)) != 0)goto pthreaderr;pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);i = pthread_mutex_init(&mqhdr->mqh_lock, &mattr);pthread_mutexattr_destroy(&mattr);if (i != 0)goto pthreaderr;/*初始化条件变量*/if ((i = pthread_condattr_init(&cattr)) != 0)goto pthreaderr;pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED);i = pthread_cond_init(&mqhdr->mqh_wait, &cattr);pthread_condattr_destroy(&cattr);if (i != 0)goto pthreaderr;/*取消文件S_IXUSR位，表明完成初始化*/if (fchmod(fd, mode) == -1)goto err;close(fd);return (mymqd_t)mqinfo;}exists:if ((fd = open(pathname, O_RDWR)) < 0) {if (errno == ENOENT && (oflag & O_CREAT))goto again;goto err;}/*等待消息队列完成初始化*/for (i = 0; i < MAX_TRIES; i++) {if (stat(pathname, &statbuff) == -1) {if (errno == ENOENT && (oflag & O_CREAT)) {close(fd);goto again;}goto err;}if ((statbuff.st_mode & S_IXUSR) == 0)break;sleep(1);}if (i == MAX_TRIES) {errno = ETIMEDOUT;goto err;}/*映射文件到共享内存*/filesize = statbuff.st_size;mptr = mmap(NULL, filesize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);if (mptr == MAP_FAILED)goto err;close(fd);/*初始化*/if ((mqinfo = malloc(sizeof(struct mq_info))) == NULL)goto err;mqinfo->mqi_hdr = mqhdr = (struct mq_hdr *)mptr;mqinfo->mqi_magic = MQI_MAGIC;mqinfo->mqi_flags = nonblock;return (mymqd_t)mqinfo;pthreaderr:errno = i;err:save_errno = errno;if (created)unlink(pathname);if (mptr != MAP_FAILED)munmap(mptr, filesize);if (mqinfo != NULL)free(mqinfo);close(fd);errno = save_errno;return (mymqd_t)-1;}/*包裹函数*/mymqd_t Mymq_open(const char *pathname, int oflag, ...){    va_list ap;    mode_t mode;    struct mymq_attr *attr;        va_start(ap, oflag);    mode = va_arg(ap, mode_t);    attr = va_arg(ap, struct mymq_attr *);    va_end(ap);    if (mymq_open(pathname, oflag, mode, attr) == (mymqd_t)-1)        err_sys("mymq_open error");}

这个函数的流程与上节mysem_open函数类似。初始化完成后：

    所有的空闲消息被组织成链表的形式。

    第一个空闲消息的偏移是消息队列头的大小（sizeof(struct mq_hdr)）。

    第一个非空闲消息的偏移是0表示队列中没有非空闲消息。

    如果没有指定消息队列属性，则默认的消息队列最多放置128个消息，每个消息的最大长度是1024字节。

关闭消息队列和删除消息队列的实现函数如下：

int mymq_close(mymqd_t mqd){long msgsize, filesize;struct mq_hdr *mqhdr;struct mymq_attr *attr;struct mq_info *mqinfo;mqinfo = mqd;if (mqinfo->mqi_magic != MQI_MAGIC) {errno = EBADF;return -1;}mqhdr = mqinfo->mqi_hdr;attr = &mqhdr->mqh_attr;/*解除进程注册的消息通知*/if (mymq_notify(mqd, NULL) != 0)return -1;msgsize = MSGSIZE(attr->mq_msgsize);filesize = sizeof(struct mq_hdr) + (attr->mq_maxmsg * (sizeof(struct msg_hdr) + msgsize));/*解除文件映射*/if (munmap(mqinfo->mqi_hdr, filesize) == -1)return -1;mqinfo->mqi_magic = 0;/*释放内存*/free(mqinfo);return 0;}int mymq_unlink(const char *pathname){if (unlink(pathname) == -1)return -1;return 0;}

获取消息队列的属性和设置消息队列的属性的实现函数如下：

int mymq_getattr(mymqd_t mqd, struct mymq_attr *mqstat){int n;struct mq_hdr *mqhdr;struct mymq_attr *attr;struct mq_info *mqinfo;mqinfo = mqd;if (mqinfo->mqi_magic != MQI_MAGIC) {errno = EBADF;return -1;}mqhdr = mqinfo->mqi_hdr;attr = &mqhdr->mqh_attr;if ((n = pthread_mutex_lock(&mqhdr->mqh_lock)) != 0) {errno = n;return -1;}mqstat->mq_flags = mqinfo->mqi_flags;mqstat->mq_maxmsg = attr->mq_maxmsg;mqstat->mq_msgsize = attr->mq_msgsize;mqstat->mq_curmsgs = attr->mq_curmsgs;pthread_mutex_unlock(&mqhdr->mqh_lock);return 0;}int mymq_setattr(mymqd_t mqd, const struct mymq_attr *mqstat, struct mymq_attr *omqstat){int n;struct mq_hdr *mqhdr;struct mymq_attr *attr;struct mq_info *mqinfo;mqinfo = mqd;if (mqinfo->mqi_magic != MQI_MAGIC) {errno = EBADF;return -1;}mqhdr = mqinfo->mqi_hdr;attr = &mqhdr->mqh_attr;if ((n = pthread_mutex_lock(&mqhdr->mqh_lock)) != 0) {errno = n;return -1;}if (omqstat != NULL) {omqstat->mq_flags = mqinfo->mqi_flags;omqstat->mq_maxmsg = attr->mq_maxmsg;omqstat->mq_msgsize = attr->mq_msgsize;omqstat->mq_curmsgs = attr->mq_curmsgs;}if (mqstat->mq_flags & O_NONBLOCK)mqinfo->mqi_flags |= O_NONBLOCK;elsemqinfo->mqi_flags &= ~O_NONBLOCK;pthread_mutex_unlock(&mqhdr->mqh_lock);return 0;}

mq_info结构中的mqi_flags成员表示是以阻塞还是非阻塞的方式访问消息队列。
消息队列实现异步通知机制的函数如下：

int mymq_notify(mymqd_t mqd, const struct sigevent *notification){int n;pid_t pid;struct mq_hdr *mqhdr;struct mq_info *mqinfo;mqinfo = mqd;if (mqinfo->mqi_magic != MQI_MAGIC) {errno = EBADF;return -1;}mqhdr = mqinfo->mqi_hdr;if ((n = pthread_mutex_lock(&mqhdr->mqh_lock)) != 0) {errno = n;return -1;}pid = getpid();if (notification == NULL) {if (mqhdr->mqh_pid == pid)mqhdr->mqh_pid = 0;} else {if (mqhdr->mqh_pid != 0) {if (kill(mqhdr->mqh_pid, 0) != -1 || errno != ESRCH) {errno = EBUSY;goto err;}}mqhdr->mqh_pid = pid;mqhdr->mqh_event = *notification;}pthread_mutex_unlock(&mqhdr->mqh_lock);return 0;err:pthread_mutex_unlock(&mqhdr->mqh_lock);return -1;}

放置一个消息到消息队列中的实现函数如下：

int mymq_send(mymqd_t mqd, const char *ptr, size_t len, unsigned int prio){int n;long index, freeindex;char *mptr;struct sigevent *sigev;struct mq_hdr *mqhdr;struct mymq_attr *attr;struct msg_hdr *msghdr, *nmsghdr, *pmsghdr;struct mq_info *mqinfo;mqinfo = mqd;if (mqinfo->mqi_magic != MQI_MAGIC) {errno = EBADF;return -1;}mqhdr = mqinfo->mqi_hdr;mptr = (char *)mqhdr;attr = &mqhdr->mqh_attr;if ((n = pthread_mutex_lock(&mqhdr->mqh_lock)) != 0) {errno = n;return -1;}if (len > attr->mq_msgsize) {errno = EMSGSIZE;goto err;}if (attr->mq_curmsgs == 0) { /*消息队列为空*/if (mqhdr->mqh_pid != 0 && mqhdr->mqh_nwait == 0) {sigev = &mqhdr->mqh_event;/*给注册接收异步信号通知的进程发送信号*/if (sigev->sigev_notify == SIGEV_SIGNAL) {sigqueue(mqhdr->mqh_pid, sigev->sigev_signo, sigev->sigev_value);}mqhdr->mqh_pid = 0;}} else if (attr->mq_curmsgs >= attr->mq_maxmsg) { /*消息队列已满*//*非阻塞*/if (mqinfo->mqi_flags & O_NONBLOCK) {errno = EAGAIN;goto err;}/*阻塞，睡眠等待*/while (attr->mq_curmsgs >= attr->mq_maxmsg)pthread_cond_wait(&mqhdr->mqh_wait, &mqhdr->mqh_lock);}if ((freeindex = mqhdr->mqh_free) == 0)err_dump("mq_send: curmsgs = %ld; free = 0", attr->mq_curmsgs);/*放置消息*/nmsghdr = (struct msg_hdr *)&mptr[freeindex];nmsghdr->msg_prio = prio;nmsghdr->msg_len = len;memcpy(nmsghdr + 1, ptr, len);/*更新空闲消息头偏移*/mqhdr->mqh_free = nmsghdr->msg_next;/*根据优先级将该消息放置正确的位置*/index = mqhdr->mqh_head;/*这里如果队列为空，那么pmsghdr->msg_next = freeindx;相当于mqhdr->mqh_head = freeindx*/pmsghdr = (struct msg_hdr *)&(mqhdr->mqh_head);while (index != 0) {msghdr = (struct msg_hdr *)&mptr[index];if (prio > msghdr->msg_prio) {nmsghdr->msg_next = index;pmsghdr->msg_next = freeindex;break;}index = msghdr->msg_next;pmsghdr = msghdr;}/*消息队列为空或者该消息放在队列尾部*/if (index == 0) {pmsghdr->msg_next = freeindex;nmsghdr->msg_next = 0;}/*唤醒等待接收消息的进程*/if (attr->mq_curmsgs == 0)pthread_cond_signal(&mqhdr->mqh_wait);attr->mq_curmsgs++;pthread_mutex_unlock(&mqhdr->mqh_lock);return 0;err:pthread_mutex_unlock(&mqhdr->mqh_lock);return -1;}

比较关键的是第59行到第77行的代码，它根据消息的优先级把消息插入非空闲消息链表中合适的位置。

从消息队列中取出一个消息的实现函数如下：

ssize_t mymq_receive(mymqd_t mqd, char *ptr, size_t maxlen, unsigned int *priop){int n;long index;char *mptr;ssize_t len;struct mq_hdr *mqhdr;struct mymq_attr *attr;struct msg_hdr *msghdr;struct mq_info *mqinfo;mqinfo = mqd;if (mqinfo->mqi_magic != MQI_MAGIC) {errno = EBADF;return -1;}mqhdr = mqinfo->mqi_hdr;mptr = (char *)mqhdr;attr = &mqhdr->mqh_attr;if ((n = pthread_mutex_lock(&mqhdr->mqh_lock)) != 0) {errno = n;return -1;}if (maxlen < attr->mq_msgsize) {errno = EMSGSIZE;goto err;}if (attr->mq_curmsgs == 0) { /*消息队列为空*//*非阻塞*/if (mqinfo->mqi_flags & O_NONBLOCK) {errno = EAGAIN;goto err;}/*阻塞等待消息*/mqhdr->mqh_nwait++;while (attr->mq_curmsgs == 0)pthread_cond_wait(&mqhdr->mqh_wait, &mqhdr->mqh_lock);mqhdr->mqh_nwait--;}if ((index = mqhdr->mqh_head) == 0)err_dump("mq_receive: curmsgs = %ld; head = 0", attr->mq_curmsgs);msghdr = (struct msg_hdr *)&mptr[index];/*更新消息队列头*/mqhdr->mqh_head = msghdr->msg_next;/*获取消息*/len = msghdr->msg_len;memcpy(ptr, msghdr + 1, len);if (priop != NULL)*priop = msghdr->msg_prio;/*该消息(已经变成空闲)msg_next指向下一个空闲消息头*/msghdr->msg_next = mqhdr->mqh_free;/*更新空闲消息队列头*/mqhdr->mqh_free = index;/*唤醒等待发送消息的进程*/if (attr->mq_curmsgs == attr->mq_maxmsg)pthread_cond_signal(&mqhdr->mqh_wait);attr->mq_curmsgs--;pthread_mutex_unlock(&mqhdr->mqh_lock);return len;err:pthread_mutex_unlock(&mqhdr->mqh_lock);return -1;}

可以看到取出的消息是第一个非空闲消息，也就是放置时间最早优先级最高的消息。