《UNIX网络编程 卷1》 笔记： 服务器程序设计范式

本节是《UNIX网络编程 卷1》 笔记 的最后一节，我们讨论TCP服务器的设计范式。

为了简单比较不同设计的服务器性能，我们实现了一个用于测试的客户端程序，它使用多进程建立多个连接，每个连接从服务器获取特定的字节数。使用如下：

client <hostname or IPaddr> <port> <#children> <#loops/child> <#bytes/request>

第一个命令行参数是hostname服务器IP，第二个参数port是服务器端口号，第三个参数children是客户要创建的子进程个数，第四个参数loops是每个子进程建立的连接数，最后一个参数bytes是每个连接请求的字节数。

在实际测试中，我执行如下的命令测试（没有好的测试环境）。
./client 127.0.0.1 9877 5 5000 9216
客户程序会向服务器发起25000次请求（被端口号范围限制，可修改net.ipv4.ip_local_port_range变量），每次请求获取9216字节的数据。

客户端的代码如下：

#define MAXLINE 4096#define MAXN 16384int main(int argc, char **argv){int i, j, fd, nchildren, nloops, nbytes;pid_t pid;ssize_t n;char request[MAXLINE], reply[MAXN];if (argc != 6)err_quit("usage: client <hostname or IPaddr> <port> <#children>"" <#loops/child> <#bytes/request>");nchildren = atoi(argv[3]); /*子进程个数*/nloops = atoi(argv[4]); /*每个子进程建立的连接数*/nbytes = atoi(argv[5]); /*每个连接从服务器请求的字节数*/snprintf(request, sizeof(request), "%d\n", nbytes);for (i = 0; i < nchildren; i++) {if ((pid = Fork()) == 0) {for (j = 0; j < nloops; j++) {fd = Tcp_connect(argv[1], argv[2]);/*发送请求的字节数*/Write(fd, request, strlen(request));/*读取请求的字节数*/if ((n = Readn(fd, reply, nbytes)) != nbytes)err_quit("server returned %d bytes", n);Close(fd);}printf("child %d done\n", i);exit(0);}/*父进程继续循环调用fork()*/}while (wait(NULL) > 0);if (errno != ECHILD)err_sys("wait error");exit(0);}

Tcp_connect函数在名字与地址转换一节中实现。

服务器解析出客户请求的字节数，然后发送相应字节的随机数据，每个服务器都执行的web_child函数代码如下：

#define MAXN 16384void web_child(int sockfd){int ntowrite;ssize_t nread;char line[MAXN], result[MAXN];for ( ; ; ) {/*获取客户请求的字节数*/if ((nread = Readline(sockfd, line, MAXLINE)) == 0)return;ntowrite = atol(line);if ((ntowrite <= 0) || (ntowrite > MAXN))err_quit("client request for %d bytes", ntowrite);/*发送客户请求的字节数*/Writen(sockfd, result, ntowrite);}}

对每个服务器程序，我们为SIGINT信号注册处理函数sig_int，内部调用pr_cpu_time。当客户终止时，在执行服务程序的终端按下Ctrl+C，就可以查看服务器程序使用CPU的用户时间和系统时间。pr_cpu_time函数代码如下：

void pr_cpu_time(void){double user, sys;struct rusage myusage, childusage;if (getrusage(RUSAGE_SELF, &myusage) < 0)err_sys("getrusage error");if (getrusage(RUSAGE_CHILDREN, &childusage) < 0)err_sys("getrusage error");user = (double)myusage.ru_utime.tv_sec + myusage.ru_utime.tv_usec / 1000000.0;user += (double)childusage.ru_utime.tv_sec + myusage.ru_utime.tv_usec / 1000000.0;sys = (double)myusage.ru_stime.tv_sec + myusage.ru_stime.tv_usec / 1000000.0;sys += (double)childusage.ru_stime.tv_sec + myusage.ru_stime.tv_usec / 1000000.0;printf("\nuser time = %g, sys time = %g\n", user, sys);}

接下来我们给出各种不同的TCP服务器设计范式。

一、TCP并发服务器程序，每个客户一个子进程

void sig_chld(int);void sig_int(int);void web_child(int);void pr_cpu_time(void);int main(int argc, char **argv){int listenfd, connfd;pid_t childpid;socklen_t clilen, addrlen;struct sockaddr *cliaddr;if (argc == 2)listenfd = Tcp_listen(NULL, argv[1], &addrlen);else if (argc == 3)listenfd = Tcp_listen(argv[1], argv[2], &addrlen);elseerr_quit("usage: serv01 [ <host> ] <port#>");cliaddr = Malloc(addrlen);Signal(SIGCHLD, sig_chld);Signal(SIGINT, sig_int);for ( ; ; ) {clilen = addrlen;if ((connfd = accept(listenfd, cliaddr, &clilen)) < 0) {if (errno == EINTR)continue;elseerr_sys("accept error");}if ((childpid = Fork()) == 0) {Close(listenfd); /*子进程关闭监听描述符*/web_child(connfd);/*处理完后终止进程*/exit(0);}Close(connfd); /*父进程关闭已连接的描述符*/}}void sig_chld(int signo){pid_t pid;int stat;/*等待所有子进程终止*/while ((pid = waitpid(-1, &stat, WNOHANG)) > 0)return;}void sig_int(int signo){pr_cpu_time();exit(0);}

Tcp_listen函数在名字与地址转换一节中实现。

程序执行结果如下：

[liu@bogon server]$ ./serv01 9877
^C
user time = 0.19197, sys time = 1.90556

二、TCP预先派生子进程服务器程序

每次在请求到来时再创建子进程是比较慢的，我们可以预先创建一些子进程，每个子进程各自accept，请求到来时就可以立即处理。

static int nchildren;static pid_t *pids;long *cptr;void sig_int(int);pid_t child_make(int, int, int);void pr_cpu_time(void);int main(int argc, char **argv){int listenfd, fd, i;socklen_t addrlen;if (argc == 3)listenfd = Tcp_listen(NULL, argv[1], &addrlen);else if (argc == 4)listenfd = Tcp_listen(argv[1], argv[2], &addrlen);elseerr_quit("usage: serv02 [ <host> ] <port#> <#children>");nchildren = atoi(argv[argc - 1]);pids = Calloc(nchildren, sizeof(pid_t));/*创建共享内存*/fd = Open("/dev/zero", O_RDWR, 0);cptr = Mmap(0, nchildren * sizeof(long), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);Close(fd);/*注意文件名必须包含六个'X'*///my_lock_init("/tmp/lock.XXXXXX");//创建子进程for (i = 0; i < nchildren; i++)pids[i] = child_make(i, listenfd, addrlen);Signal(SIGINT, sig_int);for ( ; ; )pause();}void child_main(int i, int listenfd, int addrlen){int connfd;socklen_t clilen;struct sockaddr *cliaddr;cliaddr = Malloc(addrlen);printf("child %ld starting\n", (long)getpid());for ( ; ; ) {clilen = addrlen;//my_lock_wait();connfd = Accept(listenfd, cliaddr, &clilen);//my_lock_release();/*统计处理的连接数*/cptr[i]++;web_child(connfd);Close(connfd);}}pid_t child_make(int i, int listenfd, int addrlen){pid_t pid;/*创建子进程成功后，父进程返回子进程pid*/if ((pid = Fork()) > 0)return pid;/*子进程继续执行*/child_main(i, listenfd, addrlen);}void sig_int(int signo){int i;/*终止所有的子进程，以便统计进程使用CPU的时间*/for (i = 0; i < nchildren; i++)kill(pids[i], SIGTERM);/*等待所有子进程终止*/while (wait(NULL) > 0);if (errno != ECHILD)err_sys("wait error");pr_cpu_time();/*打印出每个子进程处理的连接数*/for (i = 0; i < nchildren; i++)printf("child %d, %ld connections\n", i, cptr[i]);exit(0);}

程序执行结果如下：
[liu@bogon server]$ ./serv02 9877 5
child 28259 starting
child 28260 starting
child 28258 starting
child 28257 starting
child 28256 starting
^C
user time = 0.001998, sys time = 0.003998
child 0, 5026 connections
child 1, 5103 connections
child 2, 5117 connections
child 3, 4679 connections
child 4, 5075 connections

三、TCP预先派生子进程服务器程序，传递描述符

另一种方式是主进程调用accpet返回连接描述符后使用描述符传递技术将该描述符传递给一个空闲的进程处理，并将进程置为忙状态。为了更新子进程的状态，我们在父进程和所有的子进程间创建流管道，空闲的进程处理完后往流管道写入1字节数据，父进程监听流管道读描述符，如果读到一字节数据，置该子进程为空闲状态。

typedef struct {    pid_t child_pid; /*进程号*/    int child_pipefd; /*流管道读描述符*/    int child_status; /*状态*/    long child_count; /*处理请求的数量*/} Child;Child *cptr;static int nchildren;void sig_int(int);pid_t child_make(int, int, int);int main(int argc, char **argv){int listenfd, i, navail, maxfd, nsel, connfd, rc;ssize_t n;fd_set rset, masterset;socklen_t addrlen, clilen;struct sockaddr *cliaddr;if (argc == 3)listenfd = Tcp_listen(NULL, argv[1], &addrlen);else if (argc == 4)listenfd = Tcp_listen(argv[1], argv[2], &addrlen);elseerr_quit("usage: serv05 [ <host >] <port#> <#children>");FD_ZERO(&masterset);FD_SET(listenfd, &masterset);maxfd = listenfd;cliaddr = Malloc(addrlen);nchildren = atoi(argv[argc - 1]);navail = nchildren;cptr = Calloc(nchildren, sizeof(Child));/*创建子进程，监听监听描述符合所有父子进程之间的流管道一端的描述符*/for (i = 0; i < nchildren; i++) {child_make(i, listenfd, addrlen);FD_SET(cptr[i].child_pipefd, &masterset);maxfd = max(maxfd, cptr[i].child_pipefd);}Signal(SIGINT, sig_int);for ( ; ; ) {rset = masterset;/*如果所有的子进程都在忙状态，则不监听监听描述符*/if (navail <= 0)FD_CLR(listenfd, &rset);/*监听描述符*/nsel = Select(maxfd + 1, &rset, NULL, NULL, NULL);if (FD_ISSET(listenfd, &rset)) {clilen = addrlen;/*等待客户的连接*/connfd = Accept(listenfd, cliaddr, &clilen);/*从所有子进程中选择一个空闲的子进程*/for (i = 0; i < nchildren; i++)if (cptr[i].child_status == 0)break;if (i == nchildren)err_quit("no available children");/*状态为1表示忙状态*/cptr[i].child_status = 1;/*统计该子进程处理的连接数*/cptr[i].child_count++;navail--;/*发送已连接的描述符给子进程*/n = Write_fd(cptr[i].child_pipefd, "", 1, connfd);Close(connfd);if (--nsel == 0)continue;}/*更新子进程的状态，子进程在处理完请求后会写1字节数据到流管道*/for (i = 0; i < nchildren; i++) {if (FD_ISSET(cptr[i].child_pipefd, &rset)) {if ((n = Read(cptr[i].child_pipefd, &rc, 1)) == 0)err_quit("child %d terminated unexpectedly", i);/*进程为空闲状态*/cptr[i].child_status = 0;navail++;if (--nsel == 0)break;}}}}void child_main(int i, int listenfd, int addrlen){char c;int connfd;ssize_t n;printf("child %ld starting\n", (long)getpid());for ( ; ; ) {/*子进程通过流管道读取已建立连接的描述符*/if ((n = Read_fd(STDERR_FILENO, &c, 1, &connfd)) == 0)err_quit("read_fd returned 0");if (connfd < 0)err_quit("no descriptor from read_fd");/*处理客户的请求*/web_child(connfd);Close(connfd);/*写1字节到流管道，表示客户请求已处理完毕*/Write(STDERR_FILENO, "", 1);}}/*创建子进程*/pid_t child_make(int i, int listenfd, int addrlen){int sockfd[2];pid_t pid;/*创建流管道*/Socketpair(AF_LOCAL, SOCK_STREAM, 0, sockfd);/*父进程保存子进程信息*/if ((pid = Fork()) > 0) {Close(sockfd[1]);cptr[i].child_pid = pid;cptr[i].child_pipefd = sockfd[0];cptr[i].child_status = 0;return pid;}/*复制流管道描述符到标准错误输出*/dup2(sockfd[1], STDERR_FILENO);Close(sockfd[0]);Close(sockfd[1]);Close(listenfd);child_main(i, listenfd, addrlen);}void sig_int(int signo){int i;/*终止所有的子进程，以便统计进程使用CPU的时间*/for (i = 0; i < nchildren; i++)kill(cptr[i].child_pid, SIGTERM);/*等待所有子进程终止*/while (wait(NULL) > 0);if (errno != ECHILD)err_sys("wait error");pr_cpu_time();/*打印出每个子进程处理的连接数*/for (i = 0; i < nchildren; i++)printf("child %d, %ld connections\n", i, cptr[i].child_count);exit(0);}

执行结果如下：

[liu@bogon server]$ ./serv05 9877 5
child 21118 starting
child 21119 starting
child 21117 starting
child 21116 starting
child 21115 starting
^C
user time = 0.079986, sys time = 0.929858
child 0, 11529 connections
child 1, 11295 connections
child 2, 946 connections
child 3, 621 connections
child 4, 609 connections

四、TCP并发服务器程序，每个客户一个线程

void sig_int(int);  void* doit(void *);  void web_child(int);    int main(int argc, char **argv)  {  int listenfd, connfd;  pthread_t tid;  socklen_t clilen, addrlen;  struct sockaddr *cliaddr;    if (argc == 2)  listenfd = Tcp_listen(NULL, argv[1], &addrlen);  else if (argc == 3)  listenfd = Tcp_listen(argv[1], argv[2], &addrlen);  else  err_quit("usage: serv06 [ <host> ] <port#>");  cliaddr = Malloc(addrlen);    Signal(SIGINT, sig_int);    for ( ; ; ) {  clilen = addrlen;  connfd = Accept(listenfd, cliaddr, &clilen);  /*为每个客户创建一个线程处理*/  Pthread_create(&tid, NULL, &doit, (void *)connfd);  }  }    void* doit(void *arg)  {  Pthread_detach(pthread_self());  web_child((int)arg);  Close((int)arg);  return NULL;  }    void sig_int(int signo)  {  pr_cpu_time();  exit(0);  }  

执行结果如下：

[liu@bogon server]$ ./serv06 9877
^C
user time = 0.18997, sys time = 1.41578

五、TCP预先创建线程服务器程序，每个线程各自accept

typedef struct {  pthread_t thread_tid; //线程ID  long thread_count; //处理请求的数量  } Thread;    Thread *tptr;    int listenfd, nthreads;  socklen_t addrlen;  pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;    void sig_int(int);  void thread_make(int);    int main(int argc, char **argv)  {  int i;    if (argc == 3)  listenfd = Tcp_listen(NULL, argv[1], &addrlen);  else if (argc == 4)  listenfd = Tcp_listen(argv[1], argv[2], &addrlen);  else  err_quit("usage: serv07 [ <host> ] <port#> <#threads>");  nthreads = atoi(argv[argc - 1]);  tptr = Calloc(nthreads, sizeof(Thread));  /*预先创建多个线程*/  for (i = 0; i < nthreads; i++)  thread_make(i);  Signal(SIGINT, sig_int);    for ( ; ; )  pause();  }    void* thread_main(void *arg)  {  int connfd;  socklen_t clilen;  struct sockaddr *cliaddr;    cliaddr = Malloc(addrlen);    printf("thread %d starting\n", (int)arg);  for ( ; ; ){  clilen = addrlen;  Pthread_mutex_lock(&lock);  connfd = Accept(listenfd, cliaddr, &clilen);  Pthread_mutex_unlock(&lock);  tptr[(int)arg].thread_count++;    web_child(connfd);  Close(connfd);  }  }    void thread_make(int i)  {  Pthread_create(&tptr[i].thread_tid, NULL, &thread_main, (void *)i);  }    void sig_int(int signo)  {  int i;    pr_cpu_time();    for (i = 0; i < nthreads; i++)  printf("thread %d, %ld connections\n", i, tptr[i].thread_count);  exit(0);  }

执行结果如下：

[liu@bogon server]$ ./serv07 9877 5
thread 3 starting
thread 4 starting
thread 2 starting
thread 1 starting
thread 0 starting
^C
user time = 0.159974, sys time = 1.85372
thread 0, 5026 connections
thread 1, 5042 connections
thread 2, 4788 connections
thread 3, 5139 connections
thread 4, 5005 connections

六、TCP预先创建线程服务器程序，主线程统一accept

#define MAXNCLI     32    typedef struct {  pthread_t thread_tid;  long thread_count;  } Thread;  Thread *tptr;    static int nthreads;  int clifd[MAXNCLI], iget, iput;  pthread_mutex_t clifd_mutex = PTHREAD_MUTEX_INITIALIZER;  pthread_cond_t clifd_cond = PTHREAD_COND_INITIALIZER;    void sig_int(int);  void thread_make(int);    int main(int argc, char **argv)  {  int i, listenfd, connfd;  socklen_t addrlen, clilen;  struct sockaddr *cliaddr;    if (argc == 3)  listenfd = Tcp_listen(NULL, argv[1], &addrlen);  else if (argc == 4)  listenfd = Tcp_listen(argv[1], argv[2], &addrlen);  else  err_quit("usage: serv08 [ <host> ] <port#> <#threads>");  cliaddr = Malloc(addrlen);    nthreads = atoi(argv[argc - 1]);  tptr = Calloc(nthreads, sizeof(Thread));  iget = iput = 0;    for (i = 0; i < nthreads; i++)  thread_make(i);    Signal(SIGINT, sig_int);    for ( ; ; ) {  clilen = addrlen;  connfd = Accept(listenfd, cliaddr, &clilen);  /*主线程将描述符放入数组中，子线程从数组中获取*/  Pthread_mutex_lock(&clifd_mutex);  clifd[iput] = connfd;  if (++iput == MAXNCLI)  iput = 0;  if (iput == iget)  err_quit("put = iget = %d", iput);  Pthread_cond_signal(&clifd_cond);  Pthread_mutex_unlock(&clifd_mutex);  }  }    void* thread_main(void *arg)  {  int connfd;    printf("thread %d starting\n", (int)arg);  for ( ; ; ) {  Pthread_mutex_lock(&clifd_mutex);  /*等待有新的描述符产生*/  while (iget == iput)  Pthread_cond_wait(&clifd_cond, &clifd_mutex);  connfd = clifd[iget];  if (++iget == MAXNCLI)  iget = 0;  Pthread_mutex_unlock(&clifd_mutex);  tptr[(int)arg].thread_count++;    web_child(connfd);  Close(connfd);  }  }    void thread_make(int i)  {  Pthread_create(&tptr[i].thread_tid, NULL, &thread_main, (void *)i);  return;  }    void sig_int(int signo)  {  int i;    pr_cpu_time();    for (i = 0; i < nthreads; i++)  printf("thread %d, %ld connections\n", i, tptr[i].thread_count);  exit(0);  }  

执行结果如下：

[liu@bogon server]$ ./serv08 9877 5
thread 3 starting
thread 4 starting
thread 2 starting
thread 1 starting
thread 0 starting
^C
user time = 0.243962, sys time = 1.03969
thread 0, 4886 connections
thread 1, 5118 connections
thread 2, 4952 connections
thread 3, 5122 connections
thread 4, 4922 connections