单进程支持高并发

来源：互联网发布：vmware for mac版下载编辑：程序博客网时间：2024/04/27 15:17

学习nginx，很好奇如何做到单个进程支持几万并发（当时用3万client连接过去，都能保持连接）。

在linux服务器上：

不使用select，因为fd_set限制为1024，只能有1024个连接，而且慢。
使用epoll，man epoll可以看到资料。
尽管使用了epoll，但是超过1024连接时，还是会有too many open files的错误，每个连接都是一个文件句柄，所以要修改这个限制。
以root运行程序前，修改这个限制：
g++ server2.cpp -o nginx.winlin
sudo su
ulimit -n 65535
./nginx.winlin 1985 65535 > /dev/null 2>&1
模拟客户端：
g++ client.cpp -o client
sudo su
ulimit -n 65535
./client 10000 10.33.0.190 1985 > /dev/null 2>&1 &
统计连接：
sudo bash status.sh 10 1985
nginx-origin:0 master:23429 worker:23430 listen-1985:(24584/nginx.winlin) all:24836 established:23531

这样可以打开几万个连接：

clients:20001 established:20000

演示代码如下：

#include<sys/types.h>#include<sys/socket.h>#include<errno.h>#include<string.h>#include<sys/ioctl.h>#include<netinet/in.h>#include<sys/epoll.h>#include<iostream>using namespace std;int main(int argc, char** argv){    if(argc <= 2){        cout << "Usage: " << argv[0] << " <port> <max_clients>" << endl            << "  port: the listening port." << endl            << "  max_clients: the max accept clients." << endl;        return -1;    }        int port = atoi(argv[1]);    int max_clients = atoi(argv[2]);    cout << "port:" << port << " max_clients:" << max_clients << " pid:" << getpid() << endl;        int server_socket = socket(PF_INET, SOCK_STREAM, 0);    if(server_socket == -1){        cout << "init socket error: " << strerror(errno) << endl;        exit(-1);    }    cout << "init socket success: fd=" << server_socket << "!" << endl;        int flag;    if(ioctl(server_socket, FIONBIO, &flag) == -1){        cout << "set socket to un-blocked error: " << strerror(errno) << endl;        exit(-1);    }    cout << "set socket to non-block mode success!" << endl;        sockaddr_in addr;    memset(&addr, 0, sizeof(sockaddr_in));    addr.sin_family = AF_INET;    addr.sin_port = htons(port);    // bind    if(bind(server_socket, (const sockaddr*)&addr, sizeof(sockaddr)) == -1){        cout << "socket bind error: " << strerror(errno) << endl;        exit(-1);    }    cout << "bind socket success!" << endl;        if(listen(server_socket, 10) == -1){        cout << "socket listen error: " << strerror(errno) << endl;        exit(-1);    }    cout << "listen socket success!" << endl;        int epoll = epoll_create(max_clients);    if(epoll == -1){        cout << "create event poll error: " << strerror(errno) << endl;        exit(-1);    }    cout << "create event poll success: fd=" << epoll << "!" << endl;        if(true){        epoll_event ev;        memset(&ev, 0, sizeof(epoll_event));        ev.events = EPOLLIN | EPOLLOUT; // focus on read|write event.        ev.data.fd = server_socket;        if(epoll_ctl(epoll, EPOLL_CTL_ADD, server_socket, &ev) == -1){            cout << "event poll ctl error: " << strerror(errno) << endl;            exit(-1);        }        cout << "event poll ctl success!" << endl;    }        epoll_event* ee = new epoll_event[max_clients];    for(;;){        int active_fds = epoll_wait(epoll, ee, max_clients, 100);        if(active_fds == -1){            cout << "event poll wait error: " << strerror(errno) << endl;            exit(-1);        }        cout << "event poll wait success: active_fds=" << active_fds << "!" << endl;                for(int i = 0; i < active_fds; i++){            // if client is coming.            if(ee[i].data.fd == server_socket){                int client = accept(server_socket, (sockaddr*)NULL, NULL);                                if(client == -1){                    cout << "accept client error: " << strerror(errno) << endl;                    exit(-1);                }                cout << "accept client success: client=" << client << "!" << endl;                                epoll_event ev;                memset(&ev, 0, sizeof(epoll_event));                ev.events = EPOLLIN;                ev.data.fd = client;                if(epoll_ctl(epoll, EPOLL_CTL_ADD, client, &ev) == -1){                    cout << "client event poll ctl error: " << strerror(errno) << endl;                    exit(-1);                }                cout << "client event poll ctl success!" << endl;            }            // do read-write            else{                int client = ee[i].data.fd;                                // read                char buffer[1024];                int len;                if((len = read(client, buffer, sizeof(buffer))) <= 0){                    cout << "read from client error: " << strerror(errno) << endl;                    exit(-1);                }                cout << "read from client success: " << len << "bytes received!" << endl;                                // write                char msg[] = "server is ok!";                if(send(client, msg, sizeof(msg), 0) <= 0){                    cout << "write to client error: " << strerror(errno) << endl;                    exit(-1);                }                cout << "write to client success!" << endl;            }        }    }        cout << "server cleanup and exit." << endl;    close(epoll);    close(server_socket);        return 0;}

模拟的client，定时发送HTTP请求：

#include<signal.h>#include<errno.h>#include<sys/socket.h>#include<netinet/in.h>#include<arpa/inet.h>#include<iostream>#include<string>#include<sstream>using namespace std;void on_signal_active(int sno){    cout << "get a signal: " << sno        << ", SIGINT=" << SIGINT << ", SIGTERM=" << SIGTERM << endl;    if(sno == SIGINT || sno == SIGTERM){        cout << "get a exit signal, exit" << endl;        exit(0);    }}class Client{private:    int client;    public:    Client(){        client = 0;    }    ~Client(){        if(client > 0){            close(client);        }    }        bool initialize(int id, char* host, int port){        cout << "#" << id << ":" << "client start..." << endl;        cout << "#" << id << ":" << "register the signal function" << endl;        signal(SIGINT, on_signal_active);        signal(SIGTERM, on_signal_active);        client = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);        if(client < 0){            cout << "#" << id << ":" << "init socket error!" << "msg: " << strerror(errno) << endl;            return false;        }        cout << "#" << id << ":" << "socket init success" << endl;        sockaddr_in server_address;        memset(&server_address, 0, sizeof(server_address));        server_address.sin_family = AF_INET;        server_address.sin_addr.s_addr = inet_addr(host);        server_address.sin_port = htons(port);        if(connect(client, (sockaddr*)&server_address, sizeof(sockaddr_in)) < 0){            cout << "#" << id << ":" << "connect to server error." << "msg: " << strerror(errno) << endl;            return false;        }                cout << "#" << id << ":" << "socket connect success" << endl;        return true;    }            bool run(int id, char* host){        // HTTP request        if(true){            stringstream ss(stringstream::in | stringstream::out);            string s0x20 = string(1, (char)0x20);            string sCRLF = string(1, (char)0x0D) + string(1, (char)0x0A);            ss << string("GET") << s0x20 // Command: GET 47 45 54 20                  << string("/index.html") << s0x20 // URI: /  2F 20                 << string("HTTP/1.1") << sCRLF // ProtocolVersion: HTTP/1.1 48 54 54 50 2F 31 2E 31 0D 0A                << string("Host:") << s0x20 << string(host) << sCRLF // Host:www.baidu.com.. 48 6F 73 74 3A 20 77 77 77 2E 62 61 69 64 75 2E 63 6F 6D 0D 0A                 << string("Connection:") << s0x20 << string("keep-alive") << sCRLF // Connection: keep-alive..   43 6F 6E 6E 65 63 74 69 6F 6E 3A 20 6B 65 65 70 2D 61 6C 69 76 65 0D 0A                << string("User-Agent:") << s0x20 << string("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.162 Safari/535.19") << sCRLF // UserAgent:  Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.162 Safari/535.19                << string("Accept:") << s0x20 << string("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") << sCRLF//Accept:  text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8                << string("Accept-Encoding:") << s0x20 << string("gzip,deflate,sdch") << sCRLF//Accept-Encoding:  gzip,deflate,sdch                << string("Accept-Language:") << s0x20 << string("zh-CN,zh;q=0.8") << sCRLF//Accept-Language:  zh-CN,zh;q=0.8                << string("Accept-Charset:") << s0x20 << string("GBK,utf-8;q=0.7,*;q=0.3") << sCRLF//Accept-Charset:  GBK,utf-8;q=0.7,*;q=0.3                << string("Cookie:") << s0x20 << string("BAIDUID=3BABFCA8602431FA83A9AFBE4FBD6F85:FG=1;") << s0x20 << string("MCITY=-158%3A") << sCRLF//Cookie:  BAIDUID=3BABFCA8602431FA83A9AFBE4FBD6F85:FG=1; MCITY=-158%3A                // << string("Key:") << s0x20 << string("Value") << sCRLF //Key: Value                << sCRLF; //HeaderEnd: CRLF                        ss.seekg(0, ios::end);            int len = ss.tellg();            ss.seekg(0, ios::beg);                        char* buffer = new char[len];            ss.read(buffer, len);                        int ret = send(client, buffer, len, 0);            delete buffer;                        if(ret <= 0){                cout << "send HTTP request error" << endl;                return false;            }                        cout << "send HTTP request success" << endl;        }                // HTTP response        if(true){            char buffer[1024];            memset(buffer, 0, sizeof(buffer));                        if(read(client, buffer, sizeof(buffer)) <= 0){                cout << "receive HTTP response error" << "msg: " << strerror(errno)  << endl;                return false;            }                        cout << "receive HTTP reponse success" << "msg: " << strerror(errno)  << endl;        }                return true;    }};int main(int argc, char** argv){    int i = 0;    int id = 0;        if(argc <= 3){        cout << "Usage: " << argv[0] << " <client_count> <host> <port>" << endl            << "  client_count: the client count to run." << endl            << "  host: the host to connect to." << endl            << "  port: the port to connect to." << endl;        return -1;    }    int client_count = atoi(argv[1]);    char* host = argv[2];    int port = atoi(argv[3]);        while(true){        Client clients[client_count];                for(int j = 0; j < client_count; j++){            Client& client = clients[j];                        if(!client.initialize(++i, host, port)){                goto error;            }        }                while(true){            for(int j = 0; j < client_count; j++){                Client& client = clients[j];                if(!client.run(++id, host)){                    goto error;                }            }        }error:                        cout << "server error, reinitialize all " << client_count << " clients" << endl;        usleep(10 * 1000 * 1000);    }        return 0;}

统计连接脚本：

#!/bin/bash#vi /bin/status##################################################################################################脚本if [ $# -lt 2 ]; then    echo "Usage: $0 <sleep_time> <port>";    echo "    <sleep_time> the sleep time in seconds.";    echo "    <port> to display the program which listening at the specified port.";    echo "    e.g. $0 3 80";    exit -1;fisleep_time=$1port=$2echo "sleep_time:${sleep_time}s"for((i=0;;i++))do    program_listening=`netstat -ntlp|grep ${port}|awk -F "LISTEN" '{print $2}'|awk '{print $1}'`    if [ -z "${program_listening}" ]; then        program_listening="None";    fi    nginx_master_pid=`ps aux|grep nginx|grep master|awk '{print $2}'`    if [ -z "${nginx_master_pid}" ]; then        nginx_master_pid="0";    fi    nginx_origin_pid=`ps aux|grep nginx|grep bin|awk '{print $2}'`    if [ -z "${nginx_origin_pid}" ]; then        nginx_origin_pid=`ps aux|grep nginx|grep winlin|awk '{print $2}'`        if [ -z "${nginx_origin_pid}" ]; then            nginx_origin_pid="0";        fi    fi    if [ ${nginx_origin_pid} == ${nginx_master_pid} ]; then        # if origin equals to master, the origin is master actually.        nginx_origin_pid="0";    fi    nginx_worker_pid=`ps aux|grep nginx|grep worker|awk '{print $2}'`    if [ -z "${nginx_worker_pid}" ]; then        nginx_worker_pid="0";    fi    local_ip_address=`/sbin/ifconfig|grep "inet\ "|awk 'NR==1 {print $2}'|awk -F ':' '{print $2}'`    established_clients=`netstat -nt|grep ESTABLISHED|grep "0\ ${local_ip_address}:${port}\ "|wc -l`    all_clients=`netstat -nat|grep "0\ ${local_ip_address}:${port}\ "|wc -l`    echo "nginx-origin:${nginx_origin_pid} master:${nginx_master_pid} worker:${nginx_worker_pid} listen-${port}:(${program_listening}) all:${all_clients} established:${established_clients}"    sleep ${sleep_time}done