socket效率到底如何

两台服务器用千兆交换机连接，带宽为1000Mbps，socket的效率到底如何？若server尽量快向client写数据，client收到数据后就丢弃，是否能占满千兆带宽？

测试发现和每次发送的包大小有关系，TCP包为1000bits（125字节）以上就能占满带宽：

因此，若client足够多而且都在请求数据，但是带宽上不去，就是服务器程序的问题了。

协议本身会降低带宽，使用rtmp协议的server一个进程带宽上限估计在260Mbps（若使用阻塞模式只能到150Mbps左右），能服务570个客户端（码流为500kbps），或者能服务870个客户端（300kbps），再有新的client过来会有延时就不行了。不过使用多进程，这台server还可以多启动进程，这样就可以将带宽占满，或者和FMS一样服务其他的vhost。

rtmp默认的chunk size为128，若使用这个尺寸发包，server只能到260Mbps（见上）。但将ChunkSize设为1100以上（和表中一样），带宽就能到945Mbps，能占满千兆网所有带宽。(rtmp server对rtmp client或tcp client都能占满带宽)

估计rtmp协议的效率如下，假设客户端的缓冲区设为10秒：
1. 选择视频为11Mbps时，只需要100个client即可跑满1000Mbps带宽，实际430Mbps（估计Server效率不行）。CPU很低只有14%。vmstat显示cs在12459，in为14439，比较高。内存使用1.8G。首次加载时间稍稍变慢需要2秒左右，seek后加载速度明显变慢需要10秒。预计CPU在18%左右。
2. 选择视频为900Kbps时，需要1000个客户端跑满1000Mbps带宽。实际上只能跑到300Mbps。CPU很低只有4%。vmstat显示cs在11360，in为1943，比较高。内存使用165M。首次加载时间和seek时间不变。预计CPU在72%左右。
3. 实际上单个进程不可能跑满945Mbps带宽，那个时候client的延迟已经很大了。2000个300Kbps的客户端，跑到500Mbps左右，单个进程，保持每个client都不延迟，已经是很好的设计了。支持高并发，必须要单线程和异步，使用多线程会显著增加开销（非线性）。
4. printf打日志对系统负载有影响，若打出的日志很多，会有影响。程序保持高性能，就是不block同时尽量不做和效率无关的事情。

测试程序如下：

// simple-server: send data as fast as possible.#include <stdio.h>#include <stdlib.h>#include <assert.h>#include <iostream>using namespace std;#include <unistd.h>#include <signal.h>#include <sys/types.h>#include <sys/socket.h>#include <netinet/in.h>#include <sys/wait.h>#include <sys/epoll.h>#include <string.h>#include <errno.h>#define err_exit(msg) cout << "[error] " << msg << endl; exit(1)struct UserOptions{    int port;    int packet_size;};void discovery_user_options(int argc, char** argv, UserOptions& options){    if(argc <= 2){        cout << "Usage: " << argv[0] << " <port> <packet_size>" << endl            << "port: the port to listen" << endl            << "packet_size: the packet size in b. must >= 8" << endl            << "For example: " << argv[0] << " 1990 1000" << endl;        exit(1);    }        options.port = atoi(argv[1]);    options.packet_size = atoi(argv[2]);    assert(options.port > 0);    assert(options.packet_size >= 8);}int listen_server_socket(UserOptions& options){    int serverfd = socket(AF_INET, SOCK_STREAM, 0);        if(serverfd == -1){        err_exit("init socket error!");    }    cout << "init socket success! #" << serverfd << endl;        int reuse_socket = 1;    if(setsockopt(serverfd, SOL_SOCKET, SO_REUSEADDR, &reuse_socket, sizeof(int)) == -1){        err_exit("setsockopt reuse-addr error!");    }    cout << "setsockopt reuse-addr success!" << endl;        sockaddr_in addr;    addr.sin_family = AF_INET;    addr.sin_port = htons(options.port);    addr.sin_addr.s_addr = INADDR_ANY;    if(bind(serverfd, (const sockaddr*)&addr, sizeof(sockaddr_in)) == -1){        err_exit("bind socket error!");    }    cout << "bind socket success!" << endl;        if(listen(serverfd, 10) == -1){        err_exit("listen socket error!");    }    cout << "listen socket success! " << options.port << endl;        return serverfd;}int main(int argc, char** argv){    UserOptions options;    discovery_user_options(argc, argv, options);    int serverfd = listen_server_socket(options);        while(true){        int client = accept(serverfd, NULL, 0);        alarm(60);        cout << "get a client #" << client << ", packet size=" << options.packet_size << "b" << endl;        if(client == -1){            err_exit("accept client socket error!");        }                int size = options.packet_size / 8;        char* data = new char[size];        while(true){            if(send(client, data, size, 0) == -1){                close(client);                cout <<  "send client error!" << endl;                break;            }        }    }        return -1;}

// simple-client recv data and delete it.#include <stdio.h>#include <stdlib.h>#include <iostream>#include <assert.h>using namespace std;#include <unistd.h>#include <signal.h>#include <sys/types.h>#include <sys/socket.h>#include <netinet/in.h>#include <arpa/inet.h>#include <string.h>#define err_exit(msg) cout << "[error] " << msg << endl; exit(1)struct UserOptions{    char* server_ip;    int port;};void discovery_user_options(int argc, char** argv, UserOptions& options){    if(argc <= 2){        cout << "Usage: " << argv[0] << " <server_ip> <port>" << endl            << "server_ip: the ip address of server" << endl            << "port: the port to connect at" << endl            << "For example: " << argv[0] << " 192.168.100.145 1990" << endl;        exit(1);    }        options.server_ip = argv[1];    options.port = atoi(argv[2]);    assert(options.port > 0);}int connect_server_socket(UserOptions& options){    int clientfd = socket(AF_INET, SOCK_STREAM, 0);        if(clientfd == -1){        err_exit("init socket error!");    }    cout << "init socket success!" << endl;        sockaddr_in addr;    addr.sin_family = AF_INET;    addr.sin_port = htons(options.port);    addr.sin_addr.s_addr = inet_addr(options.server_ip);    if(connect(clientfd, (const sockaddr*)&addr, sizeof(sockaddr_in)) == -1){        err_exit("connect socket error!");    }    cout << "connect socket success!" << endl;        return clientfd;}int main(int argc, char** argv){    UserOptions options;    discovery_user_options(argc, argv, options);        int client = connect_server_socket(options);    int size = 100 /*kbps*/ * 1000 / 8;    char* buf = new char[size];    while(true){        if(recv(client, buf, size, 0) <= 0){            close(client);            err_exit("client recv error!");        }    }    return 0;}