Linux LVS高并发测试程序，内核参数设置，连接数查询指令

最近在公司参与了一个LVS系统测试的项目，学习到了一些关于高并发测试相关的知识，写到博客里记录下

Linux内核参数设置

• 在服务器端需要调整系统最大文件句柄数
  ulimit -n 1000000
  在服务器硬件支持，以及服务较轻量的情况下，最大连接数未必只限于100000,视情况而定吧

• 客户端需要发起大量连接，理论上是最多可支持65535个连接，但实际上Linux的默认可用端口号并没有这么多，需要修改
  sudo echo 1024 65534 > /proc/sys/net/ipv4/ip_local_port_range

查看连接个数

我们一开始用的是
netstat -nap | grep 程序名 | grep ESTABLISHED| wc -l
来查询，这个指令当连接数上万后就变得很慢，而且我们发现了一个略坑的问题，通过这组合指令查询到的结果是波动的，我特意试了10000个连接下，在程序没有报有连接关闭的情况下，通过netstat查出来的连接个数在9998到10000之间波动。最后我换了个指令
ss state ESTABLISHED dport = :端口号 and dst IP | wc -l
通过ss指令查询，不仅速度很快，而且不会出现计数不准的情况，需要注意的是，查询到的连接数要比实际多1

测试程序

/*client1.cpp *该程序需要libevent库支持 *测试有无LVS时连接延时区别的程序 *程序发起指定数量的连接后，将每秒给服务器发送一个报文，每个连接收到      *服务器返回的报文后计算耗时 */#include <stdlib.h>#include <stdio.h>#include <errno.h>#include <string.h>#include <unistd.h>#include <netdb.h>#include <sys/socket.h>#include <netinet/in.h>#include <sys/types.h>#include <arpa/inet.h>#include <sys/time.h>#include <iostream>#include <map>#include <vector>#include <string>#include "event2/bufferevent.h"#include "event2/buffer.h"#include "event2/listener.h"#include "event2/util.h"#include "event2/event.h"using namespace std;typedef struct bufferevent* BufferEvent;map<struct bufferevent*, long long>    bufferevent_map;map<struct bufferevent*, long long>::iterator  iter;static int      index_ = 0;struct cb_arg{    struct event *ev;    struct timeval tv;};void timeout_cb(int fd, short event, void *params){    if (iter == bufferevent_map.end()) {        return ;    }    char buf[1024];    struct cb_arg *arg = (struct cb_arg*)params;    struct event *timeout = arg->ev;    struct timeval tv = arg->tv;    struct timeval start;    struct bufferevent *bufevt = iter->first;    gettimeofday(&start,NULL);    iter->second =  1000000 * start.tv_sec + start.tv_usec;    sprintf(buf, "%lld", iter->second);    bufferevent_write(bufevt, buf, strlen(buf));    printf("fd %u send %s\n", fd, buf);    index_++;    iter++;    evtimer_add(timeout, &tv);}void read_cb(struct bufferevent *bufevt, void *arg){    char    line[1024] = {0};    size_t     n;    evutil_socket_t fd = bufferevent_getfd(bufevt);    struct timeval start;    map<struct bufferevent*, long long>::iterator it_temp;    it_temp = bufferevent_map.find(bufevt);    if (it_temp == bufferevent_map.end()) {        return ;    }    gettimeofday(&start,NULL);    long long now =  1000000 * start.tv_sec + start.tv_usec;    long long used = now - it_temp->second;    while (n = bufferevent_read(bufevt, line, 1024), n > 0)    {        printf("fd=%u, used time: %lld us\n", fd, used);    }}void error_cb(struct bufferevent *bufevt, short events, void *user_data){    evutil_socket_t fd = bufferevent_getfd(bufevt);    if (events & BEV_EVENT_EOF)    {        printf("Connection closed.\n");        printf("fd %u close !\n", fd);        bufferevent_free(bufevt);    }    else if (events & BEV_EVENT_ERROR)    {        printf("Got an error on the connection: %s\n", strerror(errno));        printf("fd %u close !\n", fd);        bufferevent_free(bufevt);    }}int main(int argc, char *argv[]){    if (argc != 4) {        printf("参数个数不正确 %d, 需要3个参数，连接个数，IP，端口号\n", argc - 1);        return 0;    }    vector<string>  argvs;    for (int i = 1; i < argc; ++i) {        argvs.push_back(argv[i]);    }    timeval     tv = {1, 0};    int         times = atoi(argvs[0].c_str());    int         port = atoi(argvs[2].c_str());    struct sockaddr_in  sin;    struct bufferevent* bufevt;    event_base  *evbase = event_base_new();    if (evbase == NULL) {        printf("err in new event_base\n");        return 0;    }    struct event*   timeout;    struct cb_arg   arg;    timeout = evtimer_new(evbase, timeout_cb, &arg);    if (timeout == NULL) {        printf("err in new evtimer\n");        return 0;    }    arg.ev = timeout;    arg.tv = tv;    memset(&sin, 0, sizeof(sin));    sin.sin_family = AF_INET;    sin.sin_addr.s_addr = inet_addr(argvs[1].c_str());    sin.sin_port = htons(port);    memset(sin.sin_zero, 0x00, 8);    for (int i = 0; i < times; ++i) {        int fd = socket(AF_INET, SOCK_STREAM, 0);        evutil_make_socket_nonblocking(fd);        bufevt = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE);        if (bufevt == NULL) {            printf("err in new bufferevent, fd %u connect failed !\n", fd);            continue ;        }        bufferevent_setcb(bufevt, read_cb, NULL, error_cb, NULL);        bufferevent_enable(bufevt, EV_READ | EV_WRITE | EV_PERSIST);        if (bufferevent_socket_connect(bufevt, (struct sockaddr *)&sin, sizeof(sin)) < 0) {            printf("err in connect with server, fd %u connect failed !\n", fd);            bufferevent_free(bufevt);            continue ;        }        bufferevent_map.insert(make_pair(bufevt, 0));    }    iter = bufferevent_map.begin();    evtimer_add(timeout, &tv);    event_base_dispatch(evbase);    evtimer_del(timeout);    event_base_free(evbase);    return 0;}

/*server.cpp *该程序需要libevent库支持 *简单的回射服务器 */#include <stdio.h>#include <stdlib.h>#include <errno.h>#include <assert.h>#include <event2/event.h>#include <event2/bufferevent.h>#define LISTEN_BACKLOG 1024static int time_out_count = 0;static int err_conn_count = 0;static int err_accept_count = 0;void do_accept(evutil_socket_t listener, short event, void *arg);void read_cb(struct bufferevent *bev, void *arg);void error_cb(struct bufferevent *bev, short event, void *arg);void write_cb(struct bufferevent *bev, void *arg);int main(int argc, char *argv[]){    if (argc != 2) {        printf("参数个数不正确 %d\n", argc);        return 0;    }    int port = atoi(argv[1]);    evutil_socket_t listener;    listener = socket(AF_INET, SOCK_STREAM, 0);    assert(listener > 0);    evutil_make_listen_socket_reuseable(listener);    struct sockaddr_in sin;    sin.sin_family = AF_INET;    sin.sin_addr.s_addr = 0;    sin.sin_port = htons(port);    if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0) {        perror("bind");        return 1;    }    if (listen(listener, LISTEN_BACKLOG) < 0) {        perror("listen");        return 1;    }    printf ("Listening...\n");    evutil_make_socket_nonblocking(listener);    struct event_base *base = event_base_new();    assert(base != NULL);    struct event *listen_event;    listen_event = event_new(base, listener, EV_READ | EV_PERSIST, do_accept, (void*)base);    event_add(listen_event, NULL);    event_base_dispatch(base);    printf("The End.");    return 0;}void do_accept(evutil_socket_t listener, short event, void *arg){    struct event_base *base = (struct event_base *)arg;    evutil_socket_t fd;    struct sockaddr_in sin;    socklen_t slen;    fd = accept(listener, (struct sockaddr *)&sin, &slen);    evutil_make_socket_nonblocking(fd);    if (fd < 0) {        printf("err in accept, err ccode %d, 发生accept错误的个数 %d", errno, ++err_accept_count);        return;    }    printf("ACCEPT: fd = %u\n", fd);    struct bufferevent *bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE);    bufferevent_setcb(bev, read_cb, NULL, error_cb, arg);    bufferevent_enable(bev, EV_READ | EV_WRITE | EV_PERSIST);}void read_cb(struct bufferevent *bev, void *arg){#define MAX_LINE    256    char line[MAX_LINE+1];    int n;    evutil_socket_t fd = bufferevent_getfd(bev);    while (n = bufferevent_read(bev, line, MAX_LINE), n > 0) {        line[n] = '\0';        bufferevent_write(bev, line, n);        printf("recv : %s\n", line);    }}void write_cb(struct bufferevent *bev, void *arg) {}void error_cb(struct bufferevent *bev, short event, void *arg){    evutil_socket_t fd = bufferevent_getfd(bev);    printf("fd = %u, ", fd);    if (event & BEV_EVENT_TIMEOUT) {        time_out_count++;        printf("Timed out, 发生超时连接个数 %d", time_out_count);        printf(", err code %d\n", errno);    }    else if (event & BEV_EVENT_EOF) {        printf("connection closed\n");    }    else if (event & BEV_EVENT_ERROR) {        err_conn_count++;        printf("some other error, 连接被异常关闭个数 %d", err_conn_count);        printf(", err code %d\n", errno);    }    bufferevent_free(bev);}

/*client2.cpp *该程序需要libevent库支持 *测试高并发量的测试客户端程序，该程序每隔10s就会通过所有连接向服务 *器发送一个报文，以免长时间没有交互，连接被LVS断掉，同时测试服务器    *的负载 */#include <stdlib.h>#include <stdio.h>#include <errno.h>#include <string.h>#include <unistd.h>#include <netdb.h>#include <sys/socket.h>#include <netinet/in.h>#include <sys/types.h>#include <arpa/inet.h>#include <sys/time.h>#include <iostream>#include <map>#include <vector>#include <string>#include "event2/bufferevent.h"#include "event2/buffer.h"#include "event2/listener.h"#include "event2/util.h"#include "event2/event.h"using namespace std;typedef struct bufferevent* BufferEvent;map<struct bufferevent*, long long>    bufferevent_map;map<struct bufferevent*, long long>::iterator  iter;FILE*           fp = NULL;static int      index_ = 0;static int      counter = 0;static int      begin_time = 0;struct cb_arg{    struct event *ev;    struct timeval tv;};void timeout_cb(int fd, short event, void *params){    char buf[1024];    struct cb_arg *arg = (struct cb_arg*)params;    struct event *timeout = arg->ev;    struct timeval tv = arg->tv;    for (iter = bufferevent_map.begin(); iter != bufferevent_map.end(); ++iter) {        memset(buf, 0, sizeof(buf));        struct bufferevent *bufevt = iter->first;        sprintf(buf, "%lld", iter->second);        bufferevent_write(bufevt, buf, strlen(buf));    }    evtimer_add(timeout, &tv);}void read_cb(struct bufferevent *bufevt, void *arg){    char    line[1024] = {0};    size_t     n;    evutil_socket_t fd = bufferevent_getfd(bufevt);    struct timeval start;    map<struct bufferevent*, long long>::iterator it_temp;    it_temp = bufferevent_map.find(bufevt);    if (it_temp == bufferevent_map.end()) {        return ;    }    while (n = bufferevent_read(bufevt, line, 1024), n > 0)    {        ;    }}void write_count(int count){    char  buf[1024] = {0};    int   now_time = (int)time(NULL);    sprintf(buf, "pass %d s, now connect count %d\n", now_time - begin_time, count);    fputs(buf, fp);}void error_cb(struct bufferevent *bufevt, short events, void *user_data){    evutil_socket_t fd = bufferevent_getfd(bufevt);    if (events & BEV_EVENT_EOF)    {        counter--;        printf("Connection closed.\n");        printf("fd %u close ! count %d\n", fd , counter);        printf("now EST connect count %d\n", counter);        bufferevent_free(bufevt);        bufferevent_map.erase(bufevt);        write_count(counter);    }    else if (events & BEV_EVENT_ERROR)    {        counter--;        printf("Got an error on the connection: %d\n", errno);        printf("fd %u close ! count %d\n", fd , counter);        printf("now EST connect count %d\n", counter);        bufferevent_free(bufevt);        bufferevent_map.erase(bufevt);        write_count(counter);    }}void create_file(){    char  buf[1024] = {0};    timeval tv;    gettimeofday(&tv, NULL);    int   time_ = (int) (1000000 * tv.tv_sec + tv.tv_usec);    sprintf(buf, "result_%d.txt", time_);    fp = fopen(buf, "w+");    if (fp == NULL) {        printf("create result file failed!\n");        exit(-1);    }}int main(int argc, char *argv[]){    if (argc != 4) {        printf("参数个数不正确 %d, 需要3个参数，连接个数，IP，端口号\n", argc - 1);        return 0;    }    vector<string>  argvs;    for (int i = 1; i < argc; ++i) {        argvs.push_back(argv[i]);    }    create_file();    timeval     tv = {10, 0};    int         times = atoi(argvs[0].c_str());    int         port = atoi(argvs[2].c_str());    struct sockaddr_in  sin;    struct bufferevent* bufevt;    event_base  *evbase = event_base_new();    if (evbase == NULL) {        printf("err %d in new event_base\n", errno);        return 0;    }    struct event*   timeout;    struct cb_arg   arg;    timeout = evtimer_new(evbase, timeout_cb, &arg);    if (timeout == NULL) {        printf("err in new evtimer\n");        return 0;    }    arg.ev = timeout;    arg.tv = tv;    memset(&sin, 0, sizeof(sin));    sin.sin_family = AF_INET;    sin.sin_addr.s_addr = inet_addr(argvs[1].c_str());    sin.sin_port = htons(port);    memset(sin.sin_zero, 0x00, 8);    for (int i = 0; i < times; ++i) {        int fd = socket(AF_INET, SOCK_STREAM, 0);        evutil_make_socket_nonblocking(fd);        bufevt = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE);        if (bufevt == NULL) {            printf("err %d in new bufferevent, fd %u connect failed !\n", fd, errno);            continue ;        }        bufferevent_setcb(bufevt, read_cb, NULL, error_cb, NULL);        bufferevent_enable(bufevt, EV_READ | EV_WRITE | EV_PERSIST);        if (bufferevent_socket_connect(bufevt, (struct sockaddr *)&sin, sizeof(sin)) < 0) {            printf("err %d in connect with server, fd %u connect failed !\n", fd, errno);            continue ;        }        char buf[64] = {0};        sprintf(buf, "%d", i);        bufferevent_map.insert(make_pair(bufevt, 0));        bufferevent_write(bufevt, buf, strlen(buf));        usleep(10);        counter++;    }    printf("now EST connect count %d\n", counter);    write_count(counter);    evtimer_add(timeout, &tv);    event_base_dispatch(evbase);    evtimer_del(timeout);    event_base_free(evbase);    return 0;}