[TCP] TCP_NODELAY?

可能很多人都像我一样，很早就听说过TCP_NODELAY这个选项，明白它跟nagle算法相关，但是就只是停留在表面上了。
最近，实验室在做一套低时延可靠传输协议。作为对比，我希望搞清楚TCP在实际场景中丢包重传到底要消耗多少时间。为此，通过netem在loopback上模拟丢包和延时，我只需要在send以及recv这两个时刻打时间戳就可以知道每个包大概的时延。
以上是背景。然后问题来了。

在MacOS 10.11上，直接进行测试，时延基本为0，符合预计，毕竟没有人为加丢包和延时；
在Linux 4.13上，进行测试，发现时延经常出现30ms、40ms这样的异常值，但是，当把单个数据包的长度降低到500B以下时，恢复正常；这个现象很有趣，也扰乱了思路。
当然，最后才想起可能是TCP_NODELAY，所以纸上得来终觉浅，设置TCP_NODELAY之后，Linux上恢复正常。

#include <stdio.h>#include <string.h>#include <sys/socket.h>#include <netinet/in.h>#include <arpa/inet.h>#include <sys/time.h>#include <fcntl.h>#include <unistd.h>#include <assert.h>#include <netinet/tcp.h>#include <errno.h>#define N (20000)#define USETCP(1)#define LEN(1024)long getts() {    struct timeval tv;    gettimeofday(&tv, NULL);    long ts = tv.tv_sec * 1000 + tv.tv_usec / 1000;    return ts;}ssize_t nrecv(int sock, void *buf, size_t len, int flags) {#ifdef USETCPsize_t nr;size_t nleft = len;void *ptr = buf;while (nleft > 0) {            if ((nr = recv(sock, ptr, nleft, flags)) < 0) {                if (errno == EINTR)    nr = 0;        else    return -1;    } else if (nr == 0)        break;    nleft -= nr;     ptr += nr;}return (len - nleft);#elsereturn recv(sock, buf, len, flags);#endif}typedef struct {long ts;char buf[LEN - sizeof(long)];} DataWrapper;int main(int argc, char *argv[]){    if (argc != 2) {        printf("Not Enough Parameter\n");        return -1;    } else if (memcmp(argv[1], "c", 1) == 0) {#ifdef USETCP        // Init Socket Begins        int sock = socket(PF_INET, SOCK_STREAM, 0);        struct sockaddr_in addr;        memset(&addr, 0, sizeof(addr));        inet_pton(PF_INET, "127.0.0.1", &addr.sin_addr);        addr.sin_family = AF_INET;        addr.sin_port = htons(7777);        connect(sock, (struct sockaddr *) &addr, sizeof(addr));        int on = 1;setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on));// Init Socket Ends#elseint sock = socket(PF_INET, SOCK_DGRAM, 0);struct sockaddr_in addr;        memset(&addr, 0, sizeof(addr));        inet_pton(PF_INET, "127.0.0.1", &addr.sin_addr);        addr.sin_family = AF_INET;        addr.sin_port = htons(7777);        connect(sock, (struct sockaddr *) &addr, sizeof(addr));#endif        DataWrapper dw;        for (int i = 0; i < N; i++) {            dw.ts = getts();            send(sock, &dw, sizeof(dw), 0);            // 1000KBps => 1KB/ms => 1 packet / 1000us            usleep(1000); // roughly        }    } else if (memcmp(argv[1], "s", 1) == 0) {#ifdef USETCP// Init Socket Begins        int listensock = socket(PF_INET, SOCK_STREAM, 0);        int opt = 1;        setsockopt(listensock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));        struct sockaddr_in addr;        memset(&addr, 0, sizeof(addr));        addr.sin_family = AF_INET;        addr.sin_addr.s_addr = htons(INADDR_ANY);        addr.sin_port = htons(7777);        if (bind(listensock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {            printf("tcpsock Bind Failed\n");            return -1;        }        listen(listensock, 128);        int sock = accept(listensock, NULL, NULL);        int on = 1;setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on));// Init Socket Ends#else        int sock = socket(PF_INET, SOCK_DGRAM, 0);        struct sockaddr_in addr;        memset(&addr, 0, sizeof(addr));        addr.sin_family = AF_INET;        addr.sin_addr.s_addr = htons(INADDR_ANY);        addr.sin_port = htons(7777);        if (bind(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {            printf("tcpsock Bind Failed\n");            return -1;        }        struct timeval timeout = {2, 0};        setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (void *)&timeout, sizeof(struct timeval));#endif        DataWrapper dw;        for (int i = 0; i < N; i++) {            int nrcv = nrecv(sock, &dw, sizeof(dw), 0);            if (nrcv != sizeof(dw)) {                fprintf(stderr, "nrcv: %d\n", nrcv);                return -1;            }            printf("%ld\n", getts() - dw.ts);        }    } else {        printf("Error Parameter\n");        return -1;    }}