利用iptabls的NFLOG记录自己的HTTP HTTPS上网行为

利用iptabls的NFLOG记录自己的HTTP HTTPS上网行为

想要记录上网行为（一天都访问了什么网站）势必要解析TCP数据包，第一种方法是自己写一个内核模块来做这件事，但调内核模块一崩溃系统就要重启，自己的真机上不方便调试。第二种方法是借助iptables的日志功能把数据包发到应用层来解析。

iptables有几个日志记录的target(LOG, ULOG, NFLOG), LOG只是简单的记录包头信息到系统日志，而ULOG， NFLOG则可以把整个数据包发到应用层，应用层可以通过解析数据包来记录自己想要的信息。

大致的过程是这样：内核数据包走到iptables的钩子点上，如果数据包和规则匹配，就通过netlink把这个数据包多播到多播组，然后应用层通过netlink来接受这个数据包。

可以通过iptables -j ULOG --help查看帮助信息，但自己添加iptables规则时（如：sudo iptables -A OUTPUT -j ULOG）发现提示：
iptables: No chain/target/match by that name.

使用命令cat /boot/config-$(uname -r) | grep ULOG得知内核没有编译ULOG进去，但是NFLOG是支持的。于是下面的程序使用NFLOG实现。

数据记录到文件，以后需要时可以再做分析：

1.tail /home/website_access_stat.txt 2.p1.music.126.net, 2017-07-19 22:17, 15004738733.p1.music.126.net, 2017-07-19 22:17, 15004738734.highlightjs.org, 2017-07-19 22:17, 15004738755.music.163.com, 2017-07-19 22:17, 15004738766.music.163.com, 2017-07-19 22:17, 15004738777.music.163.com, 2017-07-19 22:17, 15004738778.music.163.com, 2017-07-19 22:17, 15004738779.m8.music.126.net, 2017-07-19 22:17, 150047387710.music.163.com, 2017-07-19 22:17, 150047387711.www.gstatic.com, 2017-07-19 22:17, 1500473877

实现的代码：

1./************************************************************2.Copyright (C), 2017, Leon, All Rights Reserved.3.FileName: http_trace.c4.Description: 跟踪自己的http，https请求5.Author: Leon6.Version: 1.07.Date: 2017-7-19 13:47:048.Function:9.10.History:11.<author>    <time>  <version>   <description>12. Leon13.14. 安装依赖库：15. apt install libnetfilter-log-dev libnfnetlink-dev16.17. 编译方法：18. gcc http_trace.c -lnetfilter_log19.20. 需要以sudo权限运行21. sudo iptables -I OUTPUT -o enp8s0 -p tcp -j NFLOG --nflog-group 122. sudo ./a.out &23.24. 统计方法：25. cat /home/website_access_stat.txt | awk -F "," '{stat[$1]++} END {for(cmd in stat) print stat[cmd] " " cmd}' | sort -nr26. ************************************************************/27.28.#include <stdio.h>29.#include <stdlib.h>30.#include <string.h>31.#include <sys/socket.h>32.#include <arpa/inet.h>33.#include <sys/types.h>34.#include <linux/netlink.h>35.#include <sys/select.h>36.#include <sys/time.h>37.#include <unistd.h>38.#include <time.h>39.40.#include <libnetfilter_log/libnetfilter_log.h>41.42.#include <linux/ip.h>43.#include <linux/tcp.h>44.#include <linux/in.h>45.46.#include "https.h"47.48.#define LISTEN_GROUP 149.#define BUFF_SIZE 409650.51.#define DATA_FILE "/home/website_access_stat.txt"52.53.#define lprintf(format, argv...) printf("%s(%d): " format , __FUNCTION__, __LINE__, ##argv)54.55.static char host_data[BUFF_SIZE] = {0};56.static int data_idx = 0;    /* 指向host_data的当前存储位置 */57.58.void print_pkt(unsigned char *pkt, int len, char *title)59.{60.    char hex[64] = {0};61.    char assic[64] = {0};62.    char *p_hex = hex, *p_assic = assic;63.    int i = 0;64.65.    printf("*******************%s start*****************\n", title);66.    while(len--)67.    {68.        if(i != 0 && !(i%16))69.        {70.            printf("%s %s\n", hex, assic);71.            memset(hex, 0x0, sizeof(hex));72.            memset(assic, 0x0, sizeof(assic));73.            p_hex = hex;74.            p_assic = assic;75.            i = 0;76.        }77.        sprintf(p_hex, "%02x ", *pkt);78.        p_hex += 3;79.        sprintf(p_assic, "%c", *pkt < 32 || *pkt > 127? '.' : *pkt);80.        p_assic += 1;81.        i++;82.        pkt++;83.        if(!(i%8))84.        {85.            sprintf(p_hex, "  ");86.            p_hex += 2;87.            sprintf(p_assic, "  ");88.            p_assic += 2;89.        }90.    }91.    printf("%s %s\n", hex, assic);92.    printf("\n\n*******************%s end*****************\n", title);93.}94.95.void save_to_file(void)96.{97.    int len = data_idx;98.    int write_len = 0;99.100.    if(!data_idx)101.        return;102.103.    FILE *fp = fopen(DATA_FILE, "a+");104.    if(!fp)105.    {106.        perror("fopen");107.        return;108.    }109.    write_len = fwrite(host_data, len, 1, fp);110.111.    fclose(fp);112.    data_idx = 0;113.}114.115.void save_server_name(char *server_name)116.{117.    time_t sec = time(0);118.    struct tm now;119.    char buf[1024] = {0};120.    int ret = 0;121.122.    localtime_r(&sec, &now);123.    now.tm_year += 1900;124.    now.tm_mon += 1;125.126.    ret = snprintf(buf, sizeof(buf) - 1, "%s, %d-%02d-%02d %02d:%02d, %u\n", 127.            server_name, now.tm_year, now.tm_mon, now.tm_mday, now.tm_hour, 128.            now.tm_min, (unsigned int)sec);129.130.    if(ret + data_idx > BUFF_SIZE)131.    {132.        save_to_file();133.    }134.135.    strcpy(host_data + data_idx, buf);136.    data_idx += ret;137.}138.139.int find_server_name(__u8 *edata, int data_len, char *server_name, int sn_len)140.{141.    __u8 *p = edata;142.    __u16 type;143.    __u16 len;144.145.    while(p < edata + data_len)146.    {147.        type = ntohs(*(__u16 *)p);148.        p += 2;149.        len = ntohs(*(__u16 *)p);150.        p += 2;151.152.        if(type == SERVER_NAME)153.        {154.            p += 3; /* 跳过server name list length和server name type字段 */155.            len = ntohs(*(__u16 *)p);156.            p += 2;157.            if(len > sn_len)158.            {159.                memcpy(server_name, p, sn_len);160.                server_name[sn_len - 1] = '\0';161.            }162.            else163.            {164.                memcpy(server_name, p, len);165.                server_name[len] = '\0';166.            }167.            return 1;168.        }169.        p += len;170.    }171.172.    return 0;173.}174.175.176.#define POINT_OVERFLOW_CHECK(point, limit) do{\177.        if((point) > (limit))   \178.            goto point_overflow;    \179.    }while(0)180.181.void https_handle(struct iphdr *iph)182.{183.    struct tcphdr *tcph = (void*)iph + iph->ihl*4;184.    __u8 *tcpdata = (void*)tcph + tcph->doff*4;185.    int len = ntohs(iph->tot_len) - (iph->ihl*4) - (tcph->doff*4);186.187.    struct https_head *https = (struct https_head *)tcpdata;188.    __u8 *p = tcpdata;189.    __u16 tmp_len;190.    __u8 *point_limit = tcpdata + len;191.    char server_name[HOSTNAME_MAX_LEN] = {0};192.193.    if(len < sizeof(struct https_head))194.        return;195.196.    if(https->content_type != HANDSHAKE)197.        return;198.199.    if(https->handshake_type != CLIENT_HELLO)200.        return;201.202.    /* TLS1.0开始支持server name字段 */203.    if(https->version < htons(TLS1_0_VERSION) 204.        && https->version2 < htons(TLS1_0_VERSION))205.        return;206.207.    p = &(https->session_id_length);208.209.    /* 跳过session_id_length字段 */210.    tmp_len = *p;211.    p++;212.    p += tmp_len;213.    POINT_OVERFLOW_CHECK(p, point_limit);214.215.    /* 跳过cipher_suites_length字段 */216.    tmp_len = ntohs(*(__u16 *)p);217.    p += 2;218.    p += tmp_len;219.    POINT_OVERFLOW_CHECK(p, point_limit);220.221.    /* 跳过compression_methonds_length字段 */222.    tmp_len = *p;223.    p++;224.    p += tmp_len;225.    POINT_OVERFLOW_CHECK(p, point_limit);226.227.    /* 现在为扩展字段 */228.    tmp_len = ntohs(*(__u16 *)p);229.    p += 2;230.231.    if(0 == find_server_name(p, tmp_len, server_name, sizeof(server_name)))232.    {233.        return;234.    }235.    else236.    {237.        save_server_name(server_name);238.        return;239.    }240.241.point_overflow:242.    lprintf("point overflow\n");243.    return;244.245.}246.247./* 不区分大小写的strstr */248.char *strncasestr(char *str, char *sub)249.{250.    if(!str || !sub)251.        return NULL;252.253.    int len = strlen(sub);254.    if (len == 0)255.    {256.        return NULL;257.    }258.259.    while (*str)260.    {261.        if (strncasecmp(str, sub, len) == 0)262.        {263.            return str;264.        }265.        ++str;266.    }267.    return NULL;268.}269.270./* 从字符串缓冲区读取一行数据(跳过空行)，返回下一个字符串的开始位置 */271.char *readline_from_buf(char *from, int from_len, char *to, int to_len)272.{273.    int i = 0;274.275.    memset(to, 0x0, to_len);276.277.    while(*from == '\r' || *from == '\n')278.    {279.        from++;280.        from_len--;281.    }282.283.    for(i = 0; i < to_len && i < from_len; i++)284.    {285.        if(from[i] == '\r' || from[i] == '\n')286.        {287.            memcpy(to, from, i);288.            return from + i;289.        }290.    }291.    return NULL;292.}293.294.void http_handle(struct iphdr *iph)295.{296.    struct tcphdr *tcph = (void*)iph + iph->ihl*4;297.    __u8 *tcpdata = (void*)tcph + tcph->doff*4;298.    int len = ntohs(iph->tot_len) - (iph->ihl*4) - (tcph->doff*4);299.    char line[1024];300.    char *p = NULL;301.302.    if(len && tcpdata[0] != 'G' && tcpdata[0] != 'P')303.        return;304.305.    while(tcpdata = readline_from_buf(tcpdata, len, line, sizeof(line)))306.    {307.        //printf("%s\n", line);308.        if(strncasestr(line, "host:"))309.        {310.            p = (char *)line + strlen("host:");311.            while(*p == ' ')312.                p++;313.            save_server_name(p);314.            break;315.        }316.    }317.}318.319.320.static int cb(struct nflog_g_handle *gh, struct nfgenmsg *nfmsg,321.        struct nflog_data *nfa, void *data)322.{323.    struct nfulnl_msg_packet_hdr *ph = nflog_get_msg_packet_hdr(nfa);324.    u_int32_t mark = nflog_get_nfmark(nfa);325.    u_int32_t indev = nflog_get_indev(nfa);326.    u_int32_t outdev = nflog_get_outdev(nfa);327.    char *prefix = nflog_get_prefix(nfa);328.    char *payload;329.    int payload_len = nflog_get_payload(nfa, &payload);330.    struct iphdr *iph = (struct iphdr *)payload;331.332.    struct tcphdr *tcph;333.334.    if (iph->protocol != IPPROTO_TCP) /* not TCP */335.        return 0;336.337.    tcph = (void*)iph + iph->ihl*4;338.339.    if(tcph->dest == htons(443))340.    {341.        https_handle(iph);  342.        return 0;343.    }344.345.    if(tcph->dest == htons(80))346.    {347.        http_handle(iph);348.        return 0;349.    }350.351.    return 0;352.}353.354.int main(int argc, char *argv[])355.{356.    char buf[BUFF_SIZE] = {0};357.    int len = 0;358.    struct timeval tv = {10, 0};359.    int ret = 0;360.    fd_set rfds;361.    struct nflog_handle *h;362.    struct nflog_g_handle *gh;363.    int fd;364.365.    h = nflog_open();366.    if (!h) {367.        fprintf(stderr, "error during nflog_open()\n");368.        exit(1);369.    }370.371.    if (nflog_bind_pf(h, AF_INET) < 0) {372.        fprintf(stderr, "error during nflog_bind_pf()\n");373.        exit(1);374.    }375.376.    gh = nflog_bind_group(h, 1);377.    if (!gh) {378.        fprintf(stderr, "no handle for grup 1\n");379.        exit(1);380.    }381.382.    if (nflog_set_mode(gh, NFULNL_COPY_PACKET, 0xffff) < 0) {383.        fprintf(stderr, "can't set packet copy mode\n");384.        exit(1);385.    }386.387.    fd = nflog_fd(h);388.389.    nflog_callback_register(gh, &cb, NULL);390.391.    while(1)392.    {393.        tv.tv_sec = 5;394.        tv.tv_usec = 0;395.        FD_ZERO(&rfds);396.        FD_SET(fd, &rfds);397.398.        ret = select(fd + 1, &rfds, NULL, NULL, &tv);399.        if(ret > 0)400.        {401.            len = recv(fd, buf, sizeof(buf), 0); 402.            if(len)403.            {404.                nflog_handle_packet(h, buf, len);   405.            }406.        }407.        else if(ret == 0)408.        {409.            save_to_file();410.        }411.    }412.413.    return 0;414.}

还可以通过sting模块直接匹配字符串，然后再传到应用层处理，这样可以避免传递太多的数据包到应用层
iptables -m string –help

一些有帮助的网页：
ip6tables中nflog的使用

github中保存位置：
https://github.com/leon0625/practice/tree/master/http_trace