IPVS负载均衡（六）浅析ip_vs_core.c

ip_vs_core.c

先从linux内核module_init和module_exit开始。

module_init

函数：ip_vs_init

函数原型：static int __init ip_vs_init(void)

初始化ip_vs模块所需的各种

/* *  Initialize IP Virtual Server */static int __init ip_vs_init(void){    int ret;    ret = ip_vs_control_init();    if (ret < 0) {        pr_err("can't setup control.\n");        goto exit;    }    ip_vs_protocol_init();    ret = ip_vs_conn_init();    if (ret < 0) {        pr_err("can't setup connection table.\n");        goto cleanup_protocol;    }    ret = register_pernet_subsys(&ipvs_core_ops);   /* Alloc ip_vs struct */    if (ret < 0)        goto cleanup_conn;    ret = register_pernet_device(&ipvs_core_dev_ops);    if (ret < 0)        goto cleanup_sub;    ret = nf_register_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));    if (ret < 0) {        pr_err("can't register hooks.\n");        goto cleanup_dev;    }    ret = ip_vs_register_nl_ioctl();    if (ret < 0) {        pr_err("can't register netlink/ioctl.\n");        goto cleanup_hooks;    }    pr_info("ipvs loaded.\n");    return ret;cleanup_hooks:    nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));cleanup_dev:    unregister_pernet_device(&ipvs_core_dev_ops);cleanup_sub:    unregister_pernet_subsys(&ipvs_core_ops);cleanup_conn:    ip_vs_conn_cleanup();cleanup_protocol:    ip_vs_protocol_cleanup();    ip_vs_control_cleanup();exit:    return ret;}

主要调用了：

->ip_vs_control_init->ip_vs_protocol_init，协议所需初始化->ip_vs_conn_init，链接管理初始化->register_pernet_subsys->register_pernet_device->nf_register_hooks，注册netfilter钩子函数，后续代码流程会从这里开始->ip_vs_register_nl_ioctl

其中跟IPVS主要流程最相关的是nf_register_hooks、ip_vs_protocol_init、ip_vs_conn_init，后续会主要从nf_register_hooks这里–在netfilter上注册的钩子函数上梳理主干流程

module_exit

函数：ip_vs_cleanup

函数原型：static void __exit ip_vs_cleanup(void)

卸载ip_vs模块，释放占用的资源等操作

static void __exit ip_vs_cleanup(void){    ip_vs_unregister_nl_ioctl();    nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));    unregister_pernet_device(&ipvs_core_dev_ops);    unregister_pernet_subsys(&ipvs_core_ops);   /* free ip_vs struct */    ip_vs_conn_cleanup();    ip_vs_protocol_cleanup();    ip_vs_control_cleanup();    pr_info("ipvs unloaded.\n");}

主要调用：

->ip_vs_unregister_nl_ioctl->nf_unregister_hooks->unregister_pernet_device->unregister_pernet_subsys->ip_vs_conn_cleanup->ip_vs_protocol_cleanup->ip_vs_control_cleanup

可以注意到，ip_vs_cleanup资源释放顺序，与ip_vs_init申请资源顺序刚好相反；且与ip_vs_init中的出错处理部分顺序基本相同。

看完模块注册去注册流程，那么后面则开始寻找

nf_register_hooks钩子函数

通过nf_register_hooks注册的钩子函数进一步梳理代码流程

ret = nf_register_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));

ip_vs_ops的定义

static struct nf_hook_ops ip_vs_ops[] __read_mostly = {    /* After packet filtering, change source only for VS/NAT */    {        .hook       = ip_vs_reply4,        .pf     = NFPROTO_IPV4,        .hooknum    = NF_INET_LOCAL_IN,        .priority   = NF_IP_PRI_NAT_SRC - 2,    },    /* After packet filtering, forward packet through VS/DR, VS/TUN,     * or VS/NAT(change destination), so that filtering rules can be     * applied to IPVS. */    {        .hook       = ip_vs_remote_request4,        .pf     = NFPROTO_IPV4,        .hooknum    = NF_INET_LOCAL_IN,        .priority   = NF_IP_PRI_NAT_SRC - 1,    },    /* Before ip_vs_in, change source only for VS/NAT */    {        .hook       = ip_vs_local_reply4,        .pf     = NFPROTO_IPV4,        .hooknum    = NF_INET_LOCAL_OUT,        .priority   = NF_IP_PRI_NAT_DST + 1,    },    /* After mangle, schedule and forward local requests */    {        .hook       = ip_vs_local_request4,        .pf     = NFPROTO_IPV4,        .hooknum    = NF_INET_LOCAL_OUT,        .priority   = NF_IP_PRI_NAT_DST + 2,    },    /* After packet filtering (but before ip_vs_out_icmp), catch icmp     * destined for 0.0.0.0/0, which is for incoming IPVS connections */    {        .hook       = ip_vs_forward_icmp,        .pf     = NFPROTO_IPV4,        .hooknum    = NF_INET_FORWARD,        .priority   = 99,    },    /* After packet filtering, change source only for VS/NAT */    {        .hook       = ip_vs_reply4,        .pf     = NFPROTO_IPV4,        .hooknum    = NF_INET_FORWARD,        .priority   = 100,    },#ifdef CONFIG_IP_VS_IPV6    /* After packet filtering, change source only for VS/NAT */    {        .hook       = ip_vs_reply6,        .pf     = NFPROTO_IPV6,        .hooknum    = NF_INET_LOCAL_IN,        .priority   = NF_IP6_PRI_NAT_SRC - 2,    },    /* After packet filtering, forward packet through VS/DR, VS/TUN,     * or VS/NAT(change destination), so that filtering rules can be     * applied to IPVS. */    {        .hook       = ip_vs_remote_request6,        .pf     = NFPROTO_IPV6,        .hooknum    = NF_INET_LOCAL_IN,        .priority   = NF_IP6_PRI_NAT_SRC - 1,    },    /* Before ip_vs_in, change source only for VS/NAT */    {        .hook       = ip_vs_local_reply6,        .pf     = NFPROTO_IPV6,        .hooknum    = NF_INET_LOCAL_OUT,        .priority   = NF_IP6_PRI_NAT_DST + 1,    },    /* After mangle, schedule and forward local requests */    {        .hook       = ip_vs_local_request6,        .pf     = NFPROTO_IPV6,        .hooknum    = NF_INET_LOCAL_OUT,        .priority   = NF_IP6_PRI_NAT_DST + 2,    },    /* After packet filtering (but before ip_vs_out_icmp), catch icmp     * destined for 0.0.0.0/0, which is for incoming IPVS connections */    {        .hook       = ip_vs_forward_icmp_v6,        .pf     = NFPROTO_IPV6,        .hooknum    = NF_INET_FORWARD,        .priority   = 99,    },    /* After packet filtering, change source only for VS/NAT */    {        .hook       = ip_vs_reply6,        .pf     = NFPROTO_IPV6,        .hooknum    = NF_INET_FORWARD,        .priority   = 100,    },#endif};

由于是浅析，故这里就不会分析IPV6的场景，主要看ipv4的场景。

.hook涉及到的函数有：

• ip_vs_reply4
• ip_vs_remote_request4
• ip_vs_local_reply4
• ip_vs_local_request4
• ip_vs_forward_icmp
• ip_vs_reply4

netfilter钩子函数最后返回值主要有这几种：

名称 含义 NF_DROP 丢弃该数据包 NF_ACCEPT 保留该数据包 NF_STOLEN 忘掉该数据包 NF_QUEUE 将该数据包插入到用户空间 NF_REPEAT 再次调用该hook函数

.pf是IP类型，这里只讨论IPV4的情况

.hooknum则是涉及到netfilter的五处钩子点，简单可以理解为，netfilter在内核协议栈中五处地方有函数指针，如果注册了钩子函数，则会在对应点调用指定的钩子函数，处理报文。netfilter的五处分别为：NF_IP_PRE_ROUTING，在完整性校验之后，选路确定之前；NF_IP_LOCAL_IN，在选路确定之后，且数据包的目的是本地主机；NF_IP_FORWARD，目的地是其它主机地数据包；NF_IP_LOCAL_OUT，来自本机进程的数据包在其离开本地主机的过程中；NF_IP_POST_ROUTING，在数据包离开本地主机“上线”之前。

可以看到在ipvs中，主要用到了NF_IP_LOCAL_IN、NF_IP_LOCAL_OUT、NF_IP_FORWARD。

.priority则是表示优先级，因为可能会挂多个钩子函数，所以设置优先级，会先调用优先级高的，再调用优先级低的进行处理。

下一步

可以看到ip_vs_reply4、ip_vs_local_reply4都是调用了ip_vs_out

ip_vs_remote_request4、ip_vs_local_request4都是调用了ip_vs_in

下面将会简要分析ip_vs_out和ip_vs_in两个函数的处理流程