docker 1.12 网络和负载均衡初探

环境：

主机A   euca- 10-153-177-58

主机B   euca-10-153-177-76

root@euca-10-153-177-58:~# dockernode list

ID                           HOSTNAME      STATUS AVAILABILITY  MANAGER STATUS

3ddwxfmfmpvkndj0l4ynzhnhr    euca-10-153-177-76  Ready  Active       

cvfd6ufl2kwflqsga7df7rcl1 *        euca-10-153-177-58  Ready  Active        Leader

root@euca-10-153-177-58:~# dockerservice list

ID            NAME      REPLICAS IMAGE          COMMAND

aay34mgae9zf  cloudbts 2/2       cloudbts:v4.0 

d8mt79by1609  cbpts70  2/2       cbpts70:v3.0

 

service cloudbts   开放8180:8080  8122:22 端口映射

service cbpts70    开放8080:8080  8022:22 端口映射

查看NAT

root@euca-10-153-177-58:~# iptables-t nat –S

-A POSTROUTING -s 172.18.0.0/16 ! -odocker_gwbridge -j MASQUERADE

-A DOCKER -i docker0 -j RETURN

-A DOCKER -i docker_gwbridge -jRETURN

-A DOCKER-INGRESS -p tcp -m tcp--dport 8022 -j DNAT --to-destination 172.18.0.2:8022

-A DOCKER-INGRESS -p tcp -m tcp--dport 8080 -j DNAT --to-destination 172.18.0.2:8080

-A DOCKER-INGRESS -p tcp -m tcp--dport 8180 -j DNAT --to-destination 172.18.0.2:8180

-A DOCKER-INGRESS -p tcp -m tcp--dport 8122 -j DNAT --to-destination 172.18.0.2:8122

root@euca-10-153-177-76:~# iptables-t nat –S

-A POSTROUTING -s 172.17.0.0/16 ! -odocker0 -j MASQUERADE

-A POSTROUTING -o docker_gwbridge -maddrtype --src-type LOCAL -j MASQUERADE

-A POSTROUTING -s 172.18.0.0/16 ! -odocker_gwbridge -j MASQUERADE

-A DOCKER -i docker0 -j RETURN

-A DOCKER -i docker_gwbridge -jRETURN

-A DOCKER-INGRESS -p tcp -m tcp--dport 8022 -j DNAT --to-destination 172.18.0.2:8022

-A DOCKER-INGRESS -p tcp -m tcp --dport8080 -j DNAT --to-destination 172.18.0.2:8080

-A DOCKER-INGRESS -p tcp -m tcp--dport 8180 -j DNAT --to-destination 172.18.0.2:8180

-A DOCKER-INGRESS -p tcp -m tcp--dport 8122 -j DNAT --to-destination 172.18.0.2:8122

 

检查docker 网络命名空间列表

root@euca-10-153-177-58:~# ls/var/run/docker/netns

1-22hxmh4c0e  caf29e8b2fab d596a954d729  ingress_sbox

 

root@euca-10-153-177-76:~# ls/var/run/docker/netns

1-22hxmh4c0e  c329f7db767d f08c5ce5be91  ingress_sbox

 

进入ingress_sbox网络命名空间

root@euca-10-153-177-76:/var/run/docker/netns#nsenter --net=ingress_sbox sh

# ifconfig

eth0      Link encap:Ethernet  HWaddr 02:42:0a:ff:00:07 

          inet addr:10.255.0.7  Bcast:0.0.0.0 Mask:255.255.0.0

          inet6 addr: fe80::42:aff:feff:7/64 Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:112920 errors:0 dropped:0overruns:0 frame:0

          TX packets:105006 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:414828822 (414.8 MB)  TX bytes:8263325 (8.2 MB)

 

eth1      Link encap:Ethernet  HWaddr 02:42:ac:12:00:02 

          inet addr:172.18.0.2  Bcast:0.0.0.0 Mask:255.255.0.0

          inet6 addr: fe80::42:acff:fe12:2/64Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1500 Metric:1

          RX packets:105086 errors:0 dropped:0overruns:0 frame:0

          TX packets:112895 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:8268245 (8.2 MB)  TX bytes:414825888 (414.8 MB)

 

lo        Link encap:Local Loopback 

          inet addr:127.0.0.1  Mask:255.0.0.0

          inet6 addr: ::1/128 Scope:Host

          UP LOOPBACK RUNNING  MTU:65536 Metric:1

          RX packets:0 errors:0 dropped:0 overruns:0frame:0

          TX packets:0 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:1

          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

# iptables -nvL -t mangle

Chain PREROUTING (policy ACCEPT 229Kpackets, 442M bytes)

 pkts bytes target     prot opt in     out    source               destination        

   0     0 MARK       tcp --  *      *      0.0.0.0/0           0.0.0.0/0            tcp dpt:8122MARK set 0x103

 181 12004 MARK       tcp  -- *      *       0.0.0.0/0            0.0.0.0/0            tcp dpt:8180 MARK set 0x103

 110K 7153K MARK       tcp --  *      *      0.0.0.0/0           0.0.0.0/0            tcp dpt:8080MARK set 0x104

   0     0 MARK       tcp --  *      *      0.0.0.0/0           0.0.0.0/0            tcp dpt:8022 MARK set 0x104

 

Chain INPUT (policy ACCEPT 110Kpackets, 7161K bytes)

 pkts bytes target     prot opt in     out    source              destination        

 

Chain FORWARD (policy ACCEPT 119Kpackets, 435M bytes)

 pkts bytes target     prot opt in     out    source              destination        

 

Chain OUTPUT (policy ACCEPT 110Kpackets, 7161K bytes)

 pkts bytes target     prot opt in     out    source              destination        

   0     0 MARK       all --  *      *      0.0.0.0/0           10.255.0.2           MARK set0x103

   0     0 MARK       all --  *      *      0.0.0.0/0           10.255.0.6           MARK set0x104

 

Chain POSTROUTING (policy ACCEPT229K packets, 442M bytes)

 pkts bytes target     prot opt in     out    source               destination

 

# ipvsadm

IP Virtual Server version 1.2.1(size=4096)

Prot LocalAddress:Port SchedulerFlags

 -> RemoteAddress:Port          Forward Weight ActiveConn InActConn

FWM 259 (=x103)rr

 -> 10.255.0.4:0                Masq    1      0         0        

 -> 10.255.0.5:0                Masq    1      0         0        

FWM 260(=0x104)rr

 -> 10.255.0.8:0                Masq    1      0         9        

 -> 10.255.0.9:0                 Masq   1      0          9  

 

进入1-22hxmh4c0e 网络命名空间

root@euca-10-153-177-76:/var/run/docker/netns#nsenter --net=1-22hxmh4c0e sh

# ifconfig

br0       Link encap:Ethernet  HWaddr 36:ca:0a:78:14:ae 

          inet addr:10.255.0.1  Bcast:0.0.0.0 Mask:255.255.0.0

          inet6 addr: fe80::b45f:c9ff:fe6c:7216/64Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:55 errors:0 dropped:0overruns:0 frame:0

          TX packets:8 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:3412 (3.4 KB)  TX bytes:648 (648.0 B)

 

lo        Link encap:Local Loopback 

          inet addr:127.0.0.1  Mask:255.0.0.0

          inet6 addr: ::1/128 Scope:Host

          UP LOOPBACK RUNNING  MTU:65536 Metric:1

          RX packets:0 errors:0 dropped:0overruns:0 frame:0

          TX packets:0 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:1

          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

 

veth2     Link encap:Ethernet  HWaddr 36:ca:0a:78:14:ae 

          inet6 addr:fe80::34ca:aff:fe78:14ae/64 Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:114691 errors:0 dropped:0overruns:0 frame:0

          TX packets:123664 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:9026640 (9.0 MB)  TX bytes:449408513 (449.4 MB)

 

veth6     Link encap:Ethernet  HWaddr d6:2e:06:03:e9:7e 

          inet6 addr:fe80::d42e:6ff:fe03:e97e/64 Scope:Link

         UP BROADCAST RUNNINGMULTICAST  MTU:1450  Metric:1

          RX packets:446 errors:0 dropped:0overruns:0 frame:0

          TX packets:428 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:860524 (860.5 KB)  TX bytes:42790 (42.7 KB)

 

veth7     Link encap:Ethernet  HWaddr 42:63:b9:1e:1f:78 

          inet6 addr:fe80::4063:b9ff:fe1e:1f78/64 Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:86968 errors:0 dropped:0overruns:0 frame:0

          TX packets:58728 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:237555101 (237.5 MB)  TX bytes:4590982 (4.5 MB)

 

vxlan1    Link encap:Ethernet  HWaddr 5e:90:7c:34:53:d9 

          inet6 addr:fe80::5c90:7cff:fe34:53d9/64 Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:158534 errors:0 dropped:0overruns:0 frame:0

          TX packets:57099 errors:0 dropped:59overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:218690170 (218.6 MB)  TX bytes:5708548 (5.7 MB)

# ip -d link show vxlan1

12: vxlan1:<BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master br0 stateUNKNOWN mode DEFAULT group default

   link/ether 5e:90:7c:34:53:d9 brd ff:ff:ff:ff:ff:ff link-netnsid 0promiscuity 1

   vxlan id 256 srcport 0 0 dstport 4789 proxy l2miss l3miss ageing 300

bridge_slavestate forwarding priority 32 cost 100 hairpin off guard off root_block offfastleave off learning on flood on addrgenmode eui64 

 

root@euca-10-153-177-58:/var/run/docker/netns#nsenter --net=1-22hxmh4c0e sh

#ifconfig

br0       Link encap:Ethernet  HWaddr 22:d7:ae:ab:99:36 

          inet addr:10.255.0.1  Bcast:0.0.0.0 Mask:255.255.0.0

          inet6 addr:fe80::ac88:34ff:feb1:f213/64 Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:48 errors:0 dropped:0overruns:0 frame:0

          TX packets:8 errors:0 dropped:0 overruns:0carrier:0

          collisions:0 txqueuelen:0

          RX bytes:2904 (2.9 KB)  TX bytes:648 (648.0 B)

 

lo        Link encap:Local Loopback 

          inet addr:127.0.0.1  Mask:255.0.0.0

          inet6 addr: ::1/128 Scope:Host

          UP LOOPBACK RUNNING  MTU:65536 Metric:1

          RX packets:0 errors:0 dropped:0overruns:0 frame:0

          TX packets:0 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:1

          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

 

veth2     Link encap:Ethernet  HWaddr e6:f7:58:70:40:61 

          inet6 addr:fe80::e4f7:58ff:fe70:4061/64 Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:4489 errors:0 dropped:0overruns:0 frame:0

          TX packets:2850 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:316738 (316.7 KB)  TX bytes:10033175 (10.0 MB)

 

veth5     Link encap:Ethernet  HWaddr 2a:9e:6c:9c:30:99 

          inet6 addr:fe80::289e:6cff:fe9c:3099/64 Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:442 errors:0 dropped:0overruns:0 frame:0

          TX packets:488 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:761515 (761.5 KB)  TX bytes:46143 (46.1 KB)

 

veth6     Link encap:Ethernet  HWaddr 22:d7:ae:ab:99:36 

          inet6 addr:fe80::20d7:aeff:feab:9936/64 Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:121565 errors:0 dropped:0overruns:0 frame:0

          TX packets:81680 errors:0 dropped:0overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:325510290 (325.5 MB)  TX bytes:6450337 (6.4 MB)

 

vxlan1    Link encap:Ethernet  HWaddr 86:4f:c5:4d:12:18 

          inet6 addr:fe80::844f:c5ff:fe4d:1218/64 Scope:Link

          UP BROADCAST RUNNING MULTICAST  MTU:1450 Metric:1

          RX packets:82661 errors:0 dropped:0overruns:0 frame:0

          TX packets:122168 errors:0 dropped:55overruns:0 carrier:0

          collisions:0 txqueuelen:0

          RX bytes:7356251 (7.3 MB)  TX bytes:322463758 (322.4 MB)

 

# ip -dlink show vxlan1

6:vxlan1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue masterbr0 state UNKNOWN mode DEFAULT group default

    link/ether 86:4f:c5:4d:12:18 brdff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 1

    vxlan id 256 srcport 0 0 dstport 4789 proxyl2miss l3miss ageing 300

    bridge_slave state forwarding priority 32cost 100 hairpin off guard off root_block off fastleave off learning on floodon addrgenmode eui64

 

通过上述查看可见内部网络拓扑

如果外部访问服务，docker负载均衡的流程如下

1、 用户访问xx.xx.xx.76的8080端口。

2、 Iptable DNAT转发至 Ingress NS 空间里的172.18.0.2。

Ingress NS 空间里的Iptable根据不同的端口设置不同的Mark。

而服务IP 10.255.0.2是VIP，在10.255.0.2和10.255.0.4之间浮动。

FWM  259 (=x103)rr

  ->10.255.0.4:0                 Masq    1     0          0        

  ->10.255.0.5:0                 Masq    1     0          0        

 

3、 跳转至10.255.0.5，如果该IP在本容器，根据路由，数据包进入1-22xxx NS空间，

3.1然后通过veth至容器命名空间

4、 跳转至10.255.0.4，如果该IP不在本容器，根据路由，数据包进入1-22xxx NS空间。

5、 在1-22xxx NS空间里br0网桥进入vxlan link,

6、 vxlan link 最终进入物理网卡

7、 物理网卡传输至远程主机

8、 进入远程1-22xxx NS空间，并且vxlan 解包

9、 跳转至容器空间。

 

如果是容器内部主动往外发送数据，流程如下图：

 

容器访问网关docker_gwbridge,由docker_gwbridge SNAT转发至外部网络。

 

 

总的来说，docker 使用VIP来实现了负载均衡，相比较k8s使用kube-proxy 管理iptable来实现的负载均衡，看起来效率高点，但还是避免不了iptable 转发对网络效率的影响。

而且，docker 的service 对外暴露可访问IP是一刀切的，所有节点都暴露，管理粗细度还是不及k8s，k8s可单个节点暴露。