K8s ipvs mode kube-proxy
2024-10-20 11:40:38
IPVS vs. IPTABLES
IPVS模式在Kubernetes 1.8中被引入，在1.9中进入beta测试。 IPTABLES模式在1.1版本中被添加进来，在1.2开始就变成了默认的操作模式。IPVS 和 IPTABLES都是基于netfilter实现的。
IPVS与IPTABLES有以下几点不同的地方:
IPVS为大规模集群提供更好的可扩展性和性能。
IPVS比IPTABLES支持更复杂的负载均衡算法 (least load, least connections, locality, weighted, etc.)。
IPVS支持服务器健康检查和重连等等。
配置kube-proxy ipvs模式
k8s版本：1.10
默认情况下，Kube-proxy依然使用iptables来实现Service到POD之间的负载均衡。
[root@node01 ~]# kubectl get svc -o wide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
httpd-svc ClusterIP 10.110.73.171 <none> /TCP 1d run=httpd
kubernetes ClusterIP 10.96.0.1 <none> /TCP 1d <none>
[root@node01 ~]# iptables-save
.
.
.
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -m mark --mark 0x4000/0x4000 -j MASQUERADE
-A KUBE-SEP-6E7XQMQ4RAYOWTTM -s 10.244.0.3/ -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ
-A KUBE-SEP-6E7XQMQ4RAYOWTTM -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.244.0.3:
-A KUBE-SEP-7SDFQQ3CPAUK3SXM -s 10.244.1.2/32 -m comment --comment "default/httpd-svc:" -j KUBE-MARK-MASQ
-A KUBE-SEP-7SDFQQ3CPAUK3SXM -p tcp -m comment --comment "default/httpd-svc:" -m tcp -j DNAT --to-destination 10.244.1.2:80
-A KUBE-SEP-FXPWU3QR6KR3AMXC -s 172.16.65.180/ -m comment --comment "default/kubernetes:https" -j KUBE-MARK-MASQ
-A KUBE-SEP-FXPWU3QR6KR3AMXC -p tcp -m comment --comment "default/kubernetes:https" -m recent --set --name KUBE-SEP-FXPWU3QR6KR3AMXC --mask 255.255.255.255 --rsource -m tcp -j DNAT --to-destination 172.16.65.180:
-A KUBE-SEP-IT2ZTR26TO4XFPTO -s 10.244.0.2/ -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ
-A KUBE-SEP-IT2ZTR26TO4XFPTO -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.244.0.2:
-A KUBE-SEP-TOSIKZVXYIAJSFHP -s 10.244.2.2/32 -m comment --comment "default/httpd-svc:" -j KUBE-MARK-MASQ
-A KUBE-SEP-TOSIKZVXYIAJSFHP -p tcp -m comment --comment "default/httpd-svc:" -m tcp -j DNAT --to-destination 10.244.2.2:80
-A KUBE-SEP-UO4LFRT7VSL3X4EB -s 10.244.0.4/32 -m comment --comment "default/httpd-svc:" -j KUBE-MARK-MASQ
-A KUBE-SEP-UO4LFRT7VSL3X4EB -p tcp -m comment --comment "default/httpd-svc:" -m tcp -j DNAT --to-destination 10.244.0.4:80
-A KUBE-SEP-YIL6JZP7A3QYXJU2 -s 10.244.0.2/ -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ
-A KUBE-SEP-YIL6JZP7A3QYXJU2 -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.244.0.2:
-A KUBE-SEP-ZXMNUKOKXUTL2MK2 -s 10.244.0.3/ -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ
-A KUBE-SEP-ZXMNUKOKXUTL2MK2 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.244.0.3:
-A KUBE-SERVICES ! -s 10.244.0.0/ -d 10.96.0.1/ -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport -j KUBE-MARK-MASQ
-A KUBE-SERVICES -d 10.96.0.1/ -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport -j KUBE-SVC-NPX46M4PTMTKRN6Y
-A KUBE-SERVICES ! -s 10.244.0.0/ -d 10.96.0.10/ -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport -j KUBE-MARK-MASQ
-A KUBE-SERVICES -d 10.96.0.10/ -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport -j KUBE-SVC-TCOU7JCQXEZGVUNU
-A KUBE-SERVICES ! -s 10.244.0.0/ -d 10.96.0.10/ -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport -j KUBE-MARK-MASQ
-A KUBE-SERVICES -d 10.96.0.10/ -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport -j KUBE-SVC-ERIFXISQEP7F7OF4
-A KUBE-SERVICES ! -s 10.244.0.0/ -d 10.110.73.171/ -p tcp -m comment --comment "default/httpd-svc: cluster IP" -m tcp --dport -j KUBE-MARK-MASQ
-A KUBE-SERVICES -d 10.110.73.171/ -p tcp -m comment --comment "default/httpd-svc: cluster IP" -m tcp --dport -j KUBE-SVC-RL3JAE4GN7VOGDGP
-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
-A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-IT2ZTR26TO4XFPTO
-A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-SEP-ZXMNUKOKXUTL2MK2
-A KUBE-SVC-NPX46M4PTMTKRN6Y -m comment --comment "default/kubernetes:https" -m recent --rcheck --seconds --reap --name KUBE-SEP-FXPWU3QR6KR3AMXC --mask 255.255.255.255 --rsource -j KUBE-SEP-FXPWU3QR6KR3AMXC
-A KUBE-SVC-NPX46M4PTMTKRN6Y -m comment --comment "default/kubernetes:https" -j KUBE-SEP-FXPWU3QR6KR3AMXC
-A KUBE-SVC-RL3JAE4GN7VOGDGP -m comment --comment "default/httpd-svc:" -m statistic --mode random --probability 0.33332999982 -j KUBE-SEP-UO4LFRT7VSL3X4EB
-A KUBE-SVC-RL3JAE4GN7VOGDGP -m comment --comment "default/httpd-svc:" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-7SDFQQ3CPAUK3SXM
-A KUBE-SVC-RL3JAE4GN7VOGDGP -m comment --comment "default/httpd-svc:" -j KUBE-SEP-TOSIKZVXYIAJSFHP
-A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-YIL6JZP7A3QYXJU2
-A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns" -j KUBE-SEP-6E7XQMQ4RAYOWTTM
.
.
.
第一步，在内核中加载ip_vs模块：
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack_ipv4"
for kernel_module in \${ipvs_modules}; do
/sbin/modinfo -F filename \${kernel_module} > /dev/null >&
if [ $? -eq ]; then
/sbin/modprobe \${kernel_module}
fi
done
EOF
chmod /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs
输出结果应该为：
[root@node01 ~]# lsmod | grep ip_vs
ip_vs_ftp
ip_vs_sed
ip_vs_nq
ip_vs_sh
ip_vs_dh
ip_vs_lblcr
ip_vs_lblc
ip_vs_wrr
ip_vs_rr
ip_vs_wlc
ip_vs_lc
nf_nat ip_vs_ftp,nf_nat_ipv4,nf_nat_masquerade_ipv4
ip_vs ip_vs_dh,ip_vs_lc,ip_vs_nq,ip_vs_rr,ip_vs_sh,ip_vs_ftp,ip_vs_sed,ip_vs_wlc,ip_vs_wrr,ip_vs_lblcr,ip_vs_lblc
nf_conntrack ip_vs,nf_nat,nf_nat_ipv4,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_netlink,nf_conntrack_ipv4
libcrc32c xfs,ip_vs,nf_nat,nf_conntrack
第二步，安装ipvs管理工具ipvsadm
yum install -y ipvsadm
第三步，修改集群配置文件
在使用kubeadm init --config config.yaml初始化集群前，修改集群配置文件
Kubernetes v1. v1.
kind: MasterConfiguration
apiVersion: kubeadm.k8s.io/v1alpha1
...
kubeProxy:
config:
featureGates: SupportIPVSProxyMode=true
mode: ipvs
... Kubernetes v1.
kind: MasterConfiguration
apiVersion: kubeadm.k8s.io/v1alpha1
...
kubeProxy:
config:
featureGates:
SupportIPVSProxyMode: true
mode: ipvs
...
第四步，结果验证
[root@node01 ~]# ipvsadm -L -n
IP Virtual Server version 1.2. (size=)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.96.0.1: rr persistent
-> 172.16.65.180: Masq
TCP 10.96.0.10: rr
-> 10.244.0.2: Masq
-> 10.244.0.3: Masq
UDP 10.96.0.10: rr
-> 10.244.0.2: Masq
-> 10.244.0.3: Masq
第五步，测试
创建http-svc service
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: httpd
spec:
replicas:
template:
metadata:
labels:
run: httpd
spec:
containers:
- name: httpd
image: httpd
ports:
- containerPort: ---
apiVersion: v1
kind: Service
metadata:
name: httpd-svc
spec:
selector:
run: httpd
type: NodePort
ports:
- protocol: TCP
port:
targetPort: 80
nodePort: 30001
查看效果
[root@node01 yaml-store]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE
httpd-749bf8c6f4-htzgl / Running 11m 10.244.2.2 node03
httpd-749bf8c6f4-vqwff / Running 11m 10.244.1.2 node02
httpd-749bf8c6f4-wkv9w / Running 11m 10.244.0.4 node01
[root@node01 yaml-store]# kubectl get svc -o wide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
httpd-svc ClusterIP 10.99.129.90 <none> 8080:30001/TCP 11m run=httpd
kubernetes ClusterIP 10.96.0.1 <none> /TCP 58m <none>
[root@node01 yaml-store]# ipvsadm -L -n
IP Virtual Server version 1.2. (size=)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.16.65.181: rr
-> 10.244.0.4: Masq
-> 10.244.1.2: Masq
-> 10.244.2.2: Masq
TCP 172.17.0.1: rr
-> 10.244.0.4: Masq
-> 10.244.1.2: Masq
-> 10.244.2.2: Masq
TCP 10.96.0.1: rr persistent
-> 172.16.65.180: Masq
TCP 10.96.0.10: rr
-> 10.244.0.2: Masq
-> 10.244.0.3: Masq
TCP 10.99.129.90: rr
-> 10.244.0.4: Masq
-> 10.244.1.2: Masq
-> 10.244.2.2: Masq
TCP 10.244.0.0: rr
-> 10.244.0.4: Masq
-> 10.244.1.2: Masq
-> 10.244.2.2: Masq
TCP 10.244.0.1: rr
-> 10.244.0.4: Masq
-> 10.244.1.2: Masq
-> 10.244.2.2: Masq
TCP 127.0.0.1: rr
-> 10.244.0.4: Masq
-> 10.244.1.2: Masq
-> 10.244.2.2: Masq
UDP 10.96.0.10: rr
-> 10.244.0.2: Masq
-> 10.244.0.3: Masq
参考文章：
https://github.com/kubernetes/kubernetes/tree/master/pkg/proxy/ipvs
https://www.kubernetes.org.cn/3025.html
最新文章
- 【66测试20161115】【树】【DP_LIS】【SPFA】【同余最短路】【递推】【矩阵快速幂】
- ARM指令学习,王明学learn
- IOS中CocoaPods安装与使用
- 播放视频最好的 HTML 解决方法
- Python学习（2）基本语法
- centos7通过firewalld更改sshd端口
- Nagle算法，tcp小包组合(延迟)发送的算法
- MFC 构建、消亡 顺序 (二)--多文档 （MDI）
- jsp a标签传值到action中，action接收不到传值
- wcf托管在IIS上，提示未能加载
- javascript 之作用域-06
- powerdesigner将name的名字赋给comment
- CSCI 1100 — Computer Science 1 Homework
- 牛客网PAT乙级(Basic Level)真题-数字分类 (20)
- 前端javascript
- oracle 监听 添加ip
- [Hi3520DV200]烧录
- springMVC注解（一）Controller
- 个推Node.js 微服务实践：基于容器的一站式命令行工具链
- linux实时流量监控