flannel 关闭SNAT

默认情况下，flannel 访问集群外网络是通过 SNAT 成宿主机 ip 方式，在一些金融客户环境中为了能实现防火墙规则，需要直接针对 POD ip 进行进行规则配置，所以需要关闭 SNAT

关闭flannel 配置文件关于ip-masq 配置,删除配置文件 -ip-masq 参数

# cat /etc/systemd/system/flanneld.service

[Unit]

Description=Flanneld overlay address etcd agent

After=network.target

After=network-online.target

Wants=network-online.target

After=etcd.service

Before=docker.service

[Service]

Type=notify

ExecStart=/opt/k8s/bin/flanneld \

  -etcd-cafile=/data/work/ca.pem  \

  -etcd-certfile=/data/work/flanneld.pem \

  -etcd-keyfile=/data/work/flanneld-key.pem \

  -etcd-endpoints=https://10.65.91.161:2379,https://10.65.91.162:2379,https://10.65.91.163:2379 \

  -etcd-prefix=/kubernetes/network \

  -iface=ens192

ExecStartPost=/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker

Restart=always

RestartSec=5

StartLimitInterval=0

[Install]

WantedBy=multi-user.target

RequiredBy=docker.service

#重启 flanneld

systemctl  daemon-reload

systemctl  restart flanneld

修改flannel 配置文件中 mk-docker-opts.sh，将 ipmasq=true 修改为 ipmasq=ipmasq=false

#!/bin/sh

usage() {

        echo "$0 [-f FLANNEL-ENV-FILE] [-d DOCKER-ENV-FILE] [-i] [-c] [-m] [-k COMBINED-KEY]

Generate Docker daemon options based on flannel env file

OPTIONS:

        -f      Path to flannel env file. Defaults to /run/flannel/subnet.env

        -d      Path to Docker env file to write to. Defaults to /run/docker_opts.env

        -i      Output each Docker option as individual var. e.g. DOCKER_OPT_MTU=1500

        -c      Output combined Docker options into DOCKER_OPTS var

        -k      Set the combined options key to this value (default DOCKER_OPTS=)

        -m      Do not output --ip-masq (useful for older Docker version)

" >&2

        exit 1

}

flannel_env="/run/flannel/subnet.env"

docker_env="/run/docker_opts.env"

combined_opts_key="DOCKER_OPTS"

indiv_opts=false

combined_opts=false

ipmasq=false

while getopts "f:d:icmk:?h" opt; do

        case $opt in

                f)

                        flannel_env=$OPTARG

                        ;;

                d)

                        docker_env=$OPTARG

                        ;;

                i)

                        indiv_opts=true

                        ;;

                c)

                        combined_opts=true

                        ;;

                m)

                        ipmasq=false

                        ;;

                k)

                        combined_opts_key=$OPTARG

                        ;;

                [\?h])

                        usage

                        ;;

        esac

done

if [ $indiv_opts = false ] && [ $combined_opts = false ]; then

        indiv_opts=true

        combined_opts=true

fi

if [ -f "$flannel_env" ]; then

        . $flannel_env

fi

if [ -n "$FLANNEL_SUBNET" ]; then

        DOCKER_OPT_BIP="--bip=$FLANNEL_SUBNET"

fi

if [ -n "$FLANNEL_MTU" ]; then

        DOCKER_OPT_MTU="--mtu=$FLANNEL_MTU"

fi

if [ -n "$FLANNEL_IPMASQ" ] && [ $ipmasq = false ] ; then

        if [ "$FLANNEL_IPMASQ" = true ] ; then

                DOCKER_OPT_IPMASQ="--ip-masq=false"

        elif [ "$FLANNEL_IPMASQ" = false ] ; then

                DOCKER_OPT_IPMASQ="--ip-masq=false"

        else

                echo "Invalid value of FLANNEL_IPMASQ: $FLANNEL_IPMASQ" >&2

                exit 1

        fi

fi

eval docker_opts="\$${combined_opts_key}"

if [ "$docker_opts" ]; then

        docker_opts="$docker_opts ";

fi

echo -n "" >$docker_env

for opt in $(set | grep "DOCKER_OPT_"); do

        OPT_NAME=$(echo $opt | awk -F "=" '{print $1;}');

        OPT_VALUE=$(eval echo "\$$OPT_NAME");

        if [ "$indiv_opts" = true ]; then

                echo "$OPT_NAME=\"$OPT_VALUE\"" >>$docker_env;

        fi

        docker_opts="$docker_opts $OPT_VALUE";

done

if [ "$combined_opts" = true ]; then

        echo "${combined_opts_key}=\"${docker_opts}\"" >>$docker_env

fi

#重启 flanneld

systemctl  daemon-reload

systemctl  restart flanneld

docker 配置

# cat /usr/lib/systemd/system/docker.service

#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock

Environment="PATH=/opt/k8s/bin:/bin:/sbin:/usr/bin:/usr/sbin"

EnvironmentFile=-/run/flannel/docker

ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS

ExecReload=/bin/kill -s HUP $MAINPID

TimeoutSec=0

RestartSec=2

Restart=always

重启docker

systemctl daemon-reload

systemctl restart docker

查看docker 配置

#  cat /run/flannel/docker

DOCKER_OPT_BIP="--bip=10.0.79.1/24"

DOCKER_OPT_IPMASQ="--ip-masq=false"

DOCKER_OPT_MTU="--mtu=1500"

DOCKER_NETWORK_OPTIONS=" --bip=10.0.79.1/24 --ip-masq=false --mtu=1500"

[root@lgy-test-node01 10.65.91.164 ~ ]

#  cat /run/flannel/subnet.env

FLANNEL_NETWORK=10.0.0.0/16

FLANNEL_SUBNET=10.0.79.1/24

FLANNEL_MTU=1500

FLANNEL_IPMASQ=false

删除 node节点 POSTROUTING 规则,只剩下默认一条规则

iptables -t nat --line-numbers -vnL POSTROUTING

1       87  5856 KUBE-POSTROUTING  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes postrouting rules */

#删除方法

iptables -t nat -D POSTROUTING  3

iptables -t nat -D POSTROUTING  2

iptables -t nat -D POSTROUTING  1

tcpdump 抓包测试

tcpdump  -i ens192 port 80  -vnn 

10.0.79.2.59010 > 10.65.91.51.80

需要增加主机路由

route add -host  10.0.79.2    gw 10.65.91.164

巴特西

flannel 关闭SNAT

flannel 关闭SNAT

最新文章

热门文章