简介

本集群使用 nginx + keepalived 实现高可用

nginx 安装配置

下载编译nginx

nginx 只需要编译一次，把编译后的文件拷贝到其他master机器上即可

cd /opt/k8s/work

wget http://nginx.org/download/nginx-1.15.3.tar.gz

tar -xzvf nginx-1.15.3.tar.gz

#编译

cd /opt/k8s/work/nginx-1.15.3

mkdir nginx-prefix

./configure --with-stream --without-http --prefix=$(pwd)/nginx-prefix --without-http_uwsgi_module

make && make install

#############

--without-http_scgi_module --without-http_fastcgi_module

--with-stream：开启 4 层透明转发(TCP Proxy)功能；

--without-xxx：关闭所有其他功能，这样生成的动态链接二进制程序依赖最小；

查看 nginx 动态链接的库：

[root@node01 nginx-1.15.3]# ldd ./nginx-prefix/sbin/nginx

	linux-vdso.so.1 =>  (0x00007ffee18cc000)

	libdl.so.2 => /lib64/libdl.so.2 (0x00007f5e89daa000)

	libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f5e89b8e000)

	libc.so.6 => /lib64/libc.so.6 (0x00007f5e897c0000)

	/lib64/ld-linux-x86-64.so.2 (0x00007f5e89fae000)

由于只开启了 4 层透明转发功能，所以除了依赖 libc 等操作系统核心 lib 库外，没有对其它 lib 的依赖(如 libz、libssl 等)，这样可以方便部署到各版本操作系统中

创建目录结构

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    ssh root@${node_ip} "mkdir -p /opt/k8s/kube-nginx/{conf,logs,sbin}"

  done

拷贝二进制程序到其他主机 (有报错执行2遍就可以)

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    scp /opt/k8s/work/nginx-1.15.3/nginx-prefix/sbin/nginx  root@${node_ip}:/opt/k8s/kube-nginx/sbin/kube-nginx

    ssh root@${node_ip} "chmod a+x /opt/k8s/kube-nginx/sbin/*"

  done

配置Nginx文件，开启4层透明转发

cd /opt/k8s/work

cat > kube-nginx.conf <<EOF

worker_processes 1;

events {

    worker_connections  1024;

}

stream {

    upstream backend {

        hash $remote_addr consistent;

        server 10.0.20.11:6443        max_fails=3 fail_timeout=30s;

        server 10.0.20.12:6443        max_fails=3 fail_timeout=30s;

        server 10.0.20.13:6443        max_fails=3 fail_timeout=30s;

    }

    server {

        listen *:8443;

        proxy_connect_timeout 1s;

        proxy_pass backend;

    }

}

EOF

#这里需要将server替换我们自己的地址

分发配置文件

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    scp kube-nginx.conf  root@${node_ip}:/opt/k8s/kube-nginx/conf/kube-nginx.conf

  done

配置Nginx启动文件

cd /opt/k8s/work

cat > kube-nginx.service <<EOF

[Unit]

Description=kube-apiserver nginx proxy

After=network.target

After=network-online.target

Wants=network-online.target

[Service]

Type=forking

ExecStartPre=/opt/k8s/kube-nginx/sbin/kube-nginx -c /opt/k8s/kube-nginx/conf/kube-nginx.conf -p /opt/k8s/kube-nginx -t

ExecStart=/opt/k8s/kube-nginx/sbin/kube-nginx -c /opt/k8s/kube-nginx/conf/kube-nginx.conf -p /opt/k8s/kube-nginx

ExecReload=/opt/k8s/kube-nginx/sbin/kube-nginx -c /opt/k8s/kube-nginx/conf/kube-nginx.conf -p /opt/k8s/kube-nginx -s reload

PrivateTmp=true

Restart=always

RestartSec=5

StartLimitInterval=0

LimitNOFILE=65536

[Install]

WantedBy=multi-user.target

EOF

分发nginx启动文件

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    scp kube-nginx.service  root@${node_ip}:/etc/systemd/system/

  done

启动 kube-nginx 服务

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-nginx && systemctl start kube-nginx"

  done

检查 kube-nginx 服务运行状态

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    ssh root@${node_ip} "systemctl status kube-nginx |grep 'Active:'"

  done

[root@node01 work]# for node_ip in ${MASTER_IPS[@]}

>   do

>     echo ">>> ${node_ip}"

>     ssh root@${node_ip} "systemctl status kube-nginx |grep 'Active:'"

>   done

>>> 10.0.20.11

   Active: active (running) since Thu 2019-12-05 15:13:19 CST; 3s ago

>>> 10.0.20.12

   Active: active (running) since Thu 2019-12-05 15:13:19 CST; 3s ago

>>> 10.0.20.13

   Active: active (running) since Thu 2019-12-05 15:13:19 CST; 3s ago

检查 kube-nginx 端口

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    ssh root@${node_ip} "netstat -lntup | grep 8443"

  done

[root@node01 work]# for node_ip in ${MASTER_IPS[@]}

>   do

>     echo ">>> ${node_ip}"

>     ssh root@${node_ip} "netstat -lntup | grep 8443"

>   done

>>> 10.0.20.11

tcp        0      0 0.0.0.0:8443            0.0.0.0:*               LISTEN      5356/nginx: master

>>> 10.0.20.12

tcp        0      0 0.0.0.0:8443            0.0.0.0:*               LISTEN      2586/nginx: master

>>> 10.0.20.13

tcp        0      0 0.0.0.0:8443            0.0.0.0:*               LISTEN      2630/nginx: master

keepalived 安装配置

安装keeplive服务

前面我们也说了，高可用方案需要一个VIP，供集群内部访问

在所有master节点安装keeplived

yum  install -y keepalived

配置keeplive服务

配置文件模板

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

cat > keepalived.conf.template <<EOF

! Configuration File for keepalived

global_defs {

   router_id ##MASTER_IP##

}

vrrp_script chk_nginx {

    script "/etc/keepalived/check_port.sh 8443"

    interval 2

    weight -20

}

vrrp_instance VI_1 {

    state MASTER

    interface ##IFACE##

    virtual_router_id 251

    priority 100

    advert_int 1

    mcast_src_ip ##MASTER_IP##

    nopreempt

    authentication {

        auth_type PASS

        auth_pass 11111111

    }

    track_script {

         chk_nginx

    }

    virtual_ipaddress {

        ##KEEP_VIP##

    }

}

EOF

替换模板文件的变量，为各个节点生成配置文件

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for (( i=0; i < 3; i++ ))

  do

    sed -e "s/##MASTER_IP##/${MASTER_IPS[i]}/" -e "s/##KEEP_VIP##/${KEEP_VIP_ADDR}/" -e "s/##IFACE##/${IFACE}/" keepalived.conf.template >  keepalived-${MASTER_IPS[i]}.conf

  done

ls keepalived-*.conf

将对应的keepalived配置文件拷贝到对应的节点上

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for (( i=0; i < 3; i++ ))

  do

    echo ">>> ${node_ip}"

    scp  keepalived-${MASTER_IPS[i]}.conf ${MASTER_NAMES[i]}:/etc/keepalived/keepalived.conf

  done

创建健康检查脚本

cd /opt/k8s/work

cat > check_port.sh <<EOF

#!/bin/sh

CHK_PORT=\$1

if [ -n "\$CHK_PORT" ];then

  PORT_PROCESS=\`ss -lntup|grep \${CHK_PORT}|wc -l\`

    if [ \$PORT_PROCESS -eq 0 ];then

      echo -e "\033[31m ERROR: Port \$CHK_PORT Is Not Used,End. \033[0m"

      exit 1

  fi

fi

EOF

分发脚本到所有keepalived节点

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node in ${MASTER_IPS[@]}

do

  echo ">>> ${node}"

  scp check_port.sh ${node}:/etc/keepalived/

done

启动keeplived

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node in ${MASTER_IPS[@]}

do

  echo ">>> ${node}"

  ssh ${node} "systemctl enable keepalived && systemctl start keepalived && systemctl status keepalived | grep active"

done

查看VIP地址

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node in ${MASTER_IPS[@]}

do

  echo ">>> ${node}"

  ssh ${node} "ip a | grep 20.10"

done

输出结果

[root@node01 work]# for node in ${MASTER_IPS[@]}

> do

>   echo ">>> ${node}"

>   ssh ${node} "ip a | grep 20.10"

> done

>>> 10.0.20.11

>>> 10.0.20.12

>>> 10.0.20.13

    inet 10.0.20.10/32 scope global bond0

巴特西

kube-nginx 和 keepalived 部署安装

简介