【Linux】【Services】【SaaS】Docker+kubernetes（10. 利用反向代理实现服务高可用）

1. 简介

1.1. 由于K8S并没有自己的集群，所以需要借助其他软件来实现，公司的生产环境使用的是Nginx，想要支持TCP转发要额外安装模块，测试环境中我就使用HAPROXY了

1.2. 由于是做实现，我用Nginx又重新实现了一下HAPROXY的TCP转发，都会在本文中介绍

1.3. Haproxy配置参数详解请参考我前面的文章：http://www.cnblogs.com/demonzk/p/6904029.html

2. 环境

2.1. 机器列表

功能与组件	机器名	服务IP	管理IP	VIP	Processor Cores	RAM	Storage	备注
HAPROXY	HCTJOSDR01	10.30.2.48	172.16.0.48	172.16.0.148
HAPROXY	HCTJOSDR02	10.30.2.49	172.16.0.49	172.16.0.149

2.2. 架构图

2.3. 软件版本

haproxy　　　　1.5.18-6.el7

keepalived　　　　1.3.5-1.el7

nginx　　　　　　1.12.2-1.el7

3. 安装与基础配置

3.1. haproxy

yum安装

yum -y install haproxy

配置haproxy日志，修改/etc/rsyslog.conf

#去掉下面两行的注释

$ModLoad imudp

$UDPServerRun 514

添加一个配置文件/etc/rsyslog.d/haproxy.conf

local2.*                       /var/log/haproxy.log

修改/etc/sysconfig/rsyslog

#-r是允许接受外部日志

#-c 是说兼容syslog v2

#-m 是说每隔多长时间加一个时间戳，0表示不加

SYSLOGD_OPTIONS="-r -c 2"

修改haproxy配置文件，删掉没用的，添加状态监控页面

global

    # to have these messages end up in /var/log/haproxy.log you will

    # need to:

    #

    # 1) configure syslog to accept network log events.  This is done

    #    by adding the '-r' option to the SYSLOGD_OPTIONS in

    #    /etc/sysconfig/syslog

    #

    # 2) configure local2 events to go to the /var/log/haproxy.log

    #   file. A line like the following can be added to

    #   /etc/sysconfig/syslog

    #

    #    local2.*                       /var/log/haproxy.log

    #

    log         127.0.0.1 local2

    chroot      /var/lib/haproxy

    pidfile     /var/run/haproxy.pid

    maxconn     4000

    user        haproxy

    group       haproxy

    daemon

    # turn on stats unix socket

    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------

# common defaults that all the 'listen' and 'backend' sections will

# use if not designated in their block

#---------------------------------------------------------------------

defaults

    mode                    http

    log                     global

    option                  httplog

    option                  dontlognull

    option http-server-close

    option forwardfor       except 127.0.0.0/8

    option                  redispatch

    retries                 3

    timeout http-request    10s

    timeout queue           1m

    timeout connect         10s

    timeout client          1m

    timeout server          1m

    timeout http-keep-alive 10s

    timeout check           10s

    maxconn                 3000

#状态监控页面

listen stats 0.0.0.0:9001

    stats enable

    stats uri /haproxyadmin?stats

    stats realm HAProxy\ Statistics

    stats auth admin:admin

    stats admin if TRUE

启用

systemctl start haproxy && systemctl enable haproxy

3.1. 或者使用Nginx做TCP转发（本次使用Mysql做例子）

安装

yum -y install nginx

在/etc/nginx/nginx.conf中添加下面这段

stream {

    log_format tcp_proxy '$remote_addr [$time_local] '

                 '$protocol $status $bytes_sent $bytes_received '

                 '$session_time "$upstream_addr" '

                 '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';

    include /etc/nginx/tcp.d/*.conf;

}

在/etc/nginx/tcp.d/mysql.hccos.cn.conf中写入如下内容

server {

    listen          3306;

    proxy_connect_timeout 5s;

    proxy_timeout 30s;

        proxy_pass mysql;

    }

upstream mysql {

    server 172.16.0.25:3306 max_fails=3 fail_timeout=10s;

    server 172.16.0.26:3306 max_fails=3 fail_timeout=10s;

    server 172.16.0.27:3306 max_fails=3 fail_timeout=10s;

}

3.2. keepalived安装

yum安装

yum -y install keepalived

修改/etc/sysconfig/keepalived

KEEPALIVED_OPTIONS="-D -d -S 0"

修改/etc/rsyslog.d/keepalived.conf

local0.*    /var/log/keepalived.log

在两台机器上修改/etc/keepalived/keepalived.conf

10.30.2.48

global_defs {

   notification_email {

     eric.zhangtj@homecredit.cn

   }

   notification_email_from eric.zhangtj@homecredit.cn

   smtp_server 10.25.8.2

   smtp_connect_timeout 30

   router_id LVS_DEVEL

}

vrrp_script check_haproxy {

script "killall -0 haproxy"

        interval 1

        weight 21

}

vrrp_script chk_mantaince_down {

   script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"

   interval 1

   weight 2

}

vrrp_instance VI_148 {

    state MASTER

    interface ens192

    virtual_router_id 22

    garp_master_delay 1

    mcast_src_ip 172.16.0.48

    lvs_sync_daemon_interface ens192

    priority 110

    advert_int 2

    authentication {

        auth_type PASS

        auth_pass 1111

    }

    track_interface {

    ens192

    }

    virtual_ipaddress {

        172.16.0.148/24 dev ens192 label ens192:0

    }

    track_script {

    check_haproxy

    chk_mantaince_down

    }

}

vrrp_instance VI_149 {

    state BACKUP

    interface ens192

    virtual_router_id 23

    garp_master_delay 1

    mcast_src_ip 172.16.0.49

    lvs_sync_daemon_interface ens192

    priority 100

    advert_int 2

    authentication {

        auth_type PASS

        auth_pass 1111

    }

    track_interface {

        ens192

    }

    virtual_ipaddress {

        172.16.0.149/24 dev ens192 label ens192:1

    }

    track_script {

    check_haproxy

    chk_mantaince_down

    }

}

10.30.2.49

global_defs {

   notification_email {

     eric.zhangtj@homecredit.cn

   }

   notification_email_from eric.zhangtj@homecredit.cn

   smtp_server 10.25.8.2

   smtp_connect_timeout 30

   router_id LVS_DEVEL

}

vrrp_script check_haproxy {

script "killall -0 haproxy"

        interval 1

        weight 21

}

vrrp_script chk_mantaince_down {

   script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"

   interval 1

   weight 2

}

vrrp_instance VI_148 {

    state BACKUP

    interface ens192

    virtual_router_id 22

    garp_master_delay 1

    mcast_src_ip 172.16.0.48

    lvs_sync_daemon_interface ens192

    priority 100

    advert_int 2

    authentication {

        auth_type PASS

        auth_pass 1111

    }

    track_interface {

    ens192

    }

    virtual_ipaddress {

        172.16.0.148/24 dev ens192 label ens192:0

    }

    track_script {

    check_haproxy

    chk_mantaince_down

    }

}

vrrp_instance VI_49 {

    state MASTER

    interface ens192

    virtual_router_id 23

    garp_master_delay 1

    mcast_src_ip 172.16.0.49

    lvs_sync_daemon_interface ens192

    priority 110

    advert_int 2

    authentication {

        auth_type PASS

        auth_pass 1111

    }

    track_interface {

        ens192

    }

    virtual_ipaddress {

        172.16.0.149/24 dev ens192 label ens192:1

    }

    track_script {

    check_haproxy

    chk_mantaince_down

    }

}

启用服务

systemctl start keepalived && systemctl enable keepalived

3.3. 内核参数

修改/etc/sysctl.conf

# Controls IP packet forwarding

# 开启IP转发功能

net.ipv4.ip_forward = 1

# 开启允许绑定非本机的IP

net.ipv4.ip_nonlocal_bind = 1

sysctl -p

巴特西

【Linux】【Services】【SaaS】Docker+kubernetes（10. 利用反向代理实现服务高可用）

最新文章

热门文章