服务器
20.0.0.206    10.0.0.206 bs-hk-hk01 高可用负载均衡节点 2c2g
20.0.0.207    10.0.0.207 bs-hk-hk02 高可用负载均衡节点 2c2g
软件版本
Keepalived 2.0.20
haproxy 2.1.2
Keepalived 安装配置
两个节点都安装 以bs-hk-hk01为例
#安装依赖包
[root@bs-hk-hk01 tools]#yum -y install gcc openssl-devel libnl3-devel pcre-devel
[root@bs-hk-hk01 tools]# ls
haproxy-2.1.2.tar.gz keepalived-2.0.20.tar.gz
[root@bs-hk-hk01 tools]# tar -zvxf keepalived-2.0.20.tar.gz
[root@bs-hk-hk01 keepalived-2.0.20]# ./configure --prefix=/usr/local/keepalived-2.0.20
[root@bs-hk-hk01 keepalived-2.0.20]# echo $?
0
[root@bs-hk-hk01 keepalived-2.0.20]# make && make install
[root@bs-hk-hk01 keepalived-2.0.20]# echo $?
0
#配置文件放在默认路径
[root@bs-hk-hk01 keepalived-2.0.20]# mkdir /etc/keepalived/
[root@bs-hk-hk01 keepalived-2.0.20]# cp /usr/local/keepalived-2.0.20/etc/keepalived/keepalived.conf /etc/keepalived/
#keepalived启动脚本变量引用文件，默认文件路径是/etc/sysconfig/，也可以不做软链接，直接修改启动脚本中文件路径即可（安装目录下）
[root@bs-hk-hk01 keepalived-2.0.20]# cp /usr/local/keepalived-2.0.20/etc/sysconfig/keepalived /etc/sysconfig/
[root@bs-hk-hk01 keepalived-2.0.20]# cp /tools/keepalived-2.0.20/keepalived/keepalived.service /etc/systemd/system/

将keepalived主程序加入到环境变量（安装目录下）

[root@bs-hk-hk01 keepalived-2.0.20]# ln -s /usr/local/keepalived-2.0.20/sbin/keepalived /usr/sbin/
#keepalived启动脚本，这个从keepalived源码目录复制，安装目录中没有
[root@bs-hk-hk01 keepalived-2.0.20]# cp /tools/keepalived-2.0.20/keepalived/etc/init.d/keepalived /etc/init.d/
[root@bs-hk-hk01 keepalived-2.0.20]# chmod 755 /etc/init.d/keepalived
[root@bs-hk-hk01 keepalived-2.0.20]# systemctl enable keepalived.service
Created symlink from /etc/systemd/system/multi-user.target.wants/keepalived.service to /etc/systemd/system/keepalived.service.
[root@bs-hk-hk01 keepalived-2.0.20]# mkdir /var/log/keepalived
[root@bs-hk-hk01 keepalived-2.0.20]# vim /etc/sysconfig/keepalived
KEEPALIVED_OPTIONS="-D -d -S 0"
[root@bs-hk-hk01 keepalived-2.0.20]# vim /etc/rsyslog.d/keepalived.conf
local0.* /var/log/keepalived/keepalived.log
#修改 /etc/rsyslog.conf 文件，添加如下：
[root@bs-hk-hk01 log]# diff /etc/rsyslog.conf{,.bak}
74c74

< local0.* /var/log/keepalived/keepalived.log

[root@bs-hk-hk01 keepalived-2.0.20]# systemctl restart rsyslog
[root@bs-hk-hk01 keepalived-2.0.20]# systemctl start keepalived.service
[root@bs-hk-hk01 keepalived]# cp /lib/systemd/system/keepalived.service{,.bak}
[root@bs-hk-hk01 keepalived]# !vim
vim /lib/systemd/system/keepalived.service
[root@bs-hk-hk01 keepalived]# diff /lib/systemd/system/keepalived.service{,.bak}
10,11c10,11
< EnvironmentFile=/etc/sysconfig/keepalived

< ExecStart=/sbin/keepalived $KEEPALIVED_OPTIONS

EnvironmentFile=-/usr/local/keepalived-2.0.20/etc/sysconfig/keepalived
ExecStart=/usr/local/keepalived-2.0.20/sbin/keepalived $KEEPALIVED_OPTIONS
[root@bs-hk-hk01 keepalived]# systemctl daemon-reload
[root@bs-hk-hk01 keepalived]# systemctl restart keepalived.service

keepalived 配置
#bs-hk-hk01
[root@bs-hk-hk01 keepalived]# cat keepalived.conf
! Configuration File for keepalived
! By zisefeizhu

vrrp_script chk_haproxy_port {
script "/service/scripts/chk_hapro.sh"
interval 2
weight -5
fall 2
rise 1
}

vrrp_instance kubernetes_master {
state MASTER
interface eth0
virtual_router_id 1
priority 150
advert_int 1
unicast_src_ip 20.0.0.206
unicast_peer {
20.0.0.207
}
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
20.0.0.250 dev eth0 label eth0:1
}
track_script {
chk_haproxy_port
}
}

#bs-hk-hk02
[root@bs-hk-hk02 keepalived]# cp keepalived.conf{,.bak}
[root@bs-hk-hk01 keepalived]# scp keepalived.conf 20.0.0.207:/etc/keepalived/keepalived.conf
[root@bs-hk-hk02 keepalived]# cat keepalived.conf
! Configuration File for keepalived
! By zisefeizhu

vrrp_script chk_haproxy_port {
script "/service/scripts/chk_hapro.sh"
interval 2
weight -5
fall 2
rise 1
}

vrrp_instance kubernetes_master {
state BACKUP
interface eth0
virtual_router_id 1
priority 70
advert_int 1
unicast_src_ip 20.0.0.207
unicast_peer {
20.0.0.206
}
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
20.0.0.250 dev eth0 label eth0:1
}
track_script {
chk_haproxy_port
}
}
[root@bs-hk-hk02 keepalived]# systemctl restart keepalived.service

测试
[root@bs-hk-hk01 keepalived]# systemctl stop keepalived.service
[root@bs-hk-hk02 keepalived]# hostname -I
20.0.0.207 20.0.0.250 10.0.0.207
[root@bs-hk-hk01 keepalived]# systemctl restart keepalived.service
[root@bs-hk-hk01 keepalived]# hostname -I
20.0.0.206 20.0.0.250 10.0.0.206
[root@bs-hk-hk02 keepalived]# hostname -I
20.0.0.207 10.0.0.207

systemctl enable keepalived.service

Haproxy 安装配置
两个节点都安装 以bs-hk-hk01为例
[root@bs-hk-hk01 ~]# yum install vim iotop bc gcc gcc-c++ glibc glibc-devel pcre pcre-devel openssl openssl-devel zip unzip zlib-devel net-tools lrzsz tree ntpdate telnet lsof tcpdump wget libevent libevent-devel bc systemd-devel bash-completion traceroute libtermcap-devel ncurses-devel libevent-devel readline-devel -y
#LUA编译安装及Systemd开发包
[root@bs-hk-hk01 ~]# cd /tools/
[root@bs-hk-hk01 tools]# curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz
[root@bs-hk-hk01 tools]# ls
haproxy-2.1.2.tar.gz keepalived-2.0.20 keepalived-2.0.20.tar.gz lua-5.3.5.tar.gz
[root@bs-hk-hk01 tools]# tar -zxvf lua-5.3.5.tar.gz
[root@bs-hk-hk01 lua-5.3.5]# make linux
[root@bs-hk-hk01 lua-5.3.5]# echo $?
0
[root@bs-hk-hk01 lua-5.3.5]# make INSTALL_TOP=/usr/local/lua install
[root@bs-hk-hk01 lua-5.3.5]# echo $?
0
[root@bs-hk-hk01 lua-5.3.5]# yum install systemd-devel

#安装haproxy
[root@bs-hk-hk01 lua-5.3.5]# cd ..
[root@bs-hk-hk01 tools]# ls
haproxy-2.1.2.tar.gz keepalived-2.0.20 keepalived-2.0.20.tar.gz lua-5.3.5 lua-5.3.5.tar.gz
[root@bs-hk-hk01 tools]# tar xf haproxy-2.1.2.tar.gz
[root@bs-hk-hk01 tools]# cd haproxy-2.1.2/
[root@bs-hk-hk01 haproxy-2.1.2]# make -j $(nproc) TARGET=linux-glibc USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 LUA_LIB=/usr/local/lua/lib/ LUA_INC=/usr/local/lua/include/ USE_PCRE=1 USE_SYSTEMD=1 PREFIX=/usr/local/haproxy
[root@bs-hk-hk01 haproxy-2.1.2]# make install PREFIX=/usr/local/haproxy
[root@bs-hk-hk01 haproxy-2.1.2]# cp haproxy /usr/sbin/
#准备启动脚本
[root@bs-hk-hk01 haproxy-2.1.2]# vim /usr/lib/systemd/system/haproxy.service
[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target
[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid
ExecReload=/bin/kill -USR2 $MAINPID
[Install]
WantedBy=multi-user.target
#配置文件
[root@bs-hk-hk01 haproxy-2.1.2]# cd /etc/haproxy/
[root@bs-hk-hk01 log]# cat /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------

Global settings

By zisefeizhu

#---------------------------------------------------------------------
global
maxconn 10000
uid 99
gid 99
daemon
nbproc 2
cpu-map 1 0
cpu-map 2 1
log 127.0.0.1 local3 info
#log 127.0.0.1 local1 notice

#默认配置
defaults
log global
mode http
#option http-keep-alive
option httpclose
option forwardfor
option abortonclose
maxconn 10000
mode http
timeout connect 300000ms
timeout client 300000ms
timeout server 300000ms
balance leastconn

#统计页面配置
listen stats
bind 0.0.0.0:9999
mode http
option httplog
maxconn 10
stats refresh 30s
stats enable
log global
stats uri /haproxy-status
stats realm zisefeizhu\ Haproxy
stats auth admin:zisefeizhu
stats hide-version

#K8S-API-Server
frontend K8S_API
bind *:8443
mode tcp
default_backend k8s_api_nodes_6443

backend k8s_api_nodes_6443
mode tcp
balance leastconn
server 20.0.0.200 20.0.0.200:6443 check inter 2000 fall 3 rise 5
server 20.0.0.201 20.0.0.201:6443 check inter 2000 fall 3 rise 5
server 20.0.0.202 20.0.0.202:6443 check inter 2000 fall 3 rise 5

[root@bs-hk-hk01 haproxy]# mkdir /var/lib/haproxy
[root@bs-hk-hk01 haproxy]# chown 99.99 /var/lib/haproxy/ -R
[root@bs-hk-hk01 haproxy]# vim /etc/rsyslog.conf

对如下两行取消注释

$ModLoad imudp
$UDPServerRun 514

在末尾添加如下行

local3.* /var/log/haproxy/haproxy.log
*.info;mail.none;authpriv.none;cron.none;local0.none;local3.none
[root@bs-hk-hk01 log]# systemctl restart rsyslog
[root@bs-hk-hk01 log]# systemctl enable haproxy.service
Created symlink from /etc/systemd/system/multi-user.target.wants/haproxy.service to /usr/lib/systemd/system/haproxy.service.
[root@bs-hk-hk01 haproxy]# systemctl start haproxy.service

设置服务启动顺序及依赖关系
[root@bs-hk-hk01 keepalived]# diff /usr/lib/systemd/system/keepalived.service{,.bak02}
3c3

< After=network-online.target syslog.target haproxy.service

After=network-online.target syslog.target
5d4
< Requires=haproxy.service
[root@bs-hk-hk01 keepalived]# scp /usr/lib/systemd/system/keepalived.service 20.0.0.207:/usr/lib/systemd/system/keepalived.service

检查脚本
[root@bs-hk-hk01 keepalived]# mkdir /service/scripts -p
[root@bs-hk-hk01 keepalived]# vim /service/scripts/chk_hapro.sh
##########################################################################
#Author: zisefeizhu
#QQ: 2********0
#Date: 2020-02-02
#FileName: /service/scripts/chk_hapro.sh
#URL: https://www.cnblogs.com/zisefeizhu/
#Description: The test script
#Copyright (C): 2020 All rights reserved
##########################################################################
#!/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
export $PATH
counts=$(ps -ef|grep -w "haproxy"|grep -v grep|wc -l)
if [ "${counts}" = "0" ]; then
systemctl restart keepalived.service
sleep 2
counts=$(ps -ef|grep -w "haproxy"|grep -v grep|wc -l)
if [ "${counts}" = "0" ]; then
systemctl stop keepalived.service
fi
fi

查看日志
[root@bs-hk-hk01 log]# tree keepalived/ haproxy/
keepalived/
└── keepalived.log
haproxy/
└── haproxy.log

0 directories, 2 files

完成！