八，负载均衡相关配置文件

17主master：

nginx配置文件（17和18的配置文件都一样的）：

[root@master ~]# cat /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
 
include /usr/share/nginx/modules/*.conf;
 
events {
    worker_connections 1024;
}
 
stream {
 
    log_format  main  '  - []  ';
 
    access_log  /var/log/nginx/k8s-access.log  main;
 
    upstream k8s-apiserver {
       server 192.168.217.16:6443;
       server 192.168.217.11:6443;
    }
    
    server {
       listen 6443;
       proxy_pass k8s-apiserver;
    }
}
 
http {
    log_format  main  ' -  [] "" '
                      '  "" '
                      '"" ""';
 
    access_log  /var/log/nginx/access.log  main;
 
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;
 
    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;
 
    server {
        listen       80 default_server;
        server_name  _;
 
        location / {
        }
    }
}

keepalived的配置文件：

[root@master ~]# cat /etc/keepalived/keepalived.conf 
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_MASTER
} 
 
vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}
 
vrrp_instance VI_1 { 
    state MASTER 
    interface ens33
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 100    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        192.168.217.88/24
    } 
    track_script {
        check_nginx
    } 
}

18服务器上的keepalived配置文件（两个文件，其中一个是检测脚本，脚本两个节点都要有）：

[root@slave2 nginx-offline]# cat /etc/keepalived/check_nginx.sh 
#!/bin/bash
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")
if [ "$count" -eq 0 ];then
exit 1
else
exit 0
fi

[root@slave2 nginx-offline]# cat /etc/keepalived/keepalived.conf 
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_MASTER
} 
 
vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}
 
vrrp_instance VI_1 { 
    state BACKUP 
    interface ens33
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 80    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        192.168.217.88/24
    } 
    track_script {
        check_nginx
    } 
}

九，重启负载均衡相关服务

systemctl restart nginx keepalived

十，kube-apiserver服务所使用的证书文件内没有写vip地址，因此，16服务器上的kube-apiserver服务将会启动失败，需要重新生成证书：

在master节点，16服务器上，该文件内添加"192.168.217.88",

[root@master ~]# cat k8s/server-csr.json 
{
"CN": "kubernetes",
"hosts": [
"10.0.0.1",
"127.0.0.1",
"192.168.217.16",
"192.168.217.17",
"192.168.217.18",
"192.168.217.88",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}

重新生成证书：

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

拷贝证书文件（拷贝到本地和新master上）：

1. cp server*pem /opt/kubernetes/ssl/
2. scp /opt/kubernetes/ssl/server*pem master2:/opt/kubernetes/ssl/

重启服务，使得相关证书生效： 

systemctl restart kube-apiserver kubelet

十一，所有配置文件内添加VIP的IP地址192.168.217.88 ，并重启相关服务。

sed -i 's#192.168.217.16:6443#192.168.217.88:6443#' /opt/kubernetes/cfg/*
systemctl restart kubelet kube-proyx

十二，测试单元

在17服务器上，也就是负载均衡的主节点上，可以看到ens33网卡两个ip：

[root@slave1 ~]#  ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
    link/ether 00:0c:29:e9:9e:89 brd ff:ff:ff:ff:ff:ff
    inet 192.168.217.17/24 brd 192.168.217.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.217.88/24 scope global secondary ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fee9:9e89/64 scope link 
       valid_lft forever preferred_lft forever

此时，停止17上的nginx，在18服务上  ，ip a 命令应该可以看到ens33网卡两个IP，证明负载均衡漂移成功。

通过VIP 可以看到k8s版本：

[root@centos1 nginx-offline]# curl -k https://192.168.217.88:6443/version
{
  "major": "1",
  "minor": "18",
  "gitVersion": "v1.18.3",
  "gitCommit": "2e7996e3e2712684bc73f0dec0200d64eec7fe40",
  "gitTreeState": "clean",
  "buildDate": "2020-05-20T12:43:34Z",
  "goVersion": "go1.13.9",
  "compiler": "gc",
  "platform": "linux/amd64"