k8s-keepalived高可用集群

环境规划

角色	IP	主机名
Master 1	192.168.0.10	`k8s-master01`
Master 2	192.168.0.20	`k8s-master02`
Master 3	192.168.0.30	`k8s-master03`
Worker 1	192.168.0.40	`k8s-node01`
Worker 2	192.168.0.50	`k8s-node02`
Worker 3	192.168.0.60	`k8s-node03`
虚拟IP（VIP）	192.168.0.100	提供 kube-apiserver 的高可用

操作系统版本: CentOS 8.4.2105

kubernetes版本: kubernetes 1.32.0

软件源地址：

docker源:

https://mirrors.aliyun.com/docker-ce/linux/centos/8.4/x86_64/stable

kubernetes 源:

https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.32/rpm

epel 源：

https://mirrors.tuna.tsinghua.edu.cn/epel/8/Everything/x86_64

BaseOS 源：

https://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105/BaseOS/x86_64/os/

AppStream 源：

https://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105/AppStream/x86_64/os/

kernel-ml升级源:

https://mirrors.aliyun.com/elrepo/kernel/el8/x86_64

容器镜像仓库地址:

https://k8s.m.daocloud.io

https://docker.m.daocloud.io

虚拟机模板制作

#设置主机名
hostnamectl set-hostname template

#关闭防火墙和SELINUX
systemctl disable --now firewalld.service
setenforce 0
vim /etc/sysconfig/selinux
SELINUX=disabled

#配置时间同步
vim /etc/chrony.conf
server ntp.aliyun.com iburst
systemctl enable --now chronyd
chronyc sources

#配置软件源
rm -rf /etc/yum.repos.d/*
vim /etc/yum.repos.d/system.repo
[baseos]
name=BaseOS
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105/BaseOS/x86_64/os/
gpgcheck=0
enabled=1
[appstream]
name=AppStream
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105/AppStream/x86_64/os/
gpgcheck=0
enabled=1

vim /etc/yum.repos.d/epel.repo
[epel]
name=epel
baseurl=https://mirrors.tuna.tsinghua.edu.cn/epel/8/Everything/x86_64
gpgcheck=0
enabled=1

vim /etc/yum.repos.d/docker.repo
[docker]
name=docker
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/8.4/x86_64/stable
gpgcheck=0
enabled=1

vim /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.32/rpm
gpgcheck=0
enabled=1

vim /etc/yum.repos.d/elrepo.repo
[elrepo]
name=elrepo
baseurl=https://mirrors.aliyun.com/elrepo/kernel/el8/x86_64
gpgcheck=0
enabled=1

#关闭swap
swapoff -a
sed -ri 's/.*swap.*/#&/g' /etc/fstab

#内核调优配置
cat > /etc/sysctl.d/k8s_better.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF

##保证 CNI 网络插件正常工作（桥接转发和 iptables 处理）。
##提高内核资源限制（fd、inotify、conntrack），支持大规模 Pod/连接数。
##优化内存管理，避免 swap 带来的性能问题，控制 OOM 行为。
##减少 IPv6 干扰（只用IPV4时开启）

modprobe br_netfilter
modprobe ip_conntrack
sysctl -p /etc/sysctl.d/k8s_better.conf
yum install -y net-tools conntrack ipvsadm ipset iptables curl sysstat libseccomp wget

cat > /etc/sysconfig/modules/ipvs.modules <<END
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack"
for kernel_module in ${ipvs_modules};do
 /sbin/modinfo -F filename ${kernel_module} > /dev/null 2>&1
 if [ 0 -eq 0 ];then
 /sbin/modprobe ${kernel_module}
 fi
done
END
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
##验证效果
lsmod | grep -e ip_vs -e nf_conntrack

#配置containerd依赖的内核模块
cat  > /etc/modules-load.d/containerd.conf <<END
overlay
br_netfilter
END

modprobe overlay
modprobe br_netfilter

#配置hosts列表
vim /etc/hosts
192.168.0.10 k8s-master01
192.168.0.20 k8s-master02
192.168.0.30 k8s-master03
192.168.0.40 k8s-node01
192.168.0.50 k8s-node02
192.168.0.60 k8s-node03
192.168.0.100 k8s-vip

#更新CentOS8.4支持的最新内核
yum install kernel-ml -y

#设置通过新的内核启动
grub2-set-default 0
grub2-mkconfig -o /boot/grub2/grub.cfg

#启用cgroup v2的特性
grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=1"

#设置完成后，请手动重启，看是否系统可以从新内核启动，并检查cgroup v2是否启用
reboot
mount | grep cgroup

#删除网卡的UUID
sed -i '/^UUID=/d' /etc/sysconfig/network-scripts/ifcfg-*

#如果没有删除已经克隆，则需要每台机器手动删除网卡的UUID后查看验证UUID是否一致
cat /sys/class/dmi/id/product_uuid

#清空VMID
echo > /etc/machine-id

#关机并克隆六台虚拟机，该虚拟机作为默认，使用完整克隆
##注意克隆完成后，请修改每台主机的网络信息和主机名信息

高可用集群配置

#注意高可用配置仅在三台master上进行配置
#安装Nginx（每台master都需要做）
yum install nginx nginx-all-modules  -y

#配置基于TCP端口的反向代理（每台master都需要做）
vim /etc/nginx/nginx.conf
在events 下增加如下内容
stream {
        upstream api-server {
                server 192.168.0.10:6443;
                server 192.168.0.20:6443;
                server 192.168.0.30:6443;
        }
        server {
                listen 16443;
                proxy_pass api-server;
        }
}

#验证（每台master都需要做）
 systemctl enable --now nginx
 ss -tanlp|grep 16443

#安装Keepalive（每台master都需要做）
yum install  keepalived -y
#配置Keepalive（master01上操作）
vim /etc/keepalived/keepalived.conf
清空文件内容，写入以下内容
! Configuration File for keepalived
  
global_defs {
   router_id master01
}

vrrp_instance VI_1 {
    state MASTER
    interface ens160
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.0.100/24
    }
}
systemctl  enable --now keepalived

#配置Keepalive（master02上操作）
vim /etc/keepalived/keepalived.conf
清空文件内容，写入以下内容
! Configuration File for keepalived
  
global_defs {
   router_id master02
}

vrrp_instance VI_1 {
    state MASTER
    interface ens160
    virtual_router_id 51
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.0.100/24
    }
}
systemctl  enable --now keepalived
#配置Keepalive（master03上操作）
vim /etc/keepalived/keepalived.conf
清空文件内容，写入以下内容
! Configuration File for keepalived
  
global_defs {
   router_id master03
}

vrrp_instance VI_1 {
    state MASTER
    interface ens160
    virtual_router_id 51
    priority 50
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.0.100/24
    }
}
systemctl  enable --now keepalived

#验证VIP（master01上操作）
ip addr|grep 192.168.0.100

kubeadm部署

##注意以下步骤所有节点都要做
#安装containerd
yum install containerd -y

#生成配置文件
containerd config default > /etc/containerd/config.toml
#修改配置文件
vim /etc/containerd/config.toml
SystemdCgroup = false 
           改为 SystemdCgroup = true
sandbox_image = "k8s.gcr.io/pause:3.6"
           改为 sandbox_image = "uhub.service.ucloud.cn/k8s_v1.32.0/pause:3.6"
           
#配置container镜像源从/etc/containerd/certs.d获取
sed -i "s|config_path = \"\"|config_path = \"/etc/containerd/certs.d\"|g" /etc/containerd/config.toml

#配置docker镜像加速器
mkdir -p /etc/containerd/certs.d/docker.io
mkdir -p /etc/containerd/certs.d/registry.k8s.io
cat > /etc/containerd/certs.d/docker.io/hosts.toml << EOF
server = "https://registry-1.docker.io"
[host."https://docker.m.daocloud.io"]
  capabilities = ["pull","resolve"]
EOF

#配置k8s镜像加速器
cat > /etc/containerd/certs.d/registry.k8s.io/hosts.toml <<EOF
server = "https://registry.k8s.io"
[host."https://k8s.m.daocloud.io"]
  capabilities = ["pull", "resolve"]
EOF

#启动containerd
systemctl enable --now containerd

#安装k8s组件(所有节点操作)
yum install -y kubectl kubelet kubeadm 

#修改kubelet(所有节点操作)
vim /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"

#启动kubelet(所有节点操作)
systemctl enable --now kubelet

#生成初始化文件（仅master01操作）
kubeadm config print init-defaults > ~/kubeadm.yaml

#修改初始化文件
vim ~/kubeadm.yaml
advertiseAddress: 1.2.3.4
      改为 advertiseAddress: 192.168.0.10
name: node
      改为 name: k8s-master01 (当前节点的主机名)
kind: ClusterConfiguration
      下面新增一行 controlPlaneEndpoint: "192.168.0.100:16443"
serviceSubnet: 10.96.0.0/12
       下面新增一行 podSubnet: 172.16.0.0/12

#初始化集群
kubeadm init --config ~/kubeadm.yaml --upload-certs

#配置管理员认证凭据
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

#查看节点状态
kubectl  get  nodes

#下载calico的yaml文件
wget  https://docs.projectcalico.org/archive/v3.25/manifests/calico.yaml

#修改calico的pod网段和物理网卡
vim calico.yaml
- name: CALICO_IPV4POOL_CIDR (注意此处的网络配置应和kubeadm.yaml中的pod网段一致)
  value: "172.17.0.0/12"
- name: IP_AUTODETECTION_METHOD （填写物理网卡的名称）
  value: "interface=ens160"

#部署calico
kubectl apply -f calico.yaml

#验证calico是否成功运行
kubectl  get -n kube-system pod |grep calico

#使用前面初始化生成的token，或者手动生成，以下是手动生成方式
#生成token
kubeadm token create --print-join-command --ttl 24h

kubeadm join 192.168.0.100:16443 --token r5jgpa.74w1vpnq71fwq1oc --discovery-token-ca-cert-hash sha256:12420fedefa1d14b73ed467df840e71d9e37d3d50a57a25b68dd2d18afca94b5

#生成certificate-key
kubeadm init phase upload-certs --upload-certs
50358e3308dab0bed520f37fe4ada47cfcbe1b5e59b0c5d1e9bb0972fad581ae

#拼接master加入集群命令（在新的master节点上执行）
kubeadm join 192.168.0.100:16443 --token r5jgpa.74w1vpnq71fwq1oc --discovery-token-ca-cert-hash sha256:12420fedefa1d14b73ed467df840e71d9e37d3d50a57a25b68dd2d18afca94b5 --control-plane --certificate-key  50358e3308dab0bed520f37fe4ada47cfcbe1b5e59b0c5d1e9bb0972fad581ae

#验证节点是否加入成功
kubectl get nodes

#生成token
kubeadm token create --print-join-command

#在待加入node节点执行生成的token
kubeadm join 192.168.0.100:16443 --token r5jgpa.74w1vpnq71fwq1oc --discovery-token-ca-cert-hash sha256:12420fedefa1d14b73ed467df840e71d9e37d3d50a57a25b68dd2d18afca94b5

#验证节点是否加入成功
kubectl  get nodes

#编写yaml文件
vim nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
  labels:
    app: nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:latest
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-service
spec:
  type: NodePort
  selector:
    app: nginx
  ports:
    - port: 80           
      targetPort: 80     
      nodePort: 30080  
#部署Nginx
kubectl apply nginx.yaml

#访问测试
curl http://集群任意节点IP:30080 

#清理测试
kubectl  delete -f nginx.yaml

目录CONTENT

kubeadm部署高可用k8s集群

k8s-keepalived高可用集群

环境规划

虚拟机模板制作

高可用集群配置

kubeadm部署