二进制k8s 集群新增加node 节点
2024-09-08 19:10:04
环境
| 名称 | ip地址 | cpu | 内存 |
|---|---|---|---|
| lgy-k8s-master0021 | 10.65.0.21 | 4c | 8G |
| lgy-k8s-node0012 | 10.65.0.12 | 4c | 8G |
node节点初始化（以新增加 lgy-k8s-node0012 节点为例）
#!/bin/sh
# 安装yum源
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
# 安装
yum --enablerepo=elrepo-kernel install kernel-ml-devel kernel-ml -y
# 设置生成新的grub
grub2-set-default 0
grub2-mkconfig -o /etc/grub2.cfg
# 移除旧版本工具包
yum remove kernel-tools-libs.x86_64 kernel-tools.x86_64 -y
# 安装新版本
yum --disablerepo=* --enablerepo=elrepo-kernel install -y kernel-ml-tools.x86_64
# 重启
reboot
# 查看内核版本
uname -sr
#1.修改内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
net.bridge.bridge-nf-call-arptables = 1
vm.swappiness = 0 #最大限度使用物理内存，然后才是 swap空间
vm.overcommit_memory=1
vm.panic_on_oom=0
EOF
sysctl --system
#2. 临时关闭
swapoff -a
#3. 永久关闭
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
#4. 开启ipvs
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
# 查看是否加载
lsmod | grep ip_vs
# 配置开机自加载
cat <<EOF>> /etc/rc.local
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod +x /etc/rc.d/rc.local
#5. 关闭sellinux
#临时关闭
setenforce 0
#永久关闭
sed -i 's#SELINUX=enforcing#SELINUX=Disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=Disabled#g' /etc/selinux/config
#6.禁用postfix
systemctl stop postfix
systemctl disable postfix
#7.关闭swap
echo "0" > /proc/sys/vm/swappiness
#8.开启转发
echo 1 > /proc/sys/net/ipv4/ip_forward
#9. 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
#10. ntp对时间
yum install ntpdate -y
ntpdate -u cn.ntp.org.cn
echo "* * * * * ntpdate -u cn.ntp.org.cn" >> /var/spool/cron/root
#11.文件打开数
echo " * soft nofile 864000 " >> /etc/security/limits.conf
echo " * hard nofile 864000 " >> /etc/security/limits.conf
#12.安装docker-ce
yum remove docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-engine
# Install using the repository
yum install -y yum-utils \
device-mapper-persistent-data \
lvm2
#快的镜像源
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#这个比较慢
yum-config-manager \
--add-repo \
https://download.docker.com/linux/centos/docker-ce.repo
yum install docker-ce docker-ce-cli containerd.io -y
systemctl start docker
systemctl enable docker
#13. 配置加速器
cat <<EOF > /etc/docker/daemon.json
{
"registry-mirrors": [
"https://1nj0zren.mirror.aliyuncs.com",
"https://docker.mirrors.ustc.edu.cn",
"http://f1361db2.m.daocloud.io",
"https://registry.docker-cn.com"
]
}
EOF
#14. 配置日志切割
cat <<EOF > /etc/logrotate.d/docker-logs
/var/lib/docker/containers/*/*.log {
rotate 7
daily
compress
size=1M
missingok
delaycompress
copytruncate
}
EOF
#15.安装 nfs 依赖
yum install nfs-utils rpcbind vim -y
systemctl start rpcbind
systemctl start nfs
node节点环境配置
#修改解析文件
# cat /etc/resolv.conf
search k8s.host.com
nameserver 10.65.10.1
#装包,kube-proxy 依赖包
yum -y install conntrack
#修改主机名
# hostname
lgy-k8s-node0012.k8s.host.com
#操作dns主机记录
lgy-k8s-node0012.k8s IN A 10.65.0.12
#dns reload生效
/opt/named/sbin/rndc reload
#创建相关数据目录
mkdir -p /opt/kubernetes/server/bin/
mkdir -p /etc/kubernetes/pki
mkdir -p /etc/kubernetes/kubeconfig/
mkdir -p /export/kubernetes/logs/
mkdir -p /export/kubernetes/kubelet
mkdir -p /etc/flannel/pki/
master节点生成bootstrap
#在master节点 10.65.0.21 给Node节点创建bootstrap 文件
# cd /opt/kubernetes/server/bin
cat > environment.sh <<EOF
#!/usr/bin/bash
KUBE_APISERVER="https://10.65.2.10:6443"
BOOTSTRAP_TOKEN="head -c 16 /dev/urandom | od -An -t x | tr -d ' '"
NODE_NAMES=(lgy-k8s-node0012 )
CLUSTER_DNS_SVC_IP="10.254.0.2"
CLUSTER_DNS_DOMAIN="cluster.local"
EOF
#source environment.sh
#node_name=${NODE_NAMES}
# 创建 token
export BOOTSTRAP_TOKEN=$(kubeadm token create \
--description kubelet-bootstrap-token \
--groups system:bootstrappers:${node_name} \
--kubeconfig ~/.kube/config)
#设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig
# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig
#分发bootstrap token 文件
scp kubelet-bootstrap-lgy-k8s-node0012.kubeconfig 10.65.0.12:/etc/kubernetes/kubeconfig/kubelet-bootstrap.kubeconfig
#将kubelet、kube-proxy等文件拷贝至node节点
scp /opt/kubernetes/server/bin/kubelet /opt/kubernetes/server/bin/kube-proxy root@10.65.0.12:/opt/kubernetes/server/bin/
scp /etc/kubernetes/pki/ca.pem root@10.65.0.12:/etc/kubernetes/pki/
scp /usr/local/bin/flanneld root@10.65.0.12:/usr/local/bin/
scp /opt/kubernetes/server/bin/kube-proxy root@10.65.0.12:/opt/kubernetes/server/bin/
scp /etc/flannel/pki/flanneld.pem 10.65.0.12:/etc/flannel/pki/
scp /etc/flannel/pki/flanneld-key.pem 10.65.0.12:/etc/flannel/pki/
scp /usr/local/bin/mk-docker-opts.sh 10.65.0.12:/usr/local/bin/
scp /etc/kubernetes/kubeconfig/kube-proxy.kubeconfig 10.65.0.12:/etc/kubernetes/kubeconfig/
node节点操作(lgy-k8s-node0012 )
#增加启动文件
# cat /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service
[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/opt/kubernetes/server/bin/kubelet \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBELET_API_SERVER \
$KUBELET_ADDRESS \
$KUBELET_PORT \
$KUBELET_HOSTNAME \
$KUBELET_POD_INFRA_CONTAINER \
$KUBELET_ARGS
Restart=on-failure
RestartSec=5
StartLimitInterval=0
[Install]
WantedBy=multi-user.target
# cat /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=false --log-dir=/export/kubernetes/logs/"
KUBE_LOG_LEVEL="--v=2"
KUBE_ALLOW_PRIV="--allow-privileged=true"
# cat /etc/kubernetes/kubelet
KUBELET_HOSTNAME="--hostname-override=lgy-k8s-node0012" #修改为node节点本机主机名
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=harbor.k8s.moviebook.cn/library/pause:latest" #本地要配置镜像仓库，确保拉取镜像正常
KUBELET_ARGS="--cgroup-driver=systemd \
--config=/etc/kubernetes/kubelet-config.yaml \
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig/kubelet-bootstrap.kubeconfig \
--kubeconfig=/etc/kubernetes/kubeconfig/kubelet.kubeconfig \
--cert-dir=/etc/kubernetes/pki \
--root-dir=/export/kubernetes/kubelet \
--image-pull-progress-deadline=15m"
# cat /etc/kubernetes/kubelet-config.yaml
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: "10.65.0.12" #node节点ip地址
staticPodPath: ""
syncFrequency: 1m
fileCheckFrequency: 20s
httpCheckFrequency: 20s
staticPodURL: ""
port: 10250
readOnlyPort: 10255
rotateCertificates: true
serverTLSBootstrap: true
authentication:
anonymous:
enabled: false
webhook:
enabled: true
x509:
clientCAFile: "/etc/kubernetes/pki/ca.pem"
authorization:
mode: Webhook
registryPullQPS: 0
registryBurst: 20
eventRecordQPS: 0
eventBurst: 20
enableDebuggingHandlers: true
enableContentionProfiling: true
healthzPort: 10248
healthzBindAddress: "10.65.0.12"
clusterDomain: "cluster.local"
clusterDNS:
- "10.254.0.2"
nodeStatusUpdateFrequency: 10s
nodeStatusReportFrequency: 1m
imageMinimumGCAge: 2m
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
volumeStatsAggPeriod: 1m
kubeletCgroups: ""
systemCgroups: ""
cgroupRoot: ""
cgroupsPerQOS: true
cgroupDriver: cgroupfs
runtimeRequestTimeout: 10m
hairpinMode: promiscuous-bridge
maxPods: 50
podCIDR: "172.0.0.0/16"
podPidsLimit: -1
resolvConf: /etc/resolv.conf
maxOpenFiles: 1000000
kubeAPIQPS: 1000
kubeAPIBurst: 2000
serializeImagePulls: false
evictionHard:
memory.available: "100Mi"
nodefs.available: "10%"
nodefs.inodesFree: "5%"
imagefs.available: "15%"
evictionSoft: {}
enableControllerAttachDetach: true
failSwapOn: true
containerLogMaxSize: 20Mi
containerLogMaxFiles: 10
systemReserved: {}
kubeReserved: {}
systemReservedCgroup: ""
kubeReservedCgroup: ""
enforceNodeAllocatable: ["pods"]
#/etc/kubernetes/kubeconfig/kubelet-bootstrap.kubeconfig 在前面已生成
#/etc/kubernetes/kubeconfig/kubelet.kubeconfig 文件会重启kubelet自动生成
#systemctl start kubelet
#systemctl enable kubelet
#kube-proxy 安装
# cat /etc/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/proxy
ExecStart=/opt/kubernetes/server/bin/kube-proxy \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_PROXY_ARGS
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
# cat /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=false --log-dir=/export/kubernetes/logs/"
KUBE_LOG_LEVEL="--v=2"
KUBE_ALLOW_PRIV="--allow-privileged=true"
# cat /etc/kubernetes/proxy
KUBE_PROXY_ARGS="--config=/etc/kubernetes/kube-proxy-config.yaml"
# cat /etc/kubernetes/kube-proxy-config.yaml
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clientConnection:
burst: 200
kubeconfig: "/etc/kubernetes/kubeconfig/kube-proxy.kubeconfig"
qps: 100
bindAddress: 10.65.0.12
healthzBindAddress: 10.65.0.12:10256
metricsBindAddress: 10.65.0.12:10249
enableProfiling: true
clusterCIDR: 172.0.0.0/16
hostnameOverride: lgy-k8s-node0036
mode: "ipvs"
portRange: ""
iptables:
masqueradeAll: false
ipvs:
scheduler: nq
excludeCIDRs: []
# systemctl start kube-proxy
#systemctl enable kube-proxy
#flannel 安装
# cat /etc/systemd/system/flannel.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
ExecStart=/usr/local/bin/flanneld \
-etcd-cafile=/etc/kubernetes/pki/ca.pem \
-etcd-certfile=/etc/flannel/pki/flanneld.pem \
-etcd-keyfile=/etc/flannel/pki/flanneld-key.pem \
-etcd-endpoints=https://10.65.10.1:2379,https://10.65.10.2:2379,https://10.65.10.4:2379 \
-etcd-prefix=/etc/flannel/ \
-iface=bond0 \
-v=2
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure
RestartSec=5
StartLimitInterval=0
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
#启动flannel
systemctl start flannel
systemctl enable flannel
#查看kubelet 日志
#journalctl -f -u kubelet 查看日志发现有 lgy-k8s-node0012 not found 信息输出，登录master节点
#master节点执行
kubectl get csr | awk '/Pending/ {print $1}' | xargs kubectl certificate approve
#查看node节点
kubectl get node
- 修改 docker 配置文件 (如果docker ip 地址和flannel 分配的不一致，需要修改docker 配置文件),重启docker 生效,还需要删除主机所有docker 容器，停止kubelet进行操作
#增加配置文件 EnvironmentFile=-/run/flannel/docker
#cat /usr/lib/systemd/system/docker.service
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP $MAINPID
Environment="PATH=/opt/k8s/bin:/bin:/sbin:/usr/bin:/usr/sbin"
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS
TimeoutSec=0
RestartSec=2
Restart=always
最新文章
- kafka主题offset各种需求修改方法
- 网络库crash以及boost asio strand dispath分析
- SQL Server 数据类型
- 2010noip提高组解题报告
- c++分割字符串(类似于boost::split)
- 利用html5压缩图片，产出base64图片
- 日志组件 logback
- Web Api 2.0中使用Swagger生成Api文档的2个小Tips
- ES6 正则的扩展
- wireshark基础学习—第二部分wireshark的基础操作
- 关于css3 渐变色
- vue provide和inject 父组件和子孙通信
- MongoDB的基本操作：服务端启动，客户端连接，CRUD操作
- C++ leetcode Longest Substring Without Repeating Characters
- shell脚本中解决SCP命令需要输入密码的问题
- pre-processing预处理
- Java入门：基础算法之计算园的面积
- 小识.htaccess文件
- python学习笔记（excel简单操作）
- Spring-Boot ☞ ShapeFile文件读写工具类+接口调用