创建 kube-scheduler 证书和私钥

创建证书签名请求：

cd /opt/k8s/work

cat > kube-scheduler-csr.json <<EOF

{

    "CN": "system:kube-scheduler",

    "hosts": [

        "127.0.0.1",

        "10.0.20.11",

        "10.0.20.12",

        "10.0.20.13",

        "node01.k8s.com",

        "node02.k8s.com",

        "node03.k8s.com"

    ],

    "key": {

        "algo": "rsa",

        "size": 2048

    },

    "names": [

      {

        "C": "CN",

        "ST": "BeiJing",

        "L": "BeiJing",

        "O": "system:kube-scheduler",

        "OU": "4Paradigm"

      }

    ]

}

EOF

hosts 列表包含所有 kube-scheduler 节点 IP；
CN 和 O 均为 system:kube-scheduler，kubernetes 内置的 ClusterRoleBindings system:kube-scheduler 将赋予 kube-scheduler 工作所需的权限；

生成证书和私钥：

cd /opt/k8s/work

cfssl gencert -ca=/opt/k8s/work/ca.pem \

  -ca-key=/opt/k8s/work/ca-key.pem \

  -config=/opt/k8s/work/ca-config.json \

  -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler

ls kube-scheduler*pem

将生成的证书和私钥分发到所有 master 节点

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    scp kube-scheduler*.pem root@${node_ip}:/etc/kubernetes/cert/

  done

创建和分发 kubeconfig 文件

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

kubectl config set-cluster kubernetes \

  --certificate-authority=/opt/k8s/work/ca.pem \

  --embed-certs=true \

  --server=${KUBE_APISERVER} \

  --kubeconfig=kube-scheduler.kubeconfig

kubectl config set-credentials system:kube-scheduler \

  --client-certificate=kube-scheduler.pem \

  --client-key=kube-scheduler-key.pem \

  --embed-certs=true \

  --kubeconfig=kube-scheduler.kubeconfig

kubectl config set-context system:kube-scheduler \

  --cluster=kubernetes \

  --user=system:kube-scheduler \

  --kubeconfig=kube-scheduler.kubeconfig

kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig

分发 kubeconfig 到所有 master 节点

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    scp kube-scheduler.kubeconfig root@${node_ip}:/etc/kubernetes/

  done

创建 kube-scheduler 配置文件

cd /opt/k8s/work

cat >kube-scheduler.yaml.template <<EOF

apiVersion: kubescheduler.config.k8s.io/v1alpha1

kind: KubeSchedulerConfiguration

bindTimeoutSeconds: 600

clientConnection:

  burst: 200

  kubeconfig: "/etc/kubernetes/kube-scheduler.kubeconfig"

  qps: 100

enableContentionProfiling: false

enableProfiling: true

hardPodAffinitySymmetricWeight: 1

healthzBindAddress: 127.0.0.1:10251

leaderElection:

  leaderElect: true

metricsBindAddress: ##NODE_IP##:10251

EOF

–kubeconfig：指定 kubeconfig 文件路径，kube-scheduler 使用它连接和验证 kube-apiserver；
–leader-elect=true：集群运行模式，启用选举功能；被选为 leader 的节点负责处理工作，其它节点为阻塞状态；

替换模板文件中的变量：

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for (( i=0; i < 3; i++ ))

  do

    sed -e "s/##NODE_NAME##/${NODE_NAMES[i]}/" -e "s/##NODE_IP##/${NODE_IPS[i]}/" kube-scheduler.yaml.template > kube-scheduler-${NODE_IPS[i]}.yaml

  done

ls kube-scheduler*.yaml

分发 kube-scheduler 配置文件到所有 master 节点：

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    scp kube-scheduler-${node_ip}.yaml root@${node_ip}:/etc/kubernetes/kube-scheduler.yaml

  done

创建kube-scheduler启动文件

创建启动文件模板

cd /opt/k8s/work

cat > kube-scheduler.service.template <<EOF

[Unit]

Description=Kubernetes Scheduler

Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]

WorkingDirectory=${K8S_DIR}/kube-scheduler

ExecStart=/opt/k8s/bin/kube-scheduler \\

  --config=/etc/kubernetes/kube-scheduler.yaml \\

  --bind-address=##NODE_IP## \\

  --secure-port=10259 \\

  --port=0 \\

  --tls-cert-file=/etc/kubernetes/cert/kube-scheduler.pem \\

  --tls-private-key-file=/etc/kubernetes/cert/kube-scheduler-key.pem \\

  --authentication-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \\

  --client-ca-file=/etc/kubernetes/cert/ca.pem \\

  --requestheader-allowed-names="" \\

  --requestheader-client-ca-file=/etc/kubernetes/cert/ca.pem \\

  --requestheader-extra-headers-prefix="X-Remote-Extra-" \\

  --requestheader-group-headers=X-Remote-Group \\

  --requestheader-username-headers=X-Remote-User \\

  --authorization-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \\

  --logtostderr=true \\

  --v=2

Restart=always

RestartSec=5

StartLimitInterval=0

[Install]

WantedBy=multi-user.target

EOF

替换模板文件中的变量：

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for (( i=0; i < 3; i++ ))

  do

    sed -e "s/##NODE_NAME##/${NODE_NAMES[i]}/" -e "s/##NODE_IP##/${NODE_IPS[i]}/" kube-scheduler.service.template > kube-scheduler-${NODE_IPS[i]}.service

  done

ls kube-scheduler*.service

分发启动文件至所有master节点

cd /opt/k8s/work

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    scp kube-scheduler-${node_ip}.service root@${node_ip}:/etc/systemd/system/kube-scheduler.service

  done

启动kube-scheduler

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    ssh root@${node_ip} "mkdir -p ${K8S_DIR}/kube-scheduler"

    ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-scheduler && systemctl restart kube-scheduler"

done

检查服务运行状态

source /opt/k8s/bin/environment.sh

for node_ip in ${MASTER_IPS[@]}

  do

    echo ">>> ${node_ip}"

    ssh root@${node_ip} "systemctl status kube-scheduler|grep Active"

  done

查看输出的 metrics

注意：以下命令在 kube-scheduler 节点上执行。

kube-scheduler 监听 10251 和 10259 端口：
10251：接收 http 请求，非安全端口，不需要认证授权；
10259：接收 https 请求，安全端口，需要认证授权；
两个接口都对外提供 /metrics 和 /healthz 的访问。

[root@node01 work]# curl -s http://10.0.20.11:10251/metrics|head

# HELP apiserver_audit_event_total Counter of audit events generated and sent to the audit backend.

# TYPE apiserver_audit_event_total counter

apiserver_audit_event_total 0

# HELP apiserver_audit_requests_rejected_total Counter of apiserver requests rejected due to an error in audit logging backend.

# TYPE apiserver_audit_requests_rejected_total counter

apiserver_audit_requests_rejected_total 0

# HELP apiserver_client_certificate_expiration_seconds Distribution of the remaining lifetime on the certificate used to authenticate a request.

# TYPE apiserver_client_certificate_expiration_seconds histogram

apiserver_client_certificate_expiration_seconds_bucket{le="0"} 0

apiserver_client_certificate_expiration_seconds_bucket{le="1800"} 0

查看当前leader

[root@node01 work]# kubectl get endpoints kube-scheduler --namespace=kube-system  -o yaml

apiVersion: v1

kind: Endpoints

metadata:

  annotations:

    control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"node01.k8s.com_e83f3d5c-dcbe-48ad-aaa8-e1fcd2279573","leaseDurationSeconds":15,"acquireTime":"2019-12-05T08:41:51Z","renewTime":"2019-12-05T08:43:04Z","leaderTransitions":0}'

  creationTimestamp: "2019-12-05T08:41:51Z"

  name: kube-scheduler

  namespace: kube-system

  resourceVersion: "3144"

  selfLink: /api/v1/namespaces/kube-system/endpoints/kube-scheduler

  uid: 509afbcd-4369-4f44-a44a-e4d3fe84c31f

通过 apiserver 查看 schduler 状态

[root@node01 work]# kubectl get cs

NAME                 STATUS    MESSAGE             ERROR

scheduler            Healthy   ok

controller-manager   Healthy   ok

etcd-2               Healthy   {"health":"true"}

etcd-1               Healthy   {"health":"true"}

etcd-0               Healthy   {"health":"true"}

这里看到 schduler 的状态已经是 ok 了，在测试访问apiserver状态

巴特西

部署高可用 schduler