CentOS 7 二进制部署kubernetes-Master
Kubernetes 微服务
|
|
587
|
0

环境准备

1、关闭防火墙和selinux
2、关闭swap分区，swap分区会影响到容器

[root@zhupengfei-k8s-master-shanghai-area1 ~]# swapoff -a # 关闭swap分区
[root@zhupengfei-k8s-master-shanghai-area1 ~]# cat /etc/fstab
#
# /etc/fstab
# Created by anaconda on Wed Aug 15 22:19:43 2018
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
/dev/mapper/centos-root /                       xfs     defaults        0 0
UUID=658b0e1c-f7ab-4e9b-8678-347e1f73e6e9 /boot                   xfs     defaults        0 0
# /dev/mapper/centos-swap swap                    swap    defaults        0 0 # swap分区会影响到容器，注释swap相关行
[root@zhupengfei-k8s-master-shanghai-area1 ~]# vim /etc/sysctl.conf # 配置关闭swap永久生效
vm.swappiness=0 # 修改 vm.swappiness 的修改为 0
[root@zhupengfei-k8s-master-shanghai-area1 ~]# sysctl -p # 使配置生效

确认swap已关闭

[root@zhupengfei-k8s-master-shanghai-area1 ~]# free -mh
              total        used        free      shared  buff/cache   available
Mem:           7.6G        310M        7.2G        8.5M        167M        7.1G
Swap:            0B          0B          0B  # swap行都显示 0 则表示关闭成功

3、设置hosts，配置域名解析IP的方式便于后续管理和变更

[root@zhupengfei-k8s-master-shanghai-area1 ~]# cat >> /etc/hosts << EOF
192.168.13.39 node1.k8s.ponfey.com
192.168.13.44 node2.k8s.ponfey.com
192.168.13.54 node3.k8s.ponfey.com
EOF

4、设置桥接的IPv4流量传递到iptables的链

[root@zhupengfei-k8s-master-shanghai-area1 ~]# cat > /etc/sysctl.d/k8s.conf << EOF
> net.bridge.bridge-nf-call-ip6tables = 1
> net.bridge.bridge-nf-call-iptables = 1
> EOF 
[root@zhupengfei-k8s-master-shanghai-area1 ~]# sysctl --system

5、时间同步，私有云云主机管理方式，所有云主机实例已经默认和自建NTP时钟服务器（上游服务器:ntp1.aliyun.com）保持时间同步

下载Master二进制文件

[root@zhupengfei-k8s-master-shanghai-area1 packages]# ls
kubernetes-server-linux-amd64.tar.gz # [官网下载v1.20 Release，推荐用Motrix下载](https://dl.k8s.io/v1.20.0/kubernetes-server-linux-amd64.tar.gz "官网下载v1.20 Release")
[root@zhupengfei-k8s-master-shanghai-area1 packages]# tar -zxvf kubernetes-server-linux-amd64.tar.gz
[root@zhupengfei-k8s-master-shanghai-area1 bin]# pwd
/packages/kubernetes/server/bin
[root@zhupengfei-k8s-master-shanghai-area1 bin]# ls  # 主要用到 kube-apiserver（所有服务的访问入口），kube-controller-manager（属于Pod的控制器，目的是维持副本的期望数量，属于冗余方案），kube-scheduler（负责任务的节点选择和分配，关于多节点必要都需要调度器），kubectl 这4个master可执行文件服务
apiextensions-apiserver  kube-aggregator  kube-apiserver.docker_tag  kube-controller-manager             kube-controller-manager.tar  kubelet     kube-proxy.docker_tag  kube-scheduler             kube-scheduler.tar
kubeadm                  kube-apiserver   kube-apiserver.tar         kube-controller-manager.docker_tag  kubectl                      kube-proxy  kube-proxy.tar         kube-scheduler.docker_tag  mounter
[root@zhupengfei-k8s-master-shanghai-area1 bin]# cp kube-apiserver kube-controller-manager kube-scheduler kubectl /usr/local/bin
[root@zhupengfei-k8s-master-shanghai-area1 bin]# cd /usr/local/bin/
[root@zhupengfei-k8s-master-shanghai-area1 bin]# chmod 755 kube-apiserver kube-controller-manager kube-scheduler kubectl # 复制到/usr/local/bin，并授权chmod 755

部署etcd集群

[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# cat >> /etc/hosts << EOF
192.168.13.41 etcd1
192.168.13.96 etcd2
192.168.13.32 etcd3
EOF # etcd集群配置hosts

[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# yum -y install etcd
[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# mkdir /etc/etcd/cert -v  # 保证内网etcd访问的安全性，使用cfssl来生成自签证书
mkdir: created directory ‘/etc/etcd/cert’
[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  9.8M  100  9.8M    0     0  1493k      0  0:00:06  0:00:06 --:--:-- 3097k
[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 2224k  100 2224k    0     0   476k      0  0:00:04  0:00:04 --:--:--  703k
[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 6440k  100 6440k    0     0  2489k      0  0:00:02  0:00:02 --:--:-- 2489k
[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# cat > /etc/etcd/cert/ca-config.json  << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
  "expiry": "87600h"
      }
    }
  }
}
EOF

[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# cat > /etc/etcd/cert/ca-csr.json  << EOF
{
    "CN": "etcd CA",  # ST=省/L=市/O=组织名/OU=组织单位/C=国家
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Shanghai",
            "ST": "Shanghai",
      "O": "k8s",
            "OU": "System"
        }
    ],
    "ca": {
  "expiry": "87600h"
    }
}
EOF

[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# cat > /etc/etcd/cert/server-csr.json  << EOF
{
    "CN": "etcd",
    "hosts": [
    "127.0.0.1",
    "192.168.13.41",
    "192.168.13.96",
    "192.168.13.32",
    "192.168.13.43",
    "192.168.13.39",
    "192.168.13.44",
    "192.168.13.54",
    "apiserver1",
    "apiserver2",
    "apiserver3",
    "etcd1",
    "etcd2",
    "etcd3",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Shanghai",
            "ST": "Shanghai",
      "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF # 把etcd集群的所有IP、kubernetes master的IP以及kubernetes服务的IP都加入，使得它们能使用同一个密钥

[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# cd /etc/etcd/cert # 生成证书
[root@zhupengfei-k8s-etcd1-shanghai-area1 cert]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2021/01/23 15:28:47 [INFO] generating a new CA key and certificate from CSR
2021/01/23 15:28:47 [INFO] generate received request
2021/01/23 15:28:47 [INFO] received CSR
2021/01/23 15:28:47 [INFO] generating key: rsa-2048
2021/01/23 15:28:47 [INFO] encoded CSR
2021/01/23 15:28:47 [INFO] signed certificate with serial number 527652906531705292071282658243359206410281027841
[root@zhupengfei-k8s-etcd1-shanghai-area1 cert]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
2021/01/23 15:29:00 [INFO] generate received request
2021/01/23 15:29:00 [INFO] received CSR
2021/01/23 15:29:00 [INFO] generating key: rsa-2048
2021/01/23 15:29:00 [INFO] encoded CSR
2021/01/23 15:29:00 [INFO] signed certificate with serial number 617158598451077687966145923993653942612522726250
2021/01/23 15:29:00 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

[root@zhupengfei-k8s-etcd1-shanghai-area1 cert]# scp -r /etc/etcd/cert 192.168.13.96:/etc/etcd
[root@zhupengfei-k8s-etcd1-shanghai-area1 cert]# scp -r /etc/etcd/cert 192.168.13.32:/etc/etcd # 将生成的证书复制到其他etcd集群节点

[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# cat > /usr/lib/systemd/system/etcd.service << EOF
> [Unit]
> Description=Etcd Server
> After=network.target
> After=network-online.target
> Wants=network-online.target
> 
> [Service]
> Type=notify
> WorkingDirectory=/var/lib/etcd/
> EnvironmentFile=-/etc/etcd/etcd.conf
> User=etcd
> # set GOMAXPROCS to number of processors
> ExecStart=/bin/bash -c "GOMAXPROCS=\$(nproc) \
> /usr/bin/etcd --name=\"\${ETCD_NAME}\" \
> --data-dir=\"\${ETCD_DATA_DIR}\" \
> --listen-peer-urls=\"\${ETCD_LISTEN_PEER_URLS}\" \
> --listen-client-urls=\"\${ETCD_LISTEN_CLIENT_URLS}\" \
> --advertise-client-urls=\"\${ETCD_ADVERTISE_CLIENT_URLS}\" \
> --initial-cluster-token=\"\${ETCD_INITIAL_CLUSTER_TOKEN}\" \
> --initial-cluster=\"\${ETCD_INITIAL_CLUSTER}\" \
> --initial-cluster-state=\"\${ETCD_INITIAL_CLUSTER_STATE}\" \
> --cert-file=/etc/etcd/cert/server.pem \
> --key-file=/etc/etcd/cert/server-key.pem \
> --peer-cert-file=/etc/etcd/cert/server.pem \
> --peer-key-file=/etc/etcd/cert/server-key.pem \
> --trusted-ca-file=/etc/etcd/cert/ca.pem \
> --peer-trusted-ca-file=/etc/etcd/cert/ca.pem"
> 
> Restart=on-failure
> LimitNOFILE=65536
> 
> [Install]
> WantedBy=multi-user.target
> EOF # etcd 集群全部执行

配置etcd配置文件/etc/etcd/etcd.conf

[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# mv -v /etc/etcd/etcd.conf{,.bak}
‘/etc/etcd/etcd.conf’ -> ‘/etc/etcd/etcd.conf.bak’
[root@zhupengfei-k8s-etcd1-shanghai-area1 ~]# cat > /etc/etcd/etcd.conf << EOF
> ETCD_NAME=etcd1 # etcd集群中的节点名
> ETCD_DATA_DIR="/var/lib/etcd/etcd1" # 数据目录
> ETCD_LISTEN_PEER_URLS="https://192.168.13.20:2380" # 监听用于节点之间通信的url，可监听多个，集群内部将通过这些url进行数据交互
> ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.13.20:2379" # 监听的用于客户端通信的url， etcdctl客户端命令使用调用了本机服务，需要添加 127.0.0.1 
> ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.13.20:2380" # 用于节点之间通信的url，节点间将以该值进行通信
> ETCD_INITIAL_CLUSTER="etcd1=https://192.168.13.20:2380,etcd2=https://192.168.13.19:2380,etcd3=https://192.168.13.59:2380" # 集群中所有的initial-advertise-peer-urls的合集
> ETCD_INITIAL_CLUSTER_STATE="new" # 标识新建集群
> ETCD_INITIAL_CLUSTER_TOKEN="LCjJgRjfN2fIARYb" # 节点的 token 值，可使用【 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 】生成一个随机token
> ETCD_ADVERTISE_CLIENT_URLS="https://192.168.13.20:2379"
> EOF

[root@zhupengfei-k8s-etcd2-shanghai-area1 ~]# mv -v /etc/etcd/etcd.conf{,.bak}
‘/etc/etcd/etcd.conf’ -> ‘/etc/etcd/etcd.conf.bak’
[root@zhupengfei-k8s-etcd2-shanghai-area1 ~]# cat > /etc/etcd/etcd.conf << EOF
ETCD_NAME=etcd2
ETCD_DATA_DIR="/var/lib/etcd/etcd3"
ETCD_LISTEN_PEER_URLS="https://192.168.13.96:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.13.96:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.13.96:2380"
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.13.41:2380,etcd2=https://192.168.13.96:2380,etcd3=https://192.168.13.32:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="LCjJgRjfN2fIARYb"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.13.96:2379"
EOF

[root@zhupengfei-k8s-etcd3-shanghai-area1 ~]# mv -v /etc/etcd/etcd.conf{,.bak}
‘/etc/etcd/etcd.conf’ -> ‘/etc/etcd/etcd.conf.bak’
[root@zhupengfei-k8s-etcd3-shanghai-area1 ~]# cat > /etc/etcd/etcd.conf << EOF
ETCD_NAME=etcd3
ETCD_DATA_DIR="/var/lib/etcd/etcd3"
ETCD_LISTEN_PEER_URLS="https://192.168.13.32:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.13.32:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.13.32:2380"
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.13.41:2380,etcd2=https://192.168.13.96:2380,etcd3=https://192.168.13.32:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="LCjJgRjfN2fIARYb"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.13.32:2379"
EOF

启动etcd服务
$ chown etcd.etcd  -R /etc/etcd
$ systemctl daemon-reload
$ systemctl restart etcd && systemctl enable etcd

查看etcd集群的成员信息

[root@zhupengfei-k8s-etcd1-shanghai-area1 cert]# etcdctl --ca-file=/etc/etcd/cert/ca.pem --cert-file=/etc/etcd/cert/server.pem --key-file=/etc/etcd/cert/server-key.pem --endpoints="https://etcd1:2379,https://etcd2:2379,https://etcd3:2379" member list # 不带证书会报错
682c047ec309638a: name=etcd3 peerURLs=https://192.168.13.32:2380 clientURLs=https://192.168.13.32:2379 isLeader=false
7a1f36ef08d067ff: name=etcd2 peerURLs=https://192.168.13.96:2380 clientURLs=https://192.168.13.96:2379 isLeader=false
db1e5aac9dac5ac3: name=etcd1 peerURLs=https://192.168.13.41:2380 clientURLs=https://192.168.13.41:2379 isLeader=true

从列出信息可以看出，目前etcd1是主节点。

kube-apiserver配置

时间关系，后续待更新 – 2021-01-23 18:48:43 星期六

上一篇
下一篇

推荐文章

Kubernetes 容灾及高可用保障
Kubernetes 知识在于积累
All-in-One 部署 KubeSphere
部署高可用k8s集群1.20.x
理解Kubernetes的网络:Services
Kubernetes 资源清单pod生命周期
Kubernetes 资源清单 YAML格式
Kubernetes 资源清单类型
Kubernetes 集群部署
Kubernetes 基础概念