kubernetes 1.9.4
基于 二进制 文件部署 本地化 kube-apiserver, kube-controller-manager , kube-scheduler
环境说明
这里配置单个Master 2个node, Master 既是 Master 也是 Node, node 只做单纯 Node
kubernetes-2: 10.90.26.2 Master
kubernetes-3: 10.90.26.3 Node
kubernetes-4: 10.90.26.4 Node
初始化环境
hostnamectl --static set-hostname hostname
kubernetes-2: 10.90.26.2
kubernetes-3: 10.90.26.3
kubernetes-4: 10.90.26.4
#编辑 /etc/hosts 文件,配置hostname 通信
vi /etc/hosts
kubernetes-2: 10.90.26.2
kubernetes-3: 10.90.26.3
kubernetes-4: 10.90.26.4
创建 验证
这里使用 CloudFlare 的 PKI 工具集 cfssl 来生成 Certificate Authority (CA) 证书和秘钥文件。
安装 cfssl
mkdir -p /opt/local/cfssl
cd /opt/local/cfssl
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
mv cfssl_linux-amd64 cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
mv cfssljson_linux-amd64 cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
mv cfssl-certinfo_linux-amd64 cfssl-certinfo
chmod +x *
创建 CA 证书配置
mkdir /opt/ssl
cd /opt/ssl
# config.json 文件
vi config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
# csr.json 文件
vi csr.json
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
生成 CA 证书和私钥
cd /opt/ssl/
/opt/local/cfssl/cfssl gencert -initca csr.json | /opt/local/cfssl/cfssljson -bare ca
[root@kubernetes-64 ssl]# ls -lt
总用量 20
-rw-r--r-- 1 root root 1005 7月 3 17:26 ca.csr
-rw------- 1 root root 1675 7月 3 17:26 ca-key.pem
-rw-r--r-- 1 root root 1363 7月 3 17:26 ca.pem
-rw-r--r-- 1 root root 210 7月 3 17:24 csr.json
-rw-r--r-- 1 root root 292 7月 3 17:23 config.json
分发证书
# 创建证书目录
mkdir -p /etc/kubernetes/ssl
# 拷贝所有文件到目录下
cp *.pem /etc/kubernetes/ssl
cp ca.csr /etc/kubernetes/ssl
# 这里要将文件拷贝到所有的k8s 机器上
scp *.pem 10.90.26.3:/etc/kubernetes/ssl/
scp *.csr 10.90.26.3:/etc/kubernetes/ssl/
scp *.pem 10.90.26.4:/etc/kubernetes/ssl/
scp *.csr 10.90.26.4:/etc/kubernetes/ssl/
安装 docker
所有服务器预先安装 docker-ce ,官方1.9 中提示, 目前 k8s 支持最高 Docker versions 1.11.2, 1.12.6, 1.13.1, and 17.03.1
# 导入 yum 源
# 安装 yum-config-manager
yum -y install yum-utils
# 导入
yum-config-manager \
--add-repo \
https://download.docker.com/linux/centos/docker-ce.repo
# 更新 repo
yum makecache
# 查看yum 版本
yum list docker-ce.x86_64 --showduplicates |sort -r
# 安装指定版本 docker-ce 17.03 被 docker-ce-selinux 依赖, 不能直接yum 安装 docker-ce-selinux
wget https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-selinux-17.03.1.ce-1.el7.centos.noarch.rpm
rpm -ivh docker-ce-selinux-17.03.1.ce-1.el7.centos.noarch.rpm
yum -y install docker-ce-17.03.1.ce
# 查看安装
docker version
Client:
Version: 17.03.1-ce
API version: 1.27
Go version: go1.7.5
Git commit: f5ec1e2
Built: Tue Jun 27 02:21:36 2017
OS/Arch: linux/amd64
更改docker 配置
# 添加配置
vi /etc/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.com
After=network.target docker-storage-setup.service
Wants=docker-storage-setup.service
[Service]
Type=notify
Environment=GOTRACEBACK=crash
ExecReload=/bin/kill -s HUP $MAINPID
Delegate=yes
KillMode=process
ExecStart=/usr/bin/dockerd \
$DOCKER_OPTS \
$DOCKER_STORAGE_OPTIONS \
$DOCKER_NETWORK_OPTIONS \
$DOCKER_DNS_OPTIONS \
$INSECURE_REGISTRY
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
TimeoutStartSec=1min
Restart=on-abnormal
[Install]
WantedBy=multi-user.target
# 重新读取配置,启动 docker
systemctl daemon-reload
systemctl start docker
systemctl enable docker
# 如果报错 请使用
journalctl -f -t docker 和 journalctl -u docker 来定位问题
etcd 集群
etcd 是k8s集群最重要的组件, etcd 挂了,集群就挂了
安装 etcd
# 下载 二进制文件
wget https://github.com/coreos/etcd/releases/download/v3.2.14/etcd-v3.2.14-linux-amd64.tar.gz
tar zxvf etcd-v3.2.14-linux-amd64.tar.gz
cd etcd-v3.2.14-linux-amd64
mv etcd etcdctl /usr/bin/
创建 etcd 证书
etcd 证书这里,默认配置三个,后续如果需要增加,更多的 etcd 节点 这里的认证IP 请多预留几个,以备后续添加能通过认证,不需要重新签发
cd /opt/ssl/
vi etcd-csr.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"10.90.26.2",
"10.90.26.3",
"10.90.26.4"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
# 生成 etcd 密钥
/opt/local/cfssl/cfssl gencert -ca=/opt/ssl/ca.pem \
-ca-key=/opt/ssl/ca-key.pem \
-config=/opt/ssl/config.json \
-profile=kubernetes etcd-csr.json | /opt/local/cfssl/cfssljson -bare etcd
# 查看生成
[root@kubernetes-2 ssl]# ls etcd*
etcd.csr etcd-csr.json etcd-key.pem etcd.pem
# 拷贝到etcd服务器
# etcd-1
cp etcd*.pem /etc/kubernetes/ssl/
# etcd-2
scp etcd*.pem 10.90.26.3:/etc/kubernetes/ssl/
# etcd-3
scp etcd*.pem 10.90.26.4:/etc/kubernetes/ssl/
# 如果 etcd 非 root 用户,读取证书会提示没权限
chmod 644 /etc/kubernetes/ssl/etcd-key.pem
修改 etcd 配置
由于 etcd 是最重要的组件,所以 –data-dir 请配置到其他路径中
# 创建 etcd data 目录, 并授权
useradd etcd
mkdir -p /opt/etcd
chown -R etcd:etcd /opt/etcd
# etcd-1
vi /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/opt/etcd/
User=etcd
# set GOMAXPROCS to number of processors
ExecStart=/usr/bin/etcd \
--name=etcd1 \
--cert-file=/etc/kubernetes/ssl/etcd.pem \
--key-file=/etc/kubernetes/ssl/etcd-key.pem \
--peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
--peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
--trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
--peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
--initial-advertise-peer-urls=https://10.90.26.2:2380 \
--listen-peer-urls=https://10.90.26.2:2380 \
--listen-client-urls=https://10.90.26.2:2379,http://127.0.0.1:2379 \
--advertise-client-urls=https://10.90.26.2:2379 \
--initial-cluster-token=k8s-etcd-cluster \
--initial-cluster=etcd1=https://10.90.26.2:2380,etcd2=https://10.90.26.3:2380,etcd3=https://10.90.26.4:2380 \
--initial-cluster-state=new \
--data-dir=/opt/etcd/
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
# etcd-2
vi /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/opt/etcd/
User=etcd
# set GOMAXPROCS to number of processors
ExecStart=/usr/bin/etcd \
--name=etcd2 \
--cert-file=/etc/kubernetes/ssl/etcd.pem \
--key-file=/etc/kubernetes/ssl/etcd-key.pem \
--peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
--peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
--trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
--peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
--initial-advertise-peer-urls=https://10.90.26.3:2380 \
--listen-peer-urls=https://10.90.26.3:2380 \
--listen-client-urls=https://10.90.26.3:2379,http://127.0.0.1:2379 \
--advertise-client-urls=https://10.90.26.3:2379 \
--initial-cluster-token=k8s-etcd-cluster \
--initial-cluster=etcd1=https://10.90.26.2:2380,etcd2=https://10.90.26.3:2380,etcd3=https://10.90.26.4:2380 \
--initial-cluster-state=new \
--data-dir=/opt/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
# etcd-3
vi /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/opt/etcd/
User=etcd
# set GOMAXPROCS to number of processors
ExecStart=/usr/bin/etcd \
--name=etcd3 \
--cert-file=/etc/kubernetes/ssl/etcd.pem \
--key-file=/etc/kubernetes/ssl/etcd-key.pem \
--peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
--peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
--trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
--peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
--initial-advertise-peer-urls=https://10.90.26.4:2380 \
--listen-peer-urls=https://10.90.26.4:2380 \
--listen-client-urls=https://10.90.26.4:2379,http://127.0.0.1:2379 \
--advertise-client-urls=https://10.90.26.4:2379 \
--initial-cluster-token=k8s-etcd-cluster \
--initial-cluster=etcd1=https://10.90.26.2:2380,etcd2=https://10.90.26.3:2380,etcd3=https://10.90.26.4:2380 \
--initial-cluster-state=new \
--data-dir=/opt/etcd/
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
启动 etcd
分别启动 所有节点的 etcd 服务
systemctl daemon-reload
systemctl enable etcd
systemctl start etcd
systemctl status etcd
# 如果报错 请使用
journalctl -f -t etcd 和 journalctl -u etcd 来定位问题
验证 etcd 集群状态
查看 etcd 集群状态:
etcdctl --endpoints=https://10.90.26.2:2379,https://10.90.26.3:2379,https://10.90.26.4:2379\
--cert-file=/etc/kubernetes/ssl/etcd.pem \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--key-file=/etc/kubernetes/ssl/etcd-key.pem \
cluster-health
member 35eefb8e7cc93b53 is healthy: got healthy result from https://10.90.26.2:2379
member 4576ff5ed626a66b is healthy: got healthy result from https://10.90.26.3:2379
member bf3bd651ec832339 is healthy: got healthy result from https://10.90.26.4:2379
cluster is healthy
查看 etcd 集群成员:
etcdctl --endpoints=https://10.90.26.2:2379,https://10.90.26.3:2379,https://10.90.26.4:2379\
--cert-file=/etc/kubernetes/ssl/etcd.pem \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--key-file=/etc/kubernetes/ssl/etcd-key.pem \
member list
35eefb8e7cc93b53: name=etcd3 peerURLs=https://10.90.26.4:2380 clientURLs=https://10.90.26.4:2379 isLeader=false
4576ff5ed626a66b: name=etcd1 peerURLs=https://10.90.26.2:2380 clientURLs=https://10.90.26.2:2379 isLeader=true
bf3bd651ec832339: name=etcd2 peerURLs=https://10.90.26.3:2380 clientURLs=https://10.90.26.3:2379 isLeader=false
配置 Kubernetes 集群
kubectl 安装在所有需要进行操作的机器上
Master and Node
Master 需要部署 kube-apiserver , kube-scheduler , kube-controller-manager 这三个组件。 kube-scheduler 作用是调度pods分配到那个node里,简单来说就是资源调度。 kube-controller-manager 作用是 对 deployment controller , replication controller, endpoints controller, namespace controller, and serviceaccounts controller等等的循环控制,与kube-apiserver交互。
安装组件
# 从github 上下载版本
cd /tmp
wget https://dl.k8s.io/v1.9.4/kubernetes-server-linux-amd64.tar.gz
tar -xzvf kubernetes-server-linux-amd64.tar.gz
cd kubernetes
cp -r server/bin/{kube-apiserver,kube-controller-manager,kube-scheduler,kubectl} /usr/local/bin/
scp server/bin/{kube-apiserver,kube-controller-manager,kube-scheduler,kubectl,kube-proxy,kubelet} 10.90.26.3:/usr/local/bin/
scp server/bin/{kube-proxy,kubelet} 10.90.26.4:/usr/local/bin/
创建 admin 证书
kubectl 与 kube-apiserver 的安全端口通信,需要为安全通信提供 TLS 证书和秘钥。
cd /opt/ssl/
vi admin-csr.json
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
# 生成 admin 证书和私钥
cd /opt/ssl/
/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
-ca-key=/etc/kubernetes/ssl/ca-key.pem \
-config=/opt/ssl/config.json \
-profile=kubernetes admin-csr.json | /opt/local/cfssl/cfssljson -bare admin
# 查看生成
[root@kubernetes-2 ssl]# ls admin*
admin.csr admin-csr.json admin-key.pem admin.pem
cp admin*.pem /etc/kubernetes/ssl/
配置 kubectl kubeconfig 文件
生成证书相关的配置文件存储与 /root/.kube 目录中
# 配置 kubernetes 集群
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://10.90.26.2:6443
# 配置 客户端认证
kubectl config set-credentials admin \
--client-certificate=/etc/kubernetes/ssl/admin.pem \
--embed-certs=true \
--client-key=/etc/kubernetes/ssl/admin-key.pem
kubectl config set-context kubernetes \
--cluster=kubernetes \
--user=admin
kubectl config use-context kubernetes
创建 kubernetes 证书
cd /opt/ssl
vi kubernetes-csr.json
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"10.90.26.2",
"10.90.26.3",
"10.90.26.4",
"10.254.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
## 这里 hosts 字段中 三个 IP 分别为 127.0.0.1 本机, 10.90.26.2 和 10.90.26.3 ,多个Master需要写多个。 10.254.0.1 为 kubernetes SVC 的 IP, 一般是 部署网络的第一个IP , 如: 10.254.0.1 , 在启动完成后,我们使用 kubectl get svc , 就可以查看到
生成 kubernetes 证书和私钥
/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
-ca-key=/etc/kubernetes/ssl/ca-key.pem \
-config=/opt/ssl/config.json \
-profile=kubernetes kubernetes-csr.json | /opt/local/cfssl/cfssljson -bare kubernetes
# 查看生成
[root@kubernetes-2 ssl]# ls -lt kubernetes*
-rw-r--r-- 1 root root 1261 11月 16 15:12 kubernetes.csr
-rw------- 1 root root 1679 11月 16 15:12 kubernetes-key.pem
-rw-r--r-- 1 root root 1635 11月 16 15:12 kubernetes.pem
-rw-r--r-- 1 root root 475 11月 16 15:12 kubernetes-csr.json
# 拷贝到目录
cp kubernetes*.pem /etc/kubernetes/ssl/
配置 kube-apiserver
kubelet 首次启动时向 kube-apiserver 发送 TLS Bootstrapping 请求,kube-apiserver 验证 kubelet 请求中的 token 是否与它配置的 token 一致,如果一致则自动为 kubelet生成证书和秘钥。
# 生成 token
[root@kubernetes-2 ssl]# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
df3b158fbdc425ae2ac70bbef0688921
# 创建 token.csv 文件
cd /opt/ssl
vi token.csv
df3b158fbdc425ae2ac70bbef0688921,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
# 拷贝
cp token.csv /etc/kubernetes/
# 生成高级审核配置文件
cd /etc/kubernetes
cat >> audit-policy.yaml <<EOF
# Log all requests at the Metadata level.
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
- level: Metadata
EOF
创建 kube-apiserver.service 文件
# 自定义 系统 service 文件一般存于 /etc/systemd/system/ 下
# 配置为 各自的本地 IP
vi /etc/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
User=root
ExecStart=/usr/local/bin/kube-apiserver \
--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \
--advertise-address=10.90.26.4 \
--allow-privileged=true \
--apiserver-count=3 \
--audit-policy-file=/etc/kubernetes/audit-policy.yaml \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kubernetes/audit.log \
--authorization-mode=Node,RBAC \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--enable-swagger-ui=true \
--etcd-cafile=/etc/kubernetes/ssl/ca.pem \
--etcd-certfile=/etc/kubernetes/ssl/etcd.pem \
--etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem \
--etcd-servers=https://10.90.26.2:2379,https://10.90.26.3:2379,https://10.90.26.4:2379 \
--event-ttl=1h \
--kubelet-https=true \
--insecure-bind-address=127.0.0.1 \
--insecure-port=8080 \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-cluster-ip-range=10.254.0.0/18 \
--service-node-port-range=30000-32000 \
--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
--enable-bootstrap-token-auth \
--token-auth-file=/etc/kubernetes/token.csv \
--v=1
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
# k8s 1.8 开始需要 添加 --authorization-mode=Node
# k8s 1.8 开始需要 添加 --admission-control=NodeRestriction
# k8s 1.8 开始需要 添加 --audit-policy-file=/etc/kubernetes/audit-policy.yaml
# 这里面要注意的是 --service-node-port-range=30000-32000
# 这个地方是 映射外部端口时 的端口范围,随机映射也在这个范围内映射,指定映射端口必须也在这个范围内。
启动 kube-apiserver
systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver
systemctl status kube-apiserver
配置 kube-controller-manager
–cluster-signing-cert-file 与 –cluster-signing-key-file 标签将被删除。
# 创建 kube-controller-manager.service 文件
vi /etc/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
--address=0.0.0.0 \
--master=http://127.0.0.1:8080 \
--allocate-node-cidrs=true \
--service-cluster-ip-range=10.254.0.0/18 \
--cluster-cidr=10.254.64.0/18 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--leader-elect=true \
--v=1
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
启动 kube-controller-manager
systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl start kube-controller-manager
systemctl status kube-controller-manager
配置 kube-scheduler
# 创建 kube-cheduler.service 文件
vi /etc/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/usr/local/bin/kube-scheduler \
--address=0.0.0.0 \
--master=http://127.0.0.1:8080 \
--leader-elect=true \
--v=1
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
启动 kube-scheduler
systemctl daemon-reload
systemctl enable kube-scheduler
systemctl start kube-scheduler
systemctl status kube-scheduler
验证 Master 节点
[root@kubernetes-2 ~]# kubectl get componentstatuses
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-2 Healthy {"health": "true"}
etcd-0 Healthy {"health": "true"}
etcd-1 Healthy {"health": "true"}
[root@kubernetes-2 ~]# kubectl get componentstatuses
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-2 Healthy {"health": "true"}
etcd-0 Healthy {"health": "true"}
etcd-1 Healthy {"health": "true"}
配置 kubelet
kubelet 启动时向 kube-apiserver 发送 TLS bootstrapping 请求,需要先将 bootstrap token 文件中的 kubelet-bootstrap 用户赋予 system:node-bootstrapper 角色,然后 kubelet 才有权限创建认证请求(certificatesigningrequests)。
# 先创建认证请求
# user 为 master 中 token.csv 文件里配置的用户
# 只需创建一次就可以
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
创建 kubelet kubeconfig 文件
# 配置集群
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://10.90.26.4:6443 \
--kubeconfig=bootstrap.kubeconfig
# 配置客户端认证
kubectl config set-credentials kubelet-bootstrap \
--token=df3b158fbdc425ae2ac70bbef0688921 \
--kubeconfig=bootstrap.kubeconfig
# 配置关联
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig
# 配置默认关联
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
# 拷贝生成的 bootstrap.kubeconfig 文件
mv bootstrap.kubeconfig /etc/kubernetes/
scp bootstrapping 10.90.26.3:/etc/kubernetes/
scp bootstrapping 10.90.26.4:/etc/kubernetes/
创建 kubelet.service 文件
# 创建 kubelet 目录
> 配置为 node 本机 IP
mkdir /var/lib/kubelet
vi /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet \
--cgroup-driver=cgroupfs \
--hostname-override=kubernetes-2 \
--pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.0 \
--experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
--cert-dir=/etc/kubernetes/ssl \
--cluster_dns=10.254.0.2 \
--cluster_domain=cluster.local. \
--hairpin-mode promiscuous-bridge \
--allow-privileged=true \
--fail-swap-on=false \
--serialize-image-pulls=false \
--logtostderr=true \
--max-pods=512 \
--v=1
[Install]
WantedBy=multi-user.target
# 如上配置:
kubernetes-2 本机hostname
10.254.0.2 预分配的 dns 地址
cluster.local. 为 kubernetes 集群的 domain
mirrorgooglecontainers/pause-amd64:3.0 这个是 pod 的基础镜像,既 gcr 的 gcr.io/google_containers/pause-amd64:3.0 镜像, 下载下来修改为自己的仓库中的比较快。
启动 kubelet
systemctl daemon-reload
systemctl enable kubelet
systemctl start kubelet
systemctl status kubelet
# 如果报错 请使用
journalctl -f -t kubelet 和 journalctl -u kubelet 来定位问题
配置 TLS 认证
# 查看 csr 的名称
[root@kubernetes-2 ~]# kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-Pu4QYp3NAwlC6o8AG8iwdCl52CiqhjiSyrso3335JTs 1m kubelet-bootstrap Pending
node-csr-poycCHd7B8YPxc12EBgI3Rwe0wnDJah5uIGvQHzghVY 2m kubelet-bootstrap Pending
# 增加 认证
kubectl get csr | grep Pending | awk '{print $1}' | xargs kubectl certificate approve
验证 nodes
[root@kubernetes-2 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
kubernetes-2 Ready <none> 12s v1.9.4
kubernetes Ready <none> 17s v1.9.4
# 成功以后会自动生成配置文件与密钥
# 配置文件
ls /etc/kubernetes/kubelet.kubeconfig
/etc/kubernetes/kubelet.kubeconfig
# 密钥文件 这里注意如果 csr 被删除了,请删除如下文件,并重启 kubelet 服务
ls /etc/kubernetes/ssl/kubelet*
/etc/kubernetes/ssl/kubelet-client.crt /etc/kubernetes/ssl/kubelet.crt
/etc/kubernetes/ssl/kubelet-client.key /etc/kubernetes/ssl/kubelet.key
配置 kube-proxy
创建 kube-proxy 证书
# 证书方面由于我们node端没有装 cfssl
# 我们回到 master 端 机器 去配置证书,然后拷贝过来
[root@kubernetes-2 ~]# cd /opt/ssl
vi kube-proxy-csr.json
{
"CN": "system:kube-proxy",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
生成 kube-proxy 证书和私钥
/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
-ca-key=/etc/kubernetes/ssl/ca-key.pem \
-config=/opt/ssl/config.json \
-profile=kubernetes kube-proxy-csr.json | /opt/local/cfssl/cfssljson -bare kube-proxy
# 查看生成
ls kube-proxy*
kube-proxy.csr kube-proxy-csr.json kube-proxy-key.pem kube-proxy.pem
# 拷贝到目录
cp kube-proxy* /etc/kubernetes/ssl/
scp kube-proxy* 10.90.26.3:/etc/kubernetes/ssl/
scp kube-proxy* 10.90.26.4:/etc/kubernetes/ssl/
创建 kube-proxy kubeconfig 文件
# 配置集群
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://10.90.26.2:6443 \
--kubeconfig=kube-proxy.kubeconfig
# 配置客户端认证
kubectl config set-credentials kube-proxy \
--client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
# 配置关联
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
# 配置默认关联
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
# 拷贝到需要的 node 端里
scp kube-proxy.kubeconfig 10.90.26.3:/etc/kubernetes/
scp kube-proxy.kubeconfig 10.90.26.4:/etc/kubernetes/
创建 kube-proxy.service 文件
# 创建 kube-proxy 目录
mkdir -p /var/lib/kube-proxy
vi /etc/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \
--bind-address=10.90.26.2 \
--hostname-override=kubernetes-2 \
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \
--logtostderr=true \
--v=1
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
启动 kube-proxy
systemctl daemon-reload
systemctl enable kube-proxy
systemctl start kube-proxy
systemctl status kube-proxy
# 如果报错 请使用
journalctl -f -t kube-proxy 和 journalctl -u kube-proxy 来定位问题
至此 Master 端 与 Master and Node 端的安装完毕,正式使用还需要继续下一步,安装网络插件,才能部署应用。
node 节点配置参考master节点,只需配置kubelet,kube-proxy服务。
后续操作
安装cni插件: 到CNI 插件最新release页面下载cni-v0.6.0.tgz,解压后里面有很多插件,选择如下几个复制到项目 bin目录下
flannel用到的插件 bridge flannel host-local loopback portmap
修改kubelet配置,添加cni条目:
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/root/local/bin/kubelet \
--address=192.168.112.156 \
--cgroup-driver=systemd \
--hostname-override=192.168.112.156 \
--pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.0 \
--experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
--cert-dir=/etc/kubernetes/ssl \
--cluster_dns=10.254.0.2 \
--network-plugin=cni \
--cni-conf-dir=/etc/cni/net.d \
--cni-bin-dir=/usr/local/bin \
--cluster_domain=cluster.local. \
--hairpin-mode promiscuous-bridge \
--allow-privileged=true \
--fail-swap-on=false \
--serialize-image-pulls=false \
--logtostderr=true \
--max-pods=512 \
--v=3
[Install]
WantedBy=multi-user.target
重启kubelet,此时查看node状态,可能会显示notready,因为没有cni配置文件
mkdir -p /etc/cni/net.d/
vim mkdir -p /etc/cni/net.d/cni-default.conf
{
"name": "mynet",
"type": "bridge",
"bridge": "mynet0",
"isDefaultGateway": true,
"ipMasq": true,
"hairpinMode": true,
"ipam": {
"type": "host-local",
"subnet": "10.254.0.0/18"
}
}
然后重启kubelet,查看kubelet状态,变为ready。
安装网络插件
- 安装网络插件(flannel,calico)
damenset方式运行
https://github.com/coreos/flannel/tree/master/Documentation/k8s-manifests
#修改net-conf.json配置。
#需要给docker配置代理才能拉取镜像。
安装kube-dns插件
官方yaml文件
安装dashboard插件
官方yaml文件
高可用方案
- keepalived+haproxy 该方式需要使用vip实现,实际工作的apiserver只有一个,只需在master节点安装keepalived和haproxy,并配置代理即可。所有node配置连接api的地址为:虚拟ip:6443 使用这种方式需要将上面kubelet,kube-proxy的kubeconfig中apiserver改成https://vip:6443
- nginx反向代理 使用nginx的反向代理方式实现,master节点不需要特除配置,直接安装n个master节点即可,在每一个node节点安装一个nginx,版本需要1.9以上,并启用stream模块,支持tcp的反向代理,所有node连接api:127.0.0.1:6443。 使用这种方式需要将上面kubelet,kube-proxy的kubeconfig中apiserver改成https://127.0.0.1:6443
本文链接:https://zhusl.com/post/kubernetes-1-9-install-templ.html,参与评论 »
--EOF--
发表于 2018-04-01 09:31:00,并被添加「kubernetes」标签。
本站使用「署名 4.0 国际」创作共享协议,转载请注明作者及原网址。更多说明 »
提醒:本文最后更新于 2448 天前,文中所描述的信息可能已发生改变,请谨慎使用。
专题「KUBERNETES」的其它文章 »
- ubuntu16.04-kubernetes+arena搭建机器学习环境 (Sep 01, 2018)
- Kubernetes 使用Ceph作为storageClass (Apr 04, 2018)
- 使用kubeadm安装Kubernetes 1.9 (Feb 07, 2018)
Comments