kubernetes ubuntu安装部署
安装基础环境
配置系统hosts文件
cat >> /etc/hosts <<EOF
10.10.34.92 ukm01
10.10.34.93 ukm02
10.10.34.94 ukm03
10.10.34.95 uki01
10.10.34.96 uki02
10.10.34.97 uki03
10.10.34.98 ukn01
10.10.34.99 ukn02
10.10.34.100 ukn03
EOF配置主机名
hostnamectl set-hostname ukm01配置sshkey
创建sshkey
ssh-keygen -t dsa -b 1024 -C "seanzhau@mokr.cn"
Generating public/private dsa key pair.
Enter file in which to save the key (/root/.ssh/id_dsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_dsa.
Your public key has been saved in /root/.ssh/id_dsa.pub.
The key fingerprint is:
SHA256:LaO+CdmwvXlHaIPXvZf0s/XjKxPjVUdtvoHC0PCGEUs seanzhau@mokr.cn
The key's randomart image is:
+---[DSA 1024]----+
| E+ .|
| ..=. +|
| ooo .+ |
| ..o . .+|
| . .So... =|
| *..=oo .o.o |
| + +o o .o+o.|
| o +. . .+oo+|
| *o . .++*|
+----[SHA256]-----+分发key
ssh-copy-id ukm01
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_dsa.pub"
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys
Number of key(s) added: 1
...
ssh-copy-id ukn03
...校验
ssh ukm01
Welcome to Ubuntu 18.04.3 LTS (GNU/Linux 4.15.0-88-generic x86_64)
* Documentation: https://help.ubuntu.com
* Management: https://landscape.canonical.com
* Support: https://ubuntu.com/advantage
System information disabled due to load higher than 4.0
...
ssh ukn03
...配置阿里云 repo
mv /etc/apt/sources.list /etc/apt/sources.list.bak
cat > /etc/apt/sources.list <<EOF
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
EOF安装基础包
ubuntu 18.04 默认未安装python，需要每台服务器手动安装。
apt update && apt install -y software-properties-common apt-transport-https ca-certificates gnupg2 python curl安装 ansible
只需在 master 节点安装
apt-add-repository --yes --update ppa:ansible/ansible
apt install -y ansible配置 ansible.cfg
cat > /etc/ansible/ansible.cfg <<EOF
[defaults]
log_path = /var/log/ansible.log
forks = 20
host_key_checking = False
retry_files_enabled = False
deprecation_warnings = False
nocows = True
remote_user = root
roles_path = roles/
gathering = smart
fact_caching = jsonfile
fact_caching_connection = /etc/ansible/facts
fact_caching_timeout = 600
callback_whitelist = profile_tasks
inventory_ignore_extensions = secrets.py, .pyc, .cfg, .crt, .ini
timeout = 30
[inventory]
unparsed_is_failed=true
[ssh_connection]
pipelining = True
ssh_args = -o ControlMaster=auto -o ControlPersist=600s
timeout = 10
control_path = %(directory)s/%%h-%%r
EOF配置 inventory hosts
cat > /etc/ansible/hosts <<EOF
[ins]
10.10.34.92 hostname='ukm01'
10.10.34.93 hostname='ukm02'
10.10.34.94 hostname='ukm03'
10.10.34.95 hostname='uki01'
10.10.34.96 hostname='uki02'
10.10.34.97 hostname='uki03'
10.10.34.98 hostname='ukn01'
10.10.34.99 hostname='ukn02'
10.10.34.100 hostname='ukn03' ansible_ssh_user='root' ansible_ssh_pass='xxxxxxx'
[ans]
10.10.34.92
[master]
10.10.34.9[2:4]
[infra]
10.10.34.9[5:7]
[worker]
10.10.34.9[8:9]
10.10.34.100
EOF验证 ansible 配置
ansible ins -m ping
10.10.34.92 | SUCCESS => {
"ansible_facts": {
"discovered_interpreter_python": "/usr/bin/python"
},
"changed": false,
"ping": "pong"
}
...集群节点绑定主机名
ansible ins:\!ans -m shell -a 'cat >> /etc/hosts <<EOF
10.10.34.92 ukm01
10.10.34.93 ukm02
10.10.34.94 ukm03
10.10.34.95 uki01
10.10.34.96 uki02
10.10.34.97 uki03
10.10.34.98 ukn01
10.10.34.99 ukn02
10.10.34.100 ukn03
EOF'
ansible ins:\!ans -m shell -a 'cat /etc/hosts'分发配置
分发私钥
ansible master -m copy -a 'src=/root/.ssh/id_rsa dest=/root/.ssh/id_rsa mode=0600 owner=root'
分发 ansible 配置
ansible master -m copy -a 'src=/etc/ansible/hosts dest=/etc/ansible/hosts'
ansible master -m copy -a 'src=/etc/ansible/ansible.cfg dest=/etc/ansible/ansible.cfg'安装 Docker CE
添加 Docker GPG key
ansible ins -m shell -a 'curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -'配置 Docker 源
- 动态识别操作系统版本
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
ansible ins -m shell -a 'add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu bionic stable"'安装 Docker
ansible ins -m shell -a 'apt update && apt install -y docker-ce docker-ce-cli containerd.io'配置 docker daemon
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m",
"max-file": "5"
},
"registry-mirrors": ["https://docker.mirrors.ustc.edu.cn"],
"storage-driver": "overlay2"
}
EOF分发 daemon.json
ansible ins -m copy -a 'src=/etc/docker/daemon.json dest=/etc/docker/daemon.json'
ansible ins -m shell -a 'mkdir -p /etc/systemd/system/docker.service.d'重启 docker 服务
ansible ins -m shell -a 'systemctl daemon-reload'
ansible ins -m shell -a 'systemctl restart docker'
ansible ins -m shell -a 'docker info|grep -i cgroup'
Cgroup Driver: systemddaemon.json 配置详解
cat /etc/docker/daemon.json
{
"authorization-plugins": [],
"data-root": "",
"dns": [],
"dns-opts": [],
"dns-search": [],
"exec-opts": [],
"exec-root": "",
"experimental": false,
"features": {},
"storage-driver": "",
"storage-opts": [],
"labels": [],
"live-restore": true,
"log-driver": "json-file",
"log-opts": {
"max-size": "10m",
"max-file":"5",
"labels": "somelabel",
"env": "os,customer"
},
"mtu": 0,
"pidfile": "",
"cluster-store": "",
"cluster-store-opts": {},
"cluster-advertise": "",
"max-concurrent-downloads": 3,
"max-concurrent-uploads": 5,
"default-shm-size": "64M",
"shutdown-timeout": 15,
"debug": true,
"hosts": [],
"log-level": "",
"tls": true,
"tlsverify": true,
"tlscacert": "",
"tlscert": "",
"tlskey": "",
"swarm-default-advertise-addr": "",
"api-cors-header": "",
"selinux-enabled": false,
"userns-remap": "",
"group": "",
"cgroup-parent": "",
"default-ulimits": {
"nofile": {
"Name": "nofile",
"Hard": 64000,
"Soft": 64000
}
},
"init": false,
"init-path": "/usr/libexec/docker-init",
"ipv6": false,
"iptables": false,
"ip-forward": false,
"ip-masq": false,
"userland-proxy": false,
"userland-proxy-path": "/usr/libexec/docker-proxy",
"ip": "0.0.0.0",
"bridge": "",
"bip": "",
"fixed-cidr": "",
"fixed-cidr-v6": "",
"default-gateway": "",
"default-gateway-v6": "",
"icc": false,
"raw-logs": false,
"allow-nondistributable-artifacts": [],
"registry-mirrors": [],
"seccomp-profile": "",
"insecure-registries": [],
"no-new-privileges": false,
"default-runtime": "runc",
"oom-score-adjust": -500,
"node-generic-resources": ["NVIDIA-GPU=UUID1", "NVIDIA-GPU=UUID2"],
"runtimes": {
"cc-runtime": {
"path": "/usr/bin/cc-runtime"
},
"custom": {
"path": "/usr/local/bin/my-runc-replacement",
"runtimeArgs": [
"--debug"
]
}
},
"default-address-pools":[
{"base":"172.80.0.0/16","size":24},
{"base":"172.90.0.0/16","size":24}
]
}格式化查看 images
- .ID Image ID
- .Repository Image 仓库地址和名称
- .Tag Image 标签
- .Digest Image hash 值信息
- .CreatedSince Image 创建到现在的时间
- .CreatedAt Image 创建的具体时间
- .Size Image 大小
docker images --format "table {{ .Repository }} \t {{ .Tag }} \t {{ .ID }} \t {{ .CreatedSince }} \t {{ .CreatedAt }} \t {{ .Size }}" --digestsdocker pull 过程
- Docker 客户端发送 REPOSITORY 和 TAG（或者 DIGEST）给 Registry；
- Registry 根据 REPOSITORY 和 TAG（或者 DIGEST）找到相应 image manifest；
- Registry 将 manifest 返回给 Docker 客户端；
- Docker 客户端根据 manifest 读取 image 的 digest(sha256) ， sha256 就是 IMAGE ID；
- Docker 客户端根据 ID 查找本地是否存在对应的 image；
- 如果 image 存在则创建对应的 image REPOSITORY 和 TAG；
- 如果 image 不存在，则继续请求 Registry 获取 image 的配置文件；
- 根据 image 配置文件中的 diff_ids 在本地找对应的 layer 是否存在；
- 如果 layer 存在，则返回 Already exists；
- 如果 layer 不存在，则根据 manifest 中 layer 的 sha256 和 media type 去服务器下载对应的 layer；
- 下载解压之后，校验 tar 包的 sha256 是否和 Image Config 中的 diff_id 一致。
image本地存放信息
查看 image DIGEST 信息
docker images --digests
REPOSITORY TAG DIGEST IMAGE ID CREATED SIZE
busybox latest sha256:a8cf7ff6367c2afa2a90acd081b484cbded349a7076e7bdf37a05279f276bc12 be5888e67be6 2 weeks ago 1.22MB
如果是其他的文件系统的话，名字会是其他的，比如btrfs、overlay2、devicemapper等。
cat /var/lib/docker/image/overlay2/repositories.json | python -m json.tool
{
"Repositories": {
"busybox": {
"busybox:latest": "sha256:be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a",
"busybox@sha256:a8cf7ff6367c2afa2a90acd081b484cbded349a7076e7bdf37a05279f276bc12": "sha256:be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a"
}
}
}
校验 image config 的 sha256 值
sha256sum /var/lib/docker/image/overlay2/imagedb/content/sha256/be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a
be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a /var/lib/docker/image/overlay2/imagedb/content/sha256/be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a
查看 image config
cat /var/lib/docker/image/overlay2/imagedb/content/sha256/be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a | python -m json.tool
{
"architecture": "amd64",
"config": {
"ArgsEscaped": true,
"AttachStderr": false,
"AttachStdin": false,
"AttachStdout": false,
"Cmd": [
"sh"
],
"Domainname": "",
"Entrypoint": null,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Hostname": "",
"Image": "sha256:b0acc7ebf5092fcdd0fe097448529147e6619bd051f03ccf25b29bcae87e783f",
"Labels": null,
"OnBuild": null,
"OpenStdin": false,
"StdinOnce": false,
"Tty": false,
"User": "",
"Volumes": null,
"WorkingDir": ""
},
"container": "f7e67f16a539f8bbf53aae18cdb5f8c53e6a56930e7660010d9396ae77f7acfa",
"container_config": {
"ArgsEscaped": true,
"AttachStderr": false,
"AttachStdin": false,
"AttachStdout": false,
"Cmd": [
"/bin/sh",
"-c",
"(nop) ",
"CMD [\"sh\"]"
],
"Domainname": "",
"Entrypoint": null,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Hostname": "f7e67f16a539",
"Image": "sha256:b0acc7ebf5092fcdd0fe097448529147e6619bd051f03ccf25b29bcae87e783f",
"Labels": {},
"OnBuild": null,
"OpenStdin": false,
"StdinOnce": false,
"Tty": false,
"User": "",
"Volumes": null,
"WorkingDir": ""
},
"created": "2020-04-14T19:19:53.590635493Z",
"docker_version": "18.09.7",
"history": [
{
"created": "2020-04-14T19:19:53.444488372Z",
"created_by": "/bin/sh -c (nop) ADD file:09a89925137e1b768ef1f0e7d1d7325eb2b4f1a0895b3aa8dfc98b0c75f3f507 in / "
},
{
"created": "2020-04-14T19:19:53.590635493Z",
"created_by": "/bin/sh -c (nop) CMD [\"sh\"]",
"empty_layer": true
}
],
"os": "linux",
"rootfs": {
"diff_ids": [
"sha256:5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab"
],
"type": "layers"
}
}
layer diff_id 和 digest 的对应关系
diffid-by-digest： 存放 digest 到 diffid 的对应关系
v2metadata-by-diffid： 存放 diffid 到 digest 的对应关系
tree /var/lib/docker/image/overlay2/distribution/
/var/lib/docker/image/overlay2/distribution/
├── diffid-by-digest
│ └── sha256
│ ├── e2334dd9fee4b77e48a8f2d793904118a3acf26f1f2e72a3d79c6cae993e07f0
│ └── ec237e1426f0de497da8c0951d5db1d2b7f155d207cfc0df02f3fb0508a3036a
└── v2metadata-by-diffid
└── sha256
├── 5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab
└── f53db92a80829925f8ec12396ab68569d7a769d9ab94983b86dbe6b26ae3d1fa
根据 diffid 获取 digest
~ cat /var/lib/docker/image/overlay2/distribution/v2metadata-by-diffid/sha256/5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab | python -m json.tool
[
{
"Digest": "sha256:e2334dd9fee4b77e48a8f2d793904118a3acf26f1f2e72a3d79c6cae993e07f0",
"HMAC": "",
"SourceRepository": "docker.io/library/busybox"
}
]
根据 digest 获取 diffid
cat /var/lib/docker/image/overlay2/distribution/diffid-by-digest/sha256/e2334dd9fee4b77e48a8f2d793904118a3acf26f1f2e72a3d79c6cae993e07f0
sha256:5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab
根据 digest 查看 layer 元数据
cd /var/lib/docker/image/overlay2/layerdb/sha256/5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab/ && ls -l
total 24
-rw-r--r-- 1 root root 64 Apr 30 15:21 cache-id
-rw-r--r-- 1 root root 71 Apr 30 15:21 diff
-rw-r--r-- 1 root root 7 Apr 30 15:21 size
-rw-r--r-- 1 root root 12159 Apr 30 15:21 tar-split.json.gz
cache-id 是 docker 下载 layer 的时候随机生成 uuid，指向真正存放 layer 文件的位置。
cat cache-id
486d3aacd2ee026b1e3db99b23e32a9f60d0972f7224fd53657a41943c871872
diff 文件存放 layer 的 diffid
cat diff
sha256:5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab
size 当前 layer 的大小，单位是字节。
tar-split.json.gz 是 layer 的 split 文件，通过这个文件可以还原 layer 的 tar 包，在 docker save 导出 image 时会用到。
layer 数据，image 存储数据，使用 cache-id 访问。
cd /var/lib/docker/overlay2/486d3aacd2ee026b1e3db99b23e32a9f60d0972f7224fd53657a41943c871872
tree -d diff/
diff/
├── bin
├── dev
├── etc
│ └── network
│ ├── if-down.d
│ ├── if-post-down.d
│ ├── if-pre-up.d
│ └── if-up.d
├── home
├── root
├── tmp
├── usr
│ └── sbin
└── var
├── spool
│ └── mail
└── www
manifest 存储对 config 和 layer 的 sha256 + media type 描述，目的就是为了下载 config 和 layer。下载完成后，manifest 的使命就完成了，所以在本地没有存储 manifest 文件。docker 空间管理
查看空间
docker system df
TYPE TOTAL ACTIVE SIZE RECLAIMABLE
Images 11 9 1.084GB 202.9MB (18%)
Containers 18 12 10.61kB 0B (0%)
Local Volumes 0 0 0B 0B
Build Cache 0 0 0B 0B
清理文件系统
docker system prune
WARNING! This will remove:
- all stopped containers
- all networks not used by at least one container
- all dangling images
- all dangling build cache
Are you sure you want to continue? [y/N]
强制清理，忽略依赖关系。
docker system prune -a -f
清理所有状态为 <none> 的 image
docker image prune
WARNING! This will remove all images without at least one container associated to them.
Are you sure you want to continue? [y/N]
强制清理 image，忽略依赖关系。
docker image prune -a
WARNING! This will remove all images without at least one container associated to them.
Are you sure you want to continue? [y/N]
清理无用数据卷
docker volume ls
docker volume prune
WARNING! This will remove all local volumes not used by at least one container.
Are you sure you want to continue? [y/N]
清理所有停止的容器
docker container prune
WARNING! This will remove all stopped containers.
Are you sure you want to continue? [y/N]配置系统
内核关闭 swap
ansible ins -m shell -a 'echo "vm.swappiness = 0" >> /etc/sysctl.conf && sysctl -p'零时关闭 swap
ansible ins -m shell -a 'swapoff -a'系统 fstab 关闭配置
ansible ins -m shell -a "sed -i 's/^[^#].*swap*/#&/g' /etc/fstab && chattr +i /etc/fstab"- WARNING: No swap limit support
sed -i '/GRUB_CMDLINE_LINUX=/cGRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0 console=tty0 console=ttyS0,115200 cgroup_enable=memory swapaccount=1"' /etc/default/grub
ansible ins -m copy -a 'src=/etc/default/grub dest=/etc/default/grub'
ansible ins -m shell -a 'update-grub && shutdown -r +1 "reboot for turnoff swap"'安装 kubernetes
- 激活旧版本的防火墙配置（19年之后的操作系统需要修改）
apt install -y iptables arptables ebtables
update-alternatives --set iptables /usr/sbin/iptables-legacy
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
update-alternatives --set arptables /usr/sbin/arptables-legacy
update-alternatives --set ebtables /usr/sbin/ebtables-legacy配置 kubernetes repo
批量配置
ansible ins -m shell -a 'curl -fsSL https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -'
ansible ins -m shell -a 'add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main"'安装 kubernetes 基础工具
查看当前支持的版本信息
apt-cache policy kubeadm
ansible ins -m shell -a 'apt update && apt install -y kubelet=1.17.3-00 kubeadm=1.17.3-00 kubectl=1.17.3-00'配置 driver
cat > /etc/default/kubelet <<EOF
KUBELET_EXTRA_ARGS=--cgroup-driver=systemd
EOF
cat > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf <<EOF
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cgroup-driver=systemd"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# 这是 "kubeadm init" 和 "kubeadm join" 运行时生成的文件，动态地填充 KUBELET_KUBEADM_ARGS 变量
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# 这是一个文件，用户在不得已下可以将其用作替代 kubelet args。
# 用户最好使用 .NodeRegistration.KubeletExtraArgs 对象在配置文件中替代。
# KUBELET_EXTRA_ARGS 应该从此文件中获取。
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
EOF
ansible ins -m copy -a 'src=/etc/default/kubelet dest=/etc/default/kubelet'
ansible ins -m copy -a 'src=/etc/systemd/system/kubelet.service.d/10-kubeadm.conf dest=/etc/systemd/system/kubelet.service.d/10-kubeadm.conf'重启 kubelet
ansible ins -m shell -a 'systemctl daemon-reload && systemctl restart kubelet'开启 ipvs 支持
配置内核支持 IP 转发
ansible ins -m shell -a 'grep net.ipv4.ip_forward /etc/sysctl.conf && sed -i "/net.ipv4.ip_forward/c\net.ipv4.ip_forward=1" /etc/sysctl.conf || echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf'
ansible ins -m shell -a 'grep net.bridge.bridge-nf-call-iptables /etc/sysctl.conf && sed -i "/net.bridge.bridge-nf-call-iptables/c\net.bridge.bridge-nf-call-iptables=1" /etc/sysctl.conf || echo "net.bridge.bridge-nf-call-iptables=1" >> /etc/sysctl.conf'
ansible ins -m shell -a 'grep net.bridge.bridge-nf-call-ip6tables /etc/sysctl.conf && sed -i "/net.bridge.bridge-nf-call-ip6tables/c\net.bridge.bridge-nf-call-ip6tables=1" /etc/sysctl.conf || echo "net.bridge.bridge-nf-call-ip6tables=1" >> /etc/sysctl.conf'
ansible ins -m shell -a 'sysctl -p'配置内核加载 ipvs 模块
cat > /tmp/ipvs.modules <<EOF
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack_ipv4"
for kernel_module in \${ipvs_modules}; do
/sbin/modinfo -F filename \${kernel_module} > /dev/null 2>&1
if [ $? -eq 0 ]; then
/sbin/modprobe \${kernel_module}
fi
done
EOF
ansible ins -m copy -a 'src=/tmp/ipvs.modules dest=/tmp/ipvs.modules mode=0755'
ansible ins -m shell -a '/tmp/ipvs.modules'
验证 ipvs 支持
ansible ins -m shell -a 'lsmod | grep ip_vs'安装 ipvsadm
ansible ins -m yum -a 'name=ipvsadm state=present'创建 kubernetes 集群
ukm01 上执行
- 内网负载均衡地址 10.10.34.89
配置文件创建，可以修改较多的参数。
cat > kubeadm-config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 0.0.0.0
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
kubeletExtraArgs:
cgroup-driver: "systemd"
ignorePreflightErrors:
- IsPrivilegedUser
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
controlPlaneEndpoint: 192.168.0.113:6443
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
dns:
type: CoreDNS
apiServer:
timeoutForControlPlane: 5m0s
extraArgs:
authorization-mode: "Node,RBAC"
certSANs:
- "192.168.0.113"
- "192.168.0.110"
- "192.168.0.111"
- "192.168.0.112"
- "node1"
- "node2"
- "node3"
- "kubernetes"
- "kubernetes.default"
- "kubernetes.default.svc"
- "kubernetes.default.svc.cluster"
- "kubernetes.default.svc.cluster.local"
controllerManager:
extraArgs:
"node-cidr-mask-size": "20"
scheduler:
extraArgs:
address: "0.0.0.0"
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kubernetesVersion: v1.20.1
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
podSubnet: 192.168.0.0/16
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
mode: ipvs
ipvs:
scheduler: wlc
syncPeriod: 30s
minSyncPeriod: 5s
tcpTimeout: 0s
tcpFinTimeout: 0s
udpTimeout: 0s
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
rotateCertificates: true
EOF
验证 kubeadm-config 配置
kubeadm init --config=kubeadm-config.yaml --upload-certs
创建集群
kubeadm init --config=kubeadm-config.yaml --upload-certs
参数化创建集群
kubeadm init \
--apiserver-bind-port 8443 \
--control-plane-endpoint "10.10.34.89:8443" \
--image-repository registry.cn-hangzhou.aliyuncs.com/google_containers \
--kubernetes-version=v1.17.3 \
--upload-certs \
--pod-network-cidr=10.100.0.0/16 \
--service-cidr=10.96.0.0/12 \
...
kubeadm join 10.10.34.89:8443 --token 7nf8wp.cefgemzgn5xcdmln \
--discovery-token-ca-cert-hash sha256:36b328e0052c7f8e7cac826d70b21bef0cdc4e0505fddc8200c52b8b01605da3 \
--control-plane --certificate-key bd9fc1600d2db4f2b323a17b2b7be4f4234d7ed8705bc64976dcd43d929a24a8
...
kubeadm join 10.10.34.89:8443 --token 7nf8wp.cefgemzgn5xcdmln \
--discovery-token-ca-cert-hash sha256:36b328e0052c7f8e7cac826d70b21bef0cdc4e0505fddc8200c52b8b01605da3kubeadm init 常用参数
--apiserver-advertise-address string
API 服务器所公布的其正在监听的 IP 地址。如果未设置，则使用默认网络接口。
--apiserver-bind-port int32 默认值：6443
API 服务器绑定的端口。
--apiserver-cert-extra-sans stringSlice
用于 API Server 服务证书的可选附加主题备用名称（SAN）。可以是 IP 地址和 DNS 名称。
--cert-dir string 默认值："/etc/kubernetes/pki"
保存和存储证书的路径。
--certificate-key string
用于加密 kubeadm-certs Secret 中的控制平面证书的密钥。
--config string
kubeadm 配置文件的路径。
--control-plane-endpoint string
为控制平面指定一个稳定的 IP 地址或 DNS 名称。
--cri-socket string
要连接的 CRI 套接字的路径。如果为空，则 kubeadm 将尝试自动检测此值；仅当安装了多个 CRI 或具有非标准 CRI 插槽时，才使用此选项。
--dry-run
不要应用任何更改；只是输出将要执行的操作。
-k, --experimental-kustomize string
用于存储 kustomize 为静态 pod 清单所提供的补丁的路径。
--feature-gates string
一组用来描述各种功能特性的键值（key=value）对。选项是：
IPv6DualStack=true|false (ALPHA - default=false)
-h, --help
init 操作的帮助命令
--ignore-preflight-errors stringSlice
错误将显示为警告的检查列表；例如：'IsPrivilegedUser,Swap'。取值为 'all' 时将忽略检查中的所有错误。
--image-repository string 默认值："k8s.gcr.io"
选择用于拉取控制平面镜像的容器仓库
--kubernetes-version string 默认值："stable-1"
为控制平面选择一个特定的 Kubernetes 版本。
--node-name string
指定节点的名称。
--pod-network-cidr string
指明 pod 网络可以使用的 IP 地址段。如果设置了这个参数，控制平面将会为每一个节点自动分配 CIDRs。
--service-cidr string 默认值："10.96.0.0/12"
为服务的虚拟 IP 地址另外指定 IP 地址段
--service-dns-domain string 默认值："cluster.local"
为服务另外指定域名，例如："myorg.internal"。
--skip-certificate-key-print
不要打印用于加密控制平面证书的密钥。
--skip-phases stringSlice
要跳过的阶段列表
--skip-token-print
跳过打印 'kubeadm init' 生成的默认引导令牌。
--token string
这个令牌用于建立控制平面节点与工作节点间的双向通信。格式为 [a-z0-9]{6}\.[a-z0-9]{16} - 示例：abcdef.0123456789abcdef
--token-ttl duration 默认值：24h0m0s
令牌被自动删除之前的持续时间（例如 1 s，2 m，3 h）。如果设置为 '0'，则令牌将永不过期
--upload-certs
将控制平面证书上传到 kubeadm-certs Secret。master 节点扩容
在 ukm02 ukm03 上执行
ansible master:\!ans -m shell -a 'kubeadm join 10.10.34.89:8443 \
--apiserver-bind-port 8443 \
--control-plane \
--token 7nf8wp.cefgemzgn5xcdmln \
--discovery-token-ca-cert-hash sha256:36b328e0052c7f8e7cac826d70b21bef0cdc4e0505fddc8200c52b8b01605da3 \
--control-plane --certificate-key bd9fc1600d2db4f2b323a17b2b7be4f4234d7ed8705bc64976dcd43d929a24a8'kubeadm join 常用参数
--apiserver-advertise-address string
如果该节点托管一个新的控制平面实例，则 API 服务器将公布其正在侦听的 IP 地址。如果未设置，则使用默认网络接口。
--apiserver-bind-port int32 默认值: 6443
如果节点应该托管新的控制平面实例，则为 API 服务器要绑定的端口。
--certificate-key string
使用此密钥可以解密由 init 上传的证书 secret。
--config string
kubeadm 配置文件的路径。
--control-plane
在此节点上创建一个新的控制平面实例
--cri-socket string
要连接的 CRI 套接字的路径。如果为空，则 kubeadm 将尝试自动检测此值；仅当安装了多个 CRI 或具有非标准 CRI 插槽时，才使用此选项。
--discovery-file string
对于基于文件的发现，给出用于加载集群信息的文件或者 URL。
--discovery-token string
对于基于令牌的发现，该令牌用于验证从 API 服务器获取的集群信息。
--discovery-token-ca-cert-hash stringSlice
对基于令牌的发现，验证根 CA 公钥是否与此哈希匹配 (格式: "<type>:<value>")。
--discovery-token-unsafe-skip-ca-verification
对于基于令牌的发现，允许在未关联 --discovery-token-ca-cert-hash 参数的情况下添加节点。
-k, --experimental-kustomize string
用于存储 kustomize 为静态 pod 清单所提供的补丁的路径。
-h, --help
join 操作的帮助命令
--ignore-preflight-errors stringSlice
错误将显示为警告的检查列表；例如：'IsPrivilegedUser,Swap'。取值为 'all' 时将忽略检查中的所有错误。
--node-name string
指定节点的名称
--skip-phases stringSlice
要跳过的阶段列表
--tls-bootstrap-token string
指定在加入节点时用于临时通过 Kubernetes 控制平面进行身份验证的令牌。
--token string
如果未提供这些值，则将它们用于 discovery-token 令牌和 tls-bootstrap 令牌。配置 master 管理命令
ansible master -m shell -a 'echo "source <(kubectl completion bash)" >> ~/.bashrc'
ansible master -m shell -a 'mkdir -p /root/.kube'
ansible master -m copy -a 'src=/etc/kubernetes/admin.conf dest=/root/.kube/config mode=0600 owner=root remote_src=yes'验证 master
kubectl get nodes -l node-role.kubernetes.io/master
NAME STATUS ROLES AGE VERSION
ukm01 NotReady master 3m26s v1.17.3
ukm02 NotReady master 1m56s v1.17.3
ukm03 NotReady master 2m14s v1.17.3
kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-7f9c544f75-h9hmm 0/1 Pending 0 3m55s
kube-system coredns-7f9c544f75-k9pb7 0/1 Pending 0 3m55s
kube-system etcd-ukm01 1/1 Running 1 20s
kube-system etcd-ukm02 1/1 Running 0 40s
kube-system etcd-ukm03 1/1 Running 0 20s
kube-system kube-apiserver-ukm01 1/1 Running 0 20s
kube-system kube-apiserver-ukm02 1/1 Running 0 20s
kube-system kube-apiserver-ukm03 1/1 Running 0 20s
kube-system kube-controller-manager-ukm01 1/1 Running 3 5m17s
kube-system kube-controller-manager-ukm02 1/1 Running 0 74s
kube-system kube-controller-manager-ukm03 1/1 Running 2 3m27s
kube-system kube-proxy-4fkkx 1/1 Running 0 3m27s
kube-system kube-proxy-5vjpc 1/1 Running 0 2m9s
kube-system kube-proxy-plx2x 1/1 Running 0 3m55s
kube-system kube-scheduler-ukm01 1/1 Running 2 5m17s
kube-system kube-scheduler-ukm02 1/1 Running 0 75s
kube-system kube-scheduler-ukm03 1/1 Running 0 3m26s确认 kube-proxy 启用了 ipvs 模式
kubectl get configmaps kube-proxy -n kube-system -o yaml
...
iptables:
masqueradeAll: false
masqueradeBit: null
minSyncPeriod: 0s
syncPeriod: 0s
ipvs:
excludeCIDRs: null
minSyncPeriod: 5s
scheduler: wlc
strictARP: false
syncPeriod: 30s
tcpFinTimeout: 0s
tcpTimeout: 0s
udpTimeout: 0s
kind: KubeProxyConfiguration
metricsBindAddress: ""
mode: ipvs
...查看 ipvs 转发情况
ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.96.0.1:443 wlc
-> 10.10.34.92:8443 Masq 1 0 0
-> 10.10.34.93:8443 Masq 1 0 0
-> 10.10.34.94:8443 Masq 1 0 0
TCP 10.96.0.10:53 wlc
TCP 10.96.0.10:9153 wlc
UDP 10.96.0.10:53 wlc查看网卡信息
ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether fa:16:3e:cf:28:7f brd ff:ff:ff:ff:ff:ff
inet 10.10.34.92/24 brd 10.10.34.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::f816:3eff:fecf:287f/64 scope link
valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
link/ether 02:42:81:5a:e2:7c brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
inet6 fe80::42:81ff:fe5a:e27c/64 scope link
valid_lft forever preferred_lft forever
6: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
46: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN group default
link/ether 6a:10:8f:81:3e:61 brd ff:ff:ff:ff:ff:ff
inet 10.96.0.10/32 brd 10.96.0.10 scope global kube-ipvs0
valid_lft forever preferred_lft forever
inet 10.96.0.1/32 brd 10.96.0.1 scope global kube-ipvs0
valid_lft forever preferred_lft forever更新 etcd 集群配置
ansible master -m shell -a 'sed -i "/--initial-cluster=/c\ - --initial-cluster=ukm01=https://10.10.34.92:2380,ukm02=https://10.10.34.93:2380,ukm03=https://10.10.34.94:2380" /etc/kubernetes/manifests/etcd.yaml'
apiserver 默认访问本地的 etcd 节点，如果需要访问集群所有节点可以修改配置。
ansible master -m shell -a 'sed -i "/- --etcd-servers/c\ - --etcd-servers=https://10.10.34.92:2379,https://10.10.34.93:2379,https://10.10.34.94:2379" /etc/kubernetes/manifests/kube-apiserver.yaml'查看 etcd 成员状态
成员管理命令：
- add - 添加集群成员
- list - 查看集群成员信息
- promote - 提升集群成员权限
- remove - 从集群中删除成员ID
- update - 更新集群成员信息
kubectl exec -n kube-system -it $(kubectl get pod -n kube-system -l component=etcd -o jsonpath='{.items[0].metadata.name}') sh
或者下载 etcdctl
curl -L -O https://github.com/etcd-io/etcd/releases/download/v3.4.3/etcd-v3.4.3-linux-amd64.tar.gz
tar xzvf etcd-v3.4.3-linux-amd64.tar.gz
mv etcd-v3.4.3-linux-amd64/etcd* /usr/local/bin/
ETCDCTL=3 etcdctl --write-out=table \
--cert /etc/kubernetes/pki/etcd/peer.crt \
--key /etc/kubernetes/pki/etcd/peer.key \
--cacert /etc/kubernetes/pki/etcd/ca.crt \
--endpoints https://ukm01:2379,https://ukm02:2379,https://ukm03:2379 \
member list
+------------------+---------+-------+--------------------------+--------------------------+------------+
| ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS | IS LEARNER |
+------------------+---------+-------+--------------------------+--------------------------+------------+
| 7a212ff0c04f1e82 | started | ukm02 | https://10.10.34.93:2380 | https://10.10.34.93:2379 | false |
| ae758756c0c6b09c | started | ukm01 | https://10.10.34.92:2380 | https://10.10.34.92:2379 | false |
| ff90415307bdc341 | started | ukm03 | https://10.10.34.94:2380 | https://10.10.34.94:2379 | false |
+------------------+---------+-------+--------------------------+--------------------------+------------+验证 etcd 集群状态
ETCDCTL=3 etcdctl --cluster --write-out=table \
--cert /etc/kubernetes/pki/etcd/peer.crt \
--key /etc/kubernetes/pki/etcd/peer.key \
--cacert /etc/kubernetes/pki/etcd/ca.crt \
--endpoints https://ukm01:2379,https://ukm02:2379,https://ukm03:2379 \
endpoint status
+--------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+--------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.10.34.94:2379 | 4677521bc1465d7e | 3.4.3 | 1.5 MB | false | false | 9 | 1671 | 1671 | |
| https://10.10.34.92:2379 | ae758756c0c6b09c | 3.4.3 | 1.6 MB | false | false | 9 | 1671 | 1671 | |
| https://10.10.34.93:2379 | ff3c0a769521e2c2 | 3.4.3 | 1.5 MB | true | false | 9 | 1671 | 1671 | |
+--------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+查看 etcd 节点健康状况
ETCDCTL=3 etcdctl --cluster --write-out=table \
--cert /etc/kubernetes/pki/etcd/peer.crt \
--key /etc/kubernetes/pki/etcd/peer.key \
--cacert /etc/kubernetes/pki/etcd/ca.crt \
--endpoints https://ukm01:2379,https://ukm02:2379,https://ukm03:2379 \
endpoint health
+--------------------------+--------+-------------+-------+
| ENDPOINT | HEALTH | TOOK | ERROR |
+--------------------------+--------+-------------+-------+
| https://10.10.34.93:2379 | true | 18.384799ms | |
| https://10.10.34.94:2379 | true | 18.649393ms | |
| https://10.10.34.92:2379 | true | 18.831792ms | |
+--------------------------+--------+-------------+-------+增加 node 节点
在 uki01 uki02 uki03 ukn01 ukn02 ukn03 上执行
ansible infra:worker -m shell -a 'kubeadm join 10.10.34.89:8443 \
--token 7nf8wp.cefgemzgn5xcdmln \
--discovery-token-ca-cert-hash sha256:36b328e0052c7f8e7cac826d70b21bef0cdc4e0505fddc8200c52b8b01605da3'SDN
安装 calico
curl -L https://docs.projectcalico.org/manifests/calico.yaml -o calico.yaml
kubectl apply -f calico.yaml查看服务状态
- node 节点处于 Ready 状态
- 每个节点上都有一个 pod：calico-node
kubectl get nodes
NAME STATUS ROLES AGE VERSION
uki01 Ready infra 5m v1.17.3
uki02 Ready infra 5m v1.17.3
uki03 Ready infra 5m v1.17.3
ukm01 Ready master 8m v1.17.3
ukm02 Ready master 8m v1.17.3
ukm03 Ready master 8m v1.17.3
ukn01 Ready worker 5m v1.17.3
ukn02 Ready worker 5m v1.17.3
ukn03 Ready worker 5m v1.17.3
watch kubectl get pods -n kube-system -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
...
calico-kube-controllers-5b644bc49c-7xjzr 1/1 Running 0 30s 192.168.144.138 uki01 <none> <none>
calico-node-57rns 1/1 Running 0 30s 10.10.34.98 ukn01 <none> <none>
calico-node-6p8z7 1/1 Running 0 30s 10.10.34.100 ukn03 <none> <none>
calico-node-77jwm 1/1 Running 0 30s 172.17.0.1 ukn02 <none> <none>
calico-node-c4gxf 1/1 Running 0 30s 10.10.34.95 uki01 <none> <none>
calico-node-gdb5j 1/1 Running 0 30s 10.10.34.96 uki02 <none> <none>
calico-node-gmzmf 1/1 Running 0 30s 10.10.34.93 ukm02 <none> <none>
calico-node-jdd2f 1/1 Running 0 30s 10.10.34.97 uki03 <none> <none>
calico-node-mrr49 1/1 Running 0 30s 10.10.34.92 ukm01 <none> <none>
calico-node-w5b7r 1/1 Running 0 30s 10.10.34.94 ukm03 <none> <none>
...
kubectl get ippool -o wide
NAME AGE
default-ipv4-ippool 2m验证 calico
下载 calicoctl
curl -L -o /usr/local/bin/calicoctl https://github.com/projectcalico/calicoctl/releases/download/v3.11.2/calicoctl-linux-amd64
chmod +x /usr/local/bin/calicoctl
使用 kubernetes api 存储数据
mkdir -p /etc/calico
cat > /etc/calico/calicoctl.cfg <<EOF
apiVersion: projectcalico.org/v3
kind: CalicoAPIConfig
metadata:
spec:
datastoreType: kubernetes
kubeconfig: /etc/kubernetes/admin.conf 使用 admin 配置
k8sAPIEndpoint: https://10.10.34.89:8443
k8sCertFile: /etc/kubernetes/pki/apiserver-kubelet-client.crt
k8sKeyFile: /etc/kubernetes/pki/apiserver-kubelet-client.key
k8sCAFile: /etc/kubernetes/pki/ca.crt
EOF
使用独立的 etcd 存储数据
mkdir -p /etc/calico
cat > /etc/calico/calicoctl.cfg <<EOF
apiVersion: projectcalico.org/v3
kind: CalicoAPIConfig
metadata:
spec:
datastoreType: etcdv3
etcdEndpoints: https://10.10.34.92:2379,https://10.10.34.93:2379,https://10.10.34.94:2379
etcdKeyFile: /etc/kubernetes/pki/etcd/peer.key
etcdCertFile: /etc/kubernetes/pki/etcd/peer.crt
etcdCACertFile: /etc/kubernetes/pki/etcd/ca.crt
EOF
查看 node
calicoctl node status
Calico process is running.
IPv4 BGP status
+--------------+-------------------+-------+------------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+--------------+-------------------+-------+------------+-------------+
| 10.10.34.95 | node-to-node mesh | up | 2020-04-11 | Established |
| 10.10.34.96 | node-to-node mesh | up | 2020-04-11 | Established |
| 10.10.34.97 | node-to-node mesh | up | 2020-04-11 | Established |
| 10.10.34.93 | node-to-node mesh | up | 2020-04-11 | Established |
| 10.10.34.94 | node-to-node mesh | up | 2020-04-11 | Established |
| 10.10.34.98 | node-to-node mesh | up | 2020-04-11 | Established |
| 10.10.34.99 | node-to-node mesh | up | 2020-04-11 | Established |
| 10.10.34.100 | node-to-node mesh | up | 2020-04-11 | Established |
+--------------+-------------------+-------+------------+-------------+
IPv6 BGP status
No IPv6 peers found.
calicoctl get node -o wide
NAME ASN IPV4 IPV6
uki01 (64512) 10.10.34.95/24
uki02 (64512) 10.10.34.96/24
uki03 (64512) 10.10.34.97/24
ukm01 (64512) 10.10.34.92/24
ukm02 (64512) 10.10.34.93/24
ukm03 (64512) 10.10.34.94/24
ukn01 (64512) 10.10.34.98/24
ukn02 (64512) 10.10.34.99/24
ukn03 (64512) 10.10.34.100/24
查看 ippool
calicoctl get ippool -o wide
NAME CIDR NAT IPIPMODE VXLANMODE DISABLED SELECTOR
default-ipv4-ippool 192.168.0.0/16 true Always Never false all()
查看 ip 状态
calicoctl ipam show
+----------+----------------+-----------+------------+--------------+
| GROUPING | CIDR | IPS TOTAL | IPS IN USE | IPS FREE |
+----------+----------------+-----------+------------+--------------+
| IP Pool | 192.168.0.0/16 | 65536 | 5 (0%) | 65531 (100%) |
+----------+----------------+-----------+------------+--------------+
查看已分配的服务节点信息
calicoctl get workloadendpoints -o wide -n kube-system
NAMESPACE NAME WORKLOAD NODE NETWORKS INTERFACE PROFILES NATS
kube-system ukn02-k8s-calico--kube--controllers--75d56dfc47--xfdp9-eth0 calico-kube-controllers-75d56dfc47-xfdp9 ukn02 192.168.33.65/32 calif1d8798ea15 kns.kube-system,ksa.kube-system.calico-kube-controllers
kube-system uki02-k8s-coredns--546565776c--mkx9n-eth0 coredns-546565776c-mkx9n uki02 192.168.225.1/32 calib399f7859db kns.kube-system,ksa.kube-system.coredns
kube-system ukn03-k8s-coredns--546565776c--v9wgw-eth0 coredns-546565776c-v9wgw ukn03 192.168.188.129/32 cali4fffc74a7b2 kns.kube-system,ksa.kube-system.corednscalico 网络资源
官方文档： https://docs.projectcalico.org/reference/resources/overview
- BGPConfiguration
---
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
logSeverityScreen: Info
nodeToNodeMeshEnabled: true
asNumber: 63400
serviceClusterIPs:
- cidr: 10.96.0.0/12
serviceExternalIPs:
- cidr: 104.244.42.129/32
- cidr: 172.217.3.0/24- BGPPeer
---
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
name: some.name
spec:
node: rack1-host1
peerIP: 192.168.1.1
asNumber: 63400- FelixConfiguration
---
apiVersion: projectcalico.org/v3
kind: FelixConfiguration
metadata:
name: default
spec:
ipv6Support: false
ipipMTU: 1400
chainInsertMode: Append- GlobalNetworkPolicy
---
apiVersion: projectcalico.org/v3
kind: GlobalNetworkPolicy
metadata:
name: allow-tcp-6379
spec:
selector: role == 'database'
types:
- Ingress
- Egress
ingress:
- action: Allow
metadata:
annotations:
from: frontend
to: database
protocol: TCP
source:
selector: role == 'frontend'
destination:
ports:
- 6379
egress:
- action: Allow- GlobalNetworkSet
---
apiVersion: projectcalico.org/v3
kind: GlobalNetworkSet
metadata:
name: a-name-for-the-set
labels:
role: external-database
spec:
nets:
- 198.51.100.0/28
- 203.0.113.0/24- HostEndpoint
---
apiVersion: projectcalico.org/v3
kind: HostEndpoint
metadata:
name: some.name
labels:
type: production
spec:
interfaceName: eth0
node: myhost
expectedIPs:
- 192.168.0.1
- 192.168.0.2
profiles:
- profile1
- profile2
ports:
- name: some-port
port: 1234
protocol: TCP
- name: another-port
port: 5432
protocol: UDP- IPPool
---
apiVersion: projectcalico.org/v3
kind: IPPool
metadata:
name: my.ippool-1
spec:
cidr: 10.1.0.0/16
ipipMode: CrossSubnet
natOutgoing: true
disabled: false
nodeSelector: all()- NetworkPolicy
---
apiVersion: projectcalico.org/v3
kind: NetworkPolicy
metadata:
name: allow-tcp-6379
namespace: production
spec:
selector: role == 'database'
types:
- Ingress
- Egress
ingress:
- action: Allow
metadata:
annotations:
from: frontend
to: database
protocol: TCP
source:
selector: role == 'frontend'
destination:
ports:
- 6379
egress:
- action: Allow- NetworkSet
---
apiVersion: projectcalico.org/v3
kind: NetworkSet
metadata:
name: external-database
namespace: staging
labels:
role: db
spec:
nets:
- 198.51.100.0/28
- 203.0.113.0/24- Node
---
apiVersion: projectcalico.org/v3
kind: Node
metadata:
name: node-hostname
spec:
bgp:
asNumber: 64512
ipv4Address: 10.244.0.1/24
ipv6Address: 2001:db8:85a3::8a2e:370:7334/120
ipv4IPIPTunnelAddr: 192.168.0.1- Profile
---
apiVersion: projectcalico.org/v3
kind: Profile
metadata:
name: dev-apps
spec:
ingress:
- action: Deny
source:
nets:
- 10.0.20.0/24
- action: Allow
source:
selector: stage == 'development'
egress:
- action: Allow
labelsToApply:
stage: development- WorkloadEndpoint
---
apiVersion: projectcalico.org/v3
kind: WorkloadEndpoint
metadata:
name: node1-k8s-my--nginx--b1337a-eth0
namespace: default
labels:
app: frontend
projectcalico.org/namespace: default
projectcalico.org/orchestrator: k8s
spec:
node: node1
orchestrator: k8s
endpoint: eth0
containerID: 1337495556942031415926535
pod: my-nginx-b1337a
endpoint: eth0
interfaceName: cali0ef24ba
mac: ca:fe:1d:52:bb:e9
ipNetworks:
- 192.168.0.0/32
profiles:
- profile1
ports:
- name: some-port
port: 1234
protocol: TCP
- name: another-port
port: 5432
protocol: UDP卸载 calico
kubectl delete -f calico.yaml删除 calico 配置
ansible ins -m shell -a 'rm -rf /etc/cni /opt/cni/ /var/lib/calico/'集群校验
查看集群信息
kubectl config get-clusters
NAME
shenzhen
kubectl cluster-info
Kubernetes master is running at https://10.10.34.89:8443
KubeDNS is running at https://10.10.34.89:8443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy查看 namespace
kubectl get namespaces
NAME STATUS AGE
default Active 15m
kube-node-lease Active 15m
kube-public Active 15m
kube-system Active 15m查看节点状态
kubectl get nodes
NAME STATUS ROLES AGE VERSION
uki01 NotReady <none> 7m v1.17.3
uki02 NotReady <none> 7m v1.17.3
uki03 NotReady <none> 7m v1.17.3
ukm01 NotReady master 10m v1.17.3
ukm02 NotReady master 14m v1.17.3
ukm03 NotReady master 13m v1.17.3
ukn01 NotReady <none> 7m v1.17.3
ukn02 NotReady <none> 7m v1.17.3
ukn03 NotReady <none> 7m v1.17.3添加 infra 标签
kubectl label node uki01 uki02 uki03 node-role.kubernetes.io/infra=true
node/uki01 labeled
node/uki02 labeled
node/uki03 labeled
kubectl get nodes -l node-role.kubernetes.io/infra
NAME STATUS ROLES AGE VERSION
uki01 NotReady infra 8m v1.17.3
uki02 NotReady infra 8m v1.17.3
uki03 NotReady infra 8m v1.17.3添加 worker 标签
kubectl label node ukn01 ukn02 ukn03 node-role.kubernetes.io/worker=true
node/ukn01 labeled
node/ukn02 labeled
node/ukn03 labeled
kubectl get nodes -l node-role.kubernetes.io/worker
NAME STATUS ROLES AGE VERSION
ukn01 NotReady worker 8m v1.17.3
ukn02 NotReady worker 8m v1.17.3
ukn03 NotReady worker 8m v1.17.3查看所有节点标签
kubectl get nodes --show-labels
NAME STATUS ROLES AGE VERSION LABELS
uki01 NotReady infra 10m v1.17.3 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=uki01,kubernetes.io/os=linux,node-role.kubernetes.io/infra=true
uki02 NotReady infra 10m v1.17.3 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=uki02,kubernetes.io/os=linux,node-role.kubernetes.io/infra=true
uki03 NotReady infra 10m v1.17.3 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=uki03,kubernetes.io/os=linux,node-role.kubernetes.io/infra=true
ukm01 NotReady master 19m v1.17.3 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukm01,kubernetes.io/os=linux,node-role.kubernetes.io/master=
ukm02 NotReady master 17m v1.17.3 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukm02,kubernetes.io/os=linux,node-role.kubernetes.io/master=
ukm03 NotReady master 15m v1.17.3 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukm03,kubernetes.io/os=linux,node-role.kubernetes.io/master=
ukn01 NotReady worker 10m v1.17.3 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukn01,kubernetes.io/os=linux,node-role.kubernetes.io/worker=true
ukn02 NotReady worker 10m v1.17.3 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukn02,kubernetes.io/os=linux,node-role.kubernetes.io/worker=true
ukn03 NotReady worker 10m v1.17.3 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukn03,kubernetes.io/os=linux,node-role.kubernetes.io/worker=true删除节点标签
kubectl label node uki01 uki02 uki03 node-role.kubernetes.io/infra-kubectl 简介
基础命令:
create 从文件或 stdin 创建资源
expose 为 deployment、pod 创建 Service
run 使用 image 去创建一个服务
set 更新resource ,比如更新env环境变量，image，resources 资源限制，selector subject等。
管理命令:
explain 查看资源定义（文档）。如 kubectl explain replicaset
get 最基本的对象查询命令。如 kubectl get nodes/pods/deploy/rs/ns/secret等等， 加-o wide查看详细信息，-o yaml 或-o json 输出具体格式。
edit 编辑资源配置，完成对象的更新。
delete 删除指定资源，支持文件名、资源名、label selector。
部署命令:
rollout Deployment, Daemonset的升级过程管理（查看状态status、操作历史history、暂停升级、恢复升级、回滚等）
scale 修改Deployment, ReplicaSet, Replication Controller, Job的实例数,实现一个副本集的手工扩展
autoscale 为Deploy, RS, RC配置自动伸缩规则（依赖heapster和hpa）
集群管理命令:
certificate 修改证书资源
cluster-info 查看集群信息
top 查看资源占用率 (CPU/Memory/Storage)
cordon 标记节点为不可调度状态
uncordon 标记节点为可调度状态
drain 驱逐节点上的 pod ，准备下线维护
taint 设置节点污点隔离标记
故障诊断命令:
describe 查看资源详情
logs 查看 pod 内容器的日志
attach Attach 到 pod 内的一个容器
exec 进入指定容器执行命令
port-forward 为 pod 创建本地端口映射
proxy 为 Kubernetes API server 创建代理
cp 容器内外/容器间文件拷贝
auth 授权校验
高级命令:
diff 比较当前版本和要部署的版本
apply 从文件或stdin创建/更新资源
patch 使用strategic merge patch语法更新对象的某些字段
replace 从文件或stdin更新资源
wait 实验功能：等待一种或多种资源的特定条件。
convert 在不同API版本之间转换对象定义
kustomize 从指定目录或远程URL构建Kustomization目标。
配置命令:
label 给资源设置label
annotate 给资源设置annotation
completion 获取shell自动补全脚本 (bash or zsh)
其他命令:
alpha Alpha中功能的命令
api-resources 打印当前 kubernetes API 所支持的命令资源
api-versions 打印当前 kubernetes API 所支持的 api 接口以及版本信息
config 修改kubectl配置（kubeconfig文件），如context。
plugin 提供用于与插件交互的插件程序。
version 打印当前 kubernetes 的服务端和客户端版本调度策略 Scheduling Policies
kube-scheduler 给一个 pod 做调度选择包含两个步骤：
过滤 Filtering
过滤策略会检查候选 Node 是否满足 Pod 的创建需求；
在过滤之后得出一个 Node 列表，里面包含了所有可调度节点；
如果为空，代表这个 Pod 不可调度。打分 Scoring
根据当前启用的打分规则，调度器会给每一个可调度节点进行打分。
过滤策略
- PodFitsHostPorts：检查 Pod 请求的端口是否空闲（网络协议类型）。
- PodFitsHost：检查 Pod 是否通过其主机名指定了特定的 Node。
- PodFitsResources：检查节点空闲资源（例如 CPU 和内存）是否满足 Pod 的要求。
- PodMatchNodeSelector：检查 Pod label 是否与 Node label 匹配。
- NoVolumeZoneConflict：检查 Volume 存储卷是否可用。
- NoDiskConflict：检查节点 Disk 磁盘是否冲突。
- MaxCSIVolumeCount：检查节点挂载的 CSI 卷是否超出最大限制。
- CheckNodeMemoryPressure：节点正在报告内存压力，并且没有配置的异常，则不会在此处安排 Pod。
- CheckNodePIDPressure：节点正在报告进程 PID 稀缺，并且没有配置的异常，则不会在此处安排 Pod。
- CheckNodeDiskPressure：节点正在报告存储压力，并且没有配置的异常，则不会在此处安排Pod。
- CheckNodeCondition：检查节点的文件系统、网络状态或者 kubelet 是否准备好运行 Pod。
- PodToleratesNodeTaints：检查 Pod 的 Tolerates 是否可以容忍节点的 Taints。
- CheckVolumeBinding：检查 PVC 状态是否满足 Pod 创建需求。
打分策略
- SelectorSpreadPriority：将属于同一 StatefulSet、 ReplicaSet 或 Service 的 Pod 分散在不同的主机 。
- InterPodAffinityPriority：根据 Pod 亲和条目打分，匹配到给定节点的条目权重相加，结果值越大的节点得分越高。
- LeastRequestedPriority：计算Pods需要的CPU和内存在当前节点可用资源的百分比，具有最小百分比的节点就是最优。
- MostRequestedPriority：适用动态伸缩集群环境，会优先调度pod到使用率最高的主机节点。当伸缩集群时，就会腾出空闲机器，从而可以停机处理。
- RequestedToCapacityRatioPriority：为节点上每个资源占用比例设定得分值，给资源打分函数在打分时使用。
- BalancedResourceAllocation：优选那些使得资源利用率更为均衡的节点。
- NodePreferAvoidPodsPriority：根据节点是否包含 scheduler.alpha.kubernetes.io/preferAvoidPods 来计算其优先级，将两个不同 Pod 运行在不同的节点上。
- NodeAffinityPriority：基于 PreferredDuringSchedulingIgnoredDuringExecution 来进行节点亲和调度。
- TaintTolerationPriority：基于 Pod 中对节点的污点容忍程度进行优先级评估，这个策略能够调整待选节点的排名。
- ImageLocalityPriority：已经拥有 Pod 需要的 image 的节点会有较高的优先级。
- ServiceSpreadingPriority：确保归属于同一给 Service 的 Pod 调度到不同的节点上，确保节点宕机之后 Service 也具有很强容灾能力。
- EqualPriority：将所有的 Node 设置成相同的权重为 1。
- EvenPodsSpreadPriority：实现首选的Pod拓扑扩展约束。
调度配置 Scheduling Profiles
官方文档 https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/
每次调度一个Pod的尝试都分为两个阶段，调度周期 Scheduling Cycle 和 绑定周期 Binding Cycle。
- 调度周期为Pod选择一个节点，并且绑定周期将该决定应用于集群。调度周期和绑定周期一起被称为“调度上下文”。
- 调度周期是串行运行的，而绑定周期可能是同时运行的。
- 如果确定Pod不可调度或存在内部错误，则可以中止调度或绑定周期。Pod将返回队列并重试。
扩展点 Extension points
QueueSort - 对 Pod 的待调度队列进行排序，比较两个 Pod 谁更优先调度，同一时间点只能有一个 QueueSort 插件生效。
调度周期 Scheduling Cycle
- PreFilter - 对 Pod 条件信息进行预处理，如果 PreFilter 返回了 error 则调度过程终止。
- Filter - 标记不符合调度条件的
- PreScore - 对 Pod 进行预评分，为 Score 插件使用提供可共享的状态。如果PreScore插件返回错误，则调度周期将中止。
- Score - 对符合过滤条件的节点进行计分排名，在 NormalizeScore 阶段之后，调度程序将根据权重合并所有的节点分值。
- NormalizeScore - 在调度程序计算节点的最终排名之前修改分数。
- Reserve - 在符合条件的节点上为 Pod 保留资源。
- Permit - 用于阻止或者延迟 Pod 与节点的绑定。
- approve - 批准
- deny - 拒绝
- wait - 等待
绑定周期 Binding Cycle
- PreBind - 用于执行绑定 Pod 之前所需的任何工作。如果任何 PreBind 插件返回错误，则 Pod 被拒绝并返回到调度队列。
- Bind - 用于 Pod 绑定。如果绑定插件选择处理 Pod，则会跳过其余的绑定插件。
- PostBind - 成功绑定Pod后，将调用后绑定插件。绑定周期到此结束，可以用来清理关联的资源。
Unreserve - 如果节点为 Pod 预留了资源，Pod 又在被绑定过程中被拒绝绑定，则执行 unreserve 释放为 Pod 预留的资源。
调度插件 Scheduling plugins
默认情况下启用以下插件，实现这些扩展点中的一个或多个：
- DefaultTopologySpread：基于 Service、ReplicaSets 和 StatefulSets 分散部署。扩展点：PreScore， Score。
- ImageLocality：倾向于已经具有 Pod 运行的 image 的节点。扩展点：Score。
- TaintToleration：实施污点和宽容。扩展点：Filter，Prescore，Score。
- NodeName：Pod 配置的节点与节点是否匹配。扩展点：Filter。
- NodePorts：节点端口是否被占用。扩展点：PreFilter，Filter。
- NodePreferAvoidPods：根据节点 annotation 进行评分 scheduler.alpha.kubernetes.io/preferAvoidPods。扩展点：Score。
- NodeAffinity：节点的亲和与反亲和。扩展点：Filter，Score。
- PodTopologySpread：实现 Pod 拓扑传播。扩展点：PreFilter，Filter，PreScore，Score。
- NodeUnschedulable：过滤掉 .spec.unschedulable 为 true 的节点。扩展点：Filter。
- NodeResourcesFit：检查节点是否具有 Pod 请求的资源。扩展点：PreFilter，Filter。
- NodeResourcesBallancedAllocation：使用资源利用均衡的节点。扩展点：Score。
- NodeResourcesLeastAllocated：使用资源分配少的节点。扩展点：Score。
- VolumeBinding：节点是否具有或可以绑定所请求卷。扩展点：Filter。
- VolumeRestrictions：节点挂载的卷是否满足限制条件。扩展点：Filter。
- VolumeZone：检查卷是否满足任何区域要求。扩展点：Filter。
- NodeVolumeLimits：检查节点是否可以符合 CSI 限制。扩展点：Filter。
- EBSLimits：检查节点是否满足 AWS EBS 限制。扩展点：Filter。
- GCEPDLimits：检查节点是否满足 GCP-PD 限制。扩展点：Filter。
- AzureDiskLimits：检查节点是否满足 Azure Disk 限制。扩展点：Filter。
- InterPodAffinity：基于 Pod 亲和与反亲和。扩展点：PreFilter，Filter，PreScore，Score。
- PrioritySort：基于默认优先级的排序。扩展点：QueueSort。
- DefaultBinder：基于默认的绑定机制。扩展点：Bind。
您还可以通过组件配置API启用以下默认未启用的插件：
- NodeResourcesMostAllocated：倾向于资源分配高的节点。扩展点：Score。
- RequestedToCapacityRatio：基于节点的资源可用性比例。扩展点：Score。
- NodeResourceLimits：支持满足 Pod 资源限制的节点。扩展点：PreScore，Score。
- CinderVolume：检查节点是否可以满足 OpenStack Cinder 限制。扩展点：Filter。
- NodeLabel：根据节点的 label 进行评分。扩展点：Filter，Score。
- ServiceAffinity：检查 Service label 匹配的节点，该插件有利于在节点之间部署服务。扩展点：PreFilter，Filter，Score。
亲和 Affinity 与 反亲和 Anti-Affinity
亲和规则
requiredDuringSchedulingRequiredDuringExecution
该规则还未正式上线；
在调度期间必须满足亲和或者反亲和规则，如果不能满足规则，则 pod 不能被调度到对应的主机上;
在之后的运行过程中，系统会持续监规则是否满足。RequiredDuringSchedulingIgnoredDuringExecution
在调度期间必须满足亲和或者反亲和规则，如果不能满足规则，则 pod 不能被调度到对应的主机上;
在之后的运行过程中，系统不再检查这些规则是否满足。PreferredDuringSchedulingIgnoredDuringExecution
在调度期间尽量满足亲和或者反亲和规则，如果不能满足规则，pod 也有可能被调度到对应的主机上；
在之后的运行过程中，系统不再检查这些规则是否满足。
nodeAffinity
nodeAffinity - 节点亲和，使用场景 ：
- 将服务的所有Pod部署到指定的符合标签规则的主机上。
- 将服务的所有Pod部署到除部分主机外的其他主机上。
- 支持的操作符： In，NotIn，Exists，DoesNotExist，Gt，Lt。
节点亲和：
apiVersion: v1
kind: Pod
metadata:
name: with-node-affinity
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/e2e-az-name
operator: In
values:
- e2e-az1
- e2e-az2
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
preference:
matchExpressions:
- key: another-node-label-key
operator: In
values:
- another-node-label-value
containers:
- name: with-node-affinity
image: k8s.gcr.io/pause:2.0podAffinity 与 podAntiAffinity
注意：
Pod 间亲和与反亲和需要大量的处理，这可能会显著减慢大规模集群中的调度。我们不建议在超过数百个节点的集群中使用它们。
podAffinity - pod 亲和，使用场景 ：
- 将某一特定的 pod 部署在同一拓扑域中，不用指定具体的拓扑域。
- 为了减少关联的2个服务之间的网络延迟（或其它原因），将他们部署在同一拓扑域中。
- 支持的操作符： In，NotIn，Exists，DoesNotExist。
podAntiAffinity - pod 反亲和，使用场景 ：
- 将一个服务的 pod 分散在不同的主机或者拓扑域中，提高服务本身的稳定性。
- 为某一个 pod 提供节点的资源独占权限。
- 把可能会相互影响的 pod 分散在不同的主机上。
- 支持的操作符： In，NotIn，Exists，DoesNotExist。
pod 亲和与反亲和：
apiVersion: v1
kind: Pod
metadata:
name: with-pod-affinity
spec:
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: security
operator: In
values:
- S1
topologyKey: failure-domain.beta.kubernetes.io/zone
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: security
operator: In
values:
- S2
topologyKey: failure-domain.beta.kubernetes.io/zone
containers:
- name: with-pod-affinity
image: k8s.gcr.io/pause:2.0污点驱逐 Taint
- NoSchedule：表示k8s将不会将Pod调度到具有该污点的Node上
- PreferNoSchedule：表示k8s将尽量避免将Pod调度到具有该污点的Node上
- NoExecute：表示k8s将不会将Pod调度到具有该污点的Node上，同时会将Node上已经存在的Pod驱逐出去
删除污点，开启 master 节点 pod 调度功能。
kubectl taint node ukm03 node-role.kubernetes.io/master=:NoSchedule-
添加污点，关闭 master 节点 pod 调度功能。
kubectl taint node ukm03 node-role.kubernetes.io/master=:NoSchedule容忍 Toleration
设置了污点的节点，Pod 将在一定程度上不会被调度到节点上。 通过设置 pod 容忍(Toleration)，将 pod 调度到存在污点的Node上。通过在Pod的spec中设置tolerations字段，给Pod设置上容忍点：
tolerations:
- key: "key1"
operator: "Equal"
value: "value1"
effect: "NoSchedule"
tolerationSeconds: 3600
- key: "key1"
operator: "Equal"
value: "value1"
effect: "NoExecute"
- key: "key2"
operator: "Exists"
effect: "NoSchedule"- key、vaule、effect要与Node上设置的taint保持一致。
- 当不指定key值时，表示容忍所有的污点key；当不指定effect值时，表示容忍所有的污点作用。
- operator的值为Exists将会忽略value值。
- tolerationSeconds用于描述当Pod需要被驱逐时可以在Pod上继续保留运行的时间。
create 和 apply
kubectl create 命令可创建新资源。 因此再次运行该命令，则会抛出错误，因为资源名称在名称空间中应该是唯一的。
kubectl create -f test.yml
pod/myapp-pod created
kubectl create -f test.yml
Error from server (AlreadyExists): error when creating "pod.xml": pods "myapp-pod" already existskubectl apply 命令将配置应用于资源。 如果资源不在那里，那么它将被创建。 apply 命令可以二次运行，因为它只是应用配置。 在这种情况下，配置没有改变, 所以 pod 没有改变。
kubectl apply -f test.yml
pod/myapp-pod created
kubectl apply -f test.yml
pod/myapp-pod unchangedkubernetes 控制器
ReplicationController
- 确保在任何时候都有特定数量的 pod 副本处于运行状态；
- 类似于进程管理器， ReplicationController 监控跨多个节点的多个 pod；
- 通常缩写为 rc ，通过 replicas 控制副本的数量。
---
apiVersion: apps/v1
kind: ReplicationController
metadata:
...ReplicaSet
- ReplicaSet 是下一代的 ReplicationController；
- ReplicaSet 和 ReplicationController 的唯一区别是选择器的支持，ReplicaSet 支持新的基于集合的选择器需求，而 Replication Controller 仅支持基于相等选择器的需求。
- 通常缩写为 rs 。
---
apiVersion: apps/v1
kind: ReplicaSet
metadata:
...Deployment
- Deployment 管理和描述 Pods 和 ReplicaSets 部署方式；
- 清理较旧的、不再需要的 ReplicaSets 。
- 如果 Deployment 的当前状态不稳定，回滚到较早的 Deployment 版本。
---
apiVersion: apps/v1
kind: Deployment
metadata:
...StatefulSet
- 是用来管理有状态应用的对象；
- 稳定的、唯一的网络标识符；
- 稳定的、持久的存储；
- 有序的、优雅的部署和缩放；
- 有序的、自动的滚动更新。
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
...DaemonSet
- DaemonSet 确保全部或者某个 label 的节点上运行一个 Pod 的实例；
- 在每个节点上运行集群存储 DaemonSet，例如 glusterd、ceph；
- 在每个节点上运行日志收集 DaemonSet，例如 fluentd、logstash；
- 在每个节点上运行监控 DaemonSet，例如 Prometheus Node Exporter；
- 在每个 infra 节点上面运行一个 ingress 服务。
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
...Job
- Job负责处理任务，即仅执行一次的任务，它保证批处理任务的一个或多个Pod成功结束;
- 运行示例作业
- 编写工作规范
- 处理Pod和容器故障
- 作业终止和清理
- 自动清理完成的作业
- CronJob
---
apiVersion: batch/v1
kind: Job
metadata:
name: pi
spec:
template:
spec:
containers:
- name: pi
image: perl
command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"]
restartPolicy: Never
backoffLimit: 4CronJob
- 执行基于时间调度的任务，类似于 linux 系统的 crontab 任务。
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: hello
spec:
schedule: "*/1 * * * *"
jobTemplate:
spec:
template:
spec:
containers:
- name: hello
image: busybox
args:
- /bin/sh
- -c
- date; echo Hello from the Kubernetes cluster
restartPolicy: OnFailure备份 images
cat images.sh
!/bin/bash
imagePath="/root/images"
mkdir -p ${imagePath}
save_images() {
docker images|awk '!/REPOSITORY/ && !/<none>/ && !/docker-registry.default.svc/ && !/jenkins/{print $1,$2}' > /tmp/images.list
while read Name Version
do
fileName=$(echo $Name | awk -F\/ '{print $NF}')
if [ ! -f "${imagePath}/${fileName}_${Version}.tgz" ];then
echo "save image $Name:$Version"
docker save $Name:$Version | gzip > ${imagePath}/${fileName}_${Version}.tgz
fi
done < /tmp/images.list
}
load_images() {
for images in $(ls ${imagePath}/*.tgz)
do
echo "load image $images"
docker load < $images
done
}
push_images() {
newRegistry="https://docker-registry-default.pase-hrx-stg1.zhi-niao.com/"
docker images|awk '!/REPOSITORY/ && !/<none>/{print $1,$2}' > /tmp/images.list
while read Name Version
do
fileName=$(echo $Name | awk -F/ 'sub($1,"")')
echo "${Name}:${Version} | new tag | ${newRegistry}${fileName}:${Version}"
docker tag ${Name}:${Version} ${newRegistry}${fileName}:${Version}
docker push ${newRegistry}${fileName}:${Version}
docker rmi ${newRegistry}${fileName}:${Version}
done < /tmp/images.list
}
case $1 in
save|s)
save_images
;;
load|l)
load_images
;;
push|p)
push_images
;;
*)
echo "Usage: $0 {save|load}"
;;
esac
分发 images.sh
ansible master:worker:infra -m copy -a 'src=images.sh dest=/root/images.sh mode=0755'
ansible master:worker:infra -m shell -a './images.sh save'
for host in ukm02 ukm03 uki01 uki02 uki03 ukn01 ukn02 ukn03;do rsync -vzrtopg --progress -e ssh ${host}:/root/images/* /root/images/;done切换 namespace
kubectl config set-context --current --namespace=kube-system获取加入集群 token
kubeadm token list
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
gnc91d.aav2px3kb3021uxl 8h 2020-02-20T20:20:00+08:00 authentication,signing The default bootstrap token generated by 'kubeadm init'. system:bootstrappers:kubeadm:default-node-token获取加入集群 token hash
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
ae56bc3c9cabaaf04878be16436348419a0772342d5cc71c371fb5a79157b5a8重新生成 kubeadm-certs
- kubeadm-certs 密钥和解密密钥会在两个小时后失效
kubeadm init phase upload-certs --upload-certs
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
4379fa5fdc142d5e98d01a290b8490fc9a169e2a4c3a94695ec3745573b22fb3查看 kubernetes 证书状态
kubeadm alpha certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf Apr 22, 2021 09:03 UTC 364d no
apiserver Apr 22, 2021 09:03 UTC 364d ca no
apiserver-etcd-client Apr 22, 2021 09:03 UTC 364d etcd-ca no
apiserver-kubelet-client Apr 22, 2021 09:03 UTC 364d ca no
controller-manager.conf Apr 22, 2021 09:03 UTC 364d no
etcd-healthcheck-client Apr 22, 2021 09:03 UTC 364d etcd-ca no
etcd-peer Apr 22, 2021 09:03 UTC 364d etcd-ca no
etcd-server Apr 22, 2021 09:03 UTC 364d etcd-ca no
front-proxy-client Apr 22, 2021 09:03 UTC 364d front-proxy-ca no
scheduler.conf Apr 22, 2021 09:03 UTC 364d no
CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca Apr 14, 2030 11:18 UTC 9y no
etcd-ca Apr 14, 2030 11:18 UTC 9y no
front-proxy-ca Apr 14, 2030 11:18 UTC 9y no重新签发所有 kubernetes 证书
kubeadm alpha certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed更新证书签名
创建更新配置
cat > ca-sign.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
apiServer:
certSANs:
- "10.10.34.89"
- "10.10.34.92"
- "10.10.34.93"
- "10.10.34.94"
- "113.108.71.77"
- "kubernetes"
- "kubernetes.default"
- "kubernetes.default.svc"
- "kubernetes.default.svc.cluster"
- "kubernetes.default.svc.cluster.local"
controllerManager:
extraArgs:
cluster-signing-cert-file: /etc/kubernetes/pki/ca.crt
cluster-signing-key-file: /etc/kubernetes/pki/ca.key
EOF
更新 kubernetes 配置
kubeadm config upload from-file --config=ca-sign.yaml
确认更新配置生效
kubeadm config view
apiServer:
certSANs:
- 10.10.34.89
- 10.10.34.92
- 10.10.34.93
- 10.10.34.94
- 113.108.71.77
- kubernetes
- kubernetes.default
- kubernetes.default.svc
- kubernetes.default.svc.cluster
- kubernetes.default.svc.cluster.local
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager:
extraArgs:
cluster-signing-cert-file: /etc/kubernetes/pki/ca.crt
cluster-signing-key-file: /etc/kubernetes/pki/ca.key
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.18.2
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
scheduler: {}
删除原 apiserver 证书
rm -rf /etc/kubernetes/pki/apiserver.*
重新生成 apiserver 证书
kubeadm init phase certs apiserver --config=ca-sign.yaml
W0429 15:59:26.534066 10196 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [ukm01 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local ukm01 ukm02 ukm03 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 10.10.34.92 10.10.34.89 10.10.34.92 10.10.34.93 10.10.34.94 113.108.71.77]
确认 apiserver 证书更新情况
openssl x509 -text -noout -in /etc/kubernetes/pki/apiserver.crt
...
X509v3 extensions:
X509v3 Key Usage: critical
Digital Signature, Key Encipherment
X509v3 Extended Key Usage:
TLS Web Server Authentication
X509v3 Subject Alternative Name:
DNS:ukm01, DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster.local, DNS:ukm01, DNS:ukm02, DNS:ukm03, DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster, DNS:kubernetes.default.svc.cluster.local, IP Address:10.96.0.1, IP Address:10.10.34.92, IP Address:10.10.34.89, IP Address:10.10.34.92, IP Address:10.10.34.93, IP Address:10.10.34.94, IP Address:113.108.71.77
...
更新所有证书
kubeadm alpha certs renew all --config=ca-sign.yaml
W0429 15:50:57.792294 2944 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed
通过 api 更新所有证书，需要执行确认命令。
kubeadm alpha certs renew all --use-api --config=ca-sign.yaml &
...
[certs] Certificate request "kubeadm-cert-kubernetes-admin-8pvf8" created
...
批准更新
kubectl get csr | awk '!/Approved/ && !/NAME/{print "kubectl certificate approve "$1}' | bash
...
certificatesigningrequest.certificates.k8s.io/kubeadm-cert-kubernetes-admin-8pvf8 approved
...
查看证书更新状态
kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
kubeadm-cert-front-proxy-client-tq62p 101s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued
kubeadm-cert-kube-apiserver-etcd-client-rnnjn 2m15s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued
kubeadm-cert-kube-apiserver-fk62g 2m16s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued
kubeadm-cert-kube-apiserver-kubelet-client-5cmpv 2m11s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued
kubeadm-cert-kube-etcd-healthcheck-client-7lcw9 118s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued
kubeadm-cert-kubernetes-admin-8pvf8 3m24s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued
kubeadm-cert-system:kube-controller-manager-bgsmt 2m8s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued
kubeadm-cert-system:kube-scheduler-dhj6b 96s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued
kubeadm-cert-ukm01-2bwbg 115s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued
kubeadm-cert-ukm01-ftrgn 104s kubernetes.io/legacy-unknown kubernetes-admin Approved,Issued卸载 kubernetes 集群
先隔离再删除节点
kubectl drain uki01 uki02 uki03 ukm01 ukm02 ukm03 ukn01 ukn02 ukn03 --delete-local-data --force --ignore-daemonsets
kubectl delete node uki01 uki02 uki03 ukm01 ukm02 ukm03 ukn01 ukn02 ukn03重置节点状态
ansible ins -m shell -a 'kubeadm reset --force'
ansible ins -m shell -a 'rm -rf /etc/cni /opt/cni/ /var/lib/calico/ ~/.kube/'kubernetes 集群升级
检查 kubeadm 可用版本
apt update && apt-cache policy kubeadm升级 kubeadm
建议一台一台升级，批量操作存在无法预估的错误。
apt-mark unhold kubeadm && \
apt-get update && apt-get install -y kubeadm=1.18.0-00 && \
apt-mark hold kubeadm
或者
apt-get update && \
apt-get install -y --allow-change-held-packages kubeadm=1.18.0-00验证 kubeadm 版本
kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.0", GitCommit:"9e991415386e4cf155a24b1da15becaa390438d8", GitTreeState:"clean", BuildDate:"2020-03-25T14:56:30Z", GoVersion:"go1.13.8", Compiler:"gc", Platform:"linux/amd64"}隔离 ukm01
kubectl drain ukm01 --ignore-daemonsets
node/ukm01 cordoned
node/ukm01 drained查看可升级版本
kubeadm upgrade plan
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade] Fetching available versions to upgrade to
[upgrade/versions] Cluster version: v1.17.3
[upgrade/versions] kubeadm version: v1.18.0
[upgrade/versions] Latest stable version: v1.18.0
[upgrade/versions] Latest stable version: v1.18.0
[upgrade/versions] Latest version in the v1.17 series: v1.17.4
[upgrade/versions] Latest version in the v1.17 series: v1.17.4
Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
COMPONENT CURRENT AVAILABLE
Kubelet 9 x v1.17.3 v1.17.4
Upgrade to the latest version in the v1.17 series:
COMPONENT CURRENT AVAILABLE
API Server v1.17.3 v1.17.4
Controller Manager v1.17.3 v1.17.4
Scheduler v1.17.3 v1.17.4
Kube Proxy v1.17.3 v1.17.4
CoreDNS 1.6.5 1.6.7
Etcd 3.4.3 3.4.3-0
You can now apply the upgrade by executing the following command:
kubeadm upgrade apply v1.17.4
_____________________________________________________________________
Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
COMPONENT CURRENT AVAILABLE
Kubelet 9 x v1.17.3 v1.18.0
Upgrade to the latest stable version:
COMPONENT CURRENT AVAILABLE
API Server v1.17.3 v1.18.0
Controller Manager v1.17.3 v1.18.0
Scheduler v1.17.3 v1.18.0
Kube Proxy v1.17.3 v1.18.0
CoreDNS 1.6.5 1.6.7
Etcd 3.4.3 3.4.3-0
You can now apply the upgrade by executing the following command:
kubeadm upgrade apply v1.18.0
_____________________________________________________________________升级 kubernetes
kubeadm upgrade apply v1.18.0
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade/version] You have chosen to change the cluster version to "v1.18.0"
[upgrade/versions] Cluster version: v1.17.3
[upgrade/versions] kubeadm version: v1.18.0
[upgrade/confirm] Are you sure you want to proceed with the upgrade? [y/N]: y
[upgrade/prepull] Will prepull images for components [kube-apiserver kube-controller-manager kube-scheduler etcd]
[upgrade/prepull] Prepulling image for component etcd.
[upgrade/prepull] Prepulling image for component kube-apiserver.
[upgrade/prepull] Prepulling image for component kube-controller-manager.
[upgrade/prepull] Prepulling image for component kube-scheduler.
[apiclient] Found 0 Pods for label selector k8s-app=upgrade-prepull-etcd
[apiclient] Found 3 Pods for label selector k8s-app=upgrade-prepull-kube-controller-manager
[apiclient] Found 3 Pods for label selector k8s-app=upgrade-prepull-kube-apiserver
[apiclient] Found 0 Pods for label selector k8s-app=upgrade-prepull-kube-scheduler
[apiclient] Found 3 Pods for label selector k8s-app=upgrade-prepull-etcd
[apiclient] Found 3 Pods for label selector k8s-app=upgrade-prepull-kube-scheduler
[upgrade/prepull] Prepulled image for component etcd.
[upgrade/prepull] Prepulled image for component kube-scheduler.
[apiclient] Error getting Pods with label selector "k8s-app=upgrade-prepull-kube-apiserver" [etcdserver: request timed out]
[apiclient] Error getting Pods with label selector "k8s-app=upgrade-prepull-kube-controller-manager" [etcdserver: request timed out]
[upgrade/prepull] Prepulled image for component kube-controller-manager.
[upgrade/prepull] Prepulled image for component kube-apiserver.
[upgrade/prepull] Successfully prepulled the images for all the control plane components
[upgrade/apply] Upgrading your Static Pod-hosted control plane to version "v1.18.0"...
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-controller-manager-ukm01 hash: fdad41767b3b13675192402b9a840a5c
Static pod: kube-scheduler-ukm01 hash: 703c43ab97818f969f780a2cbf4d24b7
[upgrade/etcd] Upgrading to TLS for etcd
[upgrade/etcd] Non fatal issue encountered during upgrade: the desired etcd version for this Kubernetes version "v1.18.0" is "3.4.3-0", but the current etcd version is "3.4.3". Won't downgrade etcd, instead just continue
[upgrade/staticpods] Writing new Static Pod manifests to "/etc/kubernetes/tmp/kubeadm-upgraded-manifests739015179"
W0403 09:40:19.888673 21690 manifests.go:225] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[upgrade/staticpods] Preparing for "kube-apiserver" upgrade
[upgrade/staticpods] Renewing apiserver certificate
[upgrade/staticpods] Renewing apiserver-kubelet-client certificate
[upgrade/staticpods] Renewing front-proxy-client certificate
[upgrade/staticpods] Renewing apiserver-etcd-client certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/kube-apiserver.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2020-04-03-09-40-18/kube-apiserver.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: f9ef3245bc9735275ed5508f8bbea7e1
[apiclient] Found 3 Pods for label selector component=kube-apiserver
[upgrade/staticpods] Component "kube-apiserver" upgraded successfully!
[upgrade/staticpods] Preparing for "kube-controller-manager" upgrade
[upgrade/staticpods] Renewing controller-manager.conf certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/kube-controller-manager.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2020-04-03-09-40-18/kube-controller-manager.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
Static pod: kube-controller-manager-ukm01 hash: fdad41767b3b13675192402b9a840a5c
Static pod: kube-controller-manager-ukm01 hash: 24e6dad5d81a4c5257a7b29861a48543
[apiclient] Found 3 Pods for label selector component=kube-controller-manager
[upgrade/staticpods] Component "kube-controller-manager" upgraded successfully!
[upgrade/staticpods] Preparing for "kube-scheduler" upgrade
[upgrade/staticpods] Renewing scheduler.conf certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/kube-scheduler.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2020-04-03-09-40-18/kube-scheduler.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
Static pod: kube-scheduler-ukm01 hash: 703c43ab97818f969f780a2cbf4d24b7
Static pod: kube-scheduler-ukm01 hash: b10b96b094242086df7de02b74f10de2
[apiclient] Found 3 Pods for label selector component=kube-scheduler
[upgrade/staticpods] Component "kube-scheduler" upgraded successfully!
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.18" in namespace kube-system with the configuration for the kubelets in the cluster
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.18.0". Enjoy!
[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.取消隔离
kubectl uncordon ukm01
node/ukm01 uncordoned升级 kubectl 和 kubelet
apt-mark unhold kubelet kubectl && \
apt-get update && apt-get install -y kubelet=1.18.0-00 kubectl=1.18.0-00 && \
apt-mark hold kubelet kubectl
或者
apt-get update && \
apt-get install -y --allow-change-held-packages kubelet=1.18.0-00 kubectl=1.18.0-00升级其他master节点 kubeadm
ansible master:\!ans -m shell -a 'apt-mark unhold kubeadm && apt-get update && apt-get install -y kubeadm=1.18.0-00 && apt-mark hold kubeadm'
ansible master:\!ans -m shell -a 'apt-mark unhold kubelet kubectl && apt-get update && apt-get install -y kubelet=1.18.0-00 kubectl=1.18.0-00 && apt-mark hold kubelet kubectl'
ansible master:\!ans -m shell -a 'kubeadm version'升级 master 节点 kubernetes
kubectl drain ukm02 --ignore-daemonsets
ssh ukm02
kubeadm upgrade node
exit
kubectl uncordon ukm02升级 master 节点 kubectl 和 kubelet
ansible master -m shell -a 'apt-mark unhold kubelet kubectl && apt-get update && apt-get install -y kubelet=1.18.0-00 kubectl=1.18.0-00 && apt-mark hold kubelet kubectl'重启 master 节点 kubelet
ansible master -m shell -a 'systemctl restart kubelet'确认升级成功
kubectl get nodes
NAME STATUS ROLES AGE VERSION
uki01 Ready infra 8d v1.17.3
uki02 Ready infra 8d v1.17.3
uki03 Ready infra 8d v1.17.3
ukm01 Ready master 8d v1.18.0
ukm02 Ready master 8d v1.18.0
ukm03 Ready master 8d v1.18.0
ukn01 Ready worker 8d v1.17.3
ukn02 Ready worker 8d v1.17.3
ukn03 Ready worker 8d v1.17.3升级 infra 和 worker 节点 kubeadm
ansible infra:worker -m shell -a 'apt-mark unhold kubeadm && apt-get update && apt-get install -y kubeadm=1.18.0-00 && apt-mark hold kubeadm'升级 infra 和 worker 节点 kubernetes
kubectl drain uki01 --ignore-daemonsets
ssh uki01
kubeadm upgrade node
exit
kubectl uncordon uki01升级 infra 和 worker 节点 kubectl 和 kubelet
ansible infra:worker -m shell -a 'apt-mark unhold kubelet kubectl && apt-get update && apt-get install -y kubelet=1.18.0-00 kubectl=1.18.0-00 && apt-mark hold kubelet kubectl'重启 infra 和 worker 节点 kubelet
ansible infra:worker -m shell -a 'systemctl restart kubelet'检查 kubernetes 升级状态
kubectl get nodes
NAME STATUS ROLES AGE VERSION
uki01 Ready infra 8d v1.18.0
uki02 Ready infra 8d v1.18.0
uki03 Ready infra 8d v1.18.0
ukm01 Ready master 8d v1.18.0
ukm02 Ready master 8d v1.18.0
ukm03 Ready master 8d v1.18.0
ukn01 Ready worker 8d v1.18.0
ukn02 Ready worker 8d v1.18.0
ukn03 Ready worker 8d v1.18.0安装 router
router 是 openshift 开源的 ingress 服务，router 服务默认安装在 default namespace 下面。
配置 rbac 文件
cat > router/rbac.yaml <<EOF
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: openshift-router
namespace: default
rules:
- apiGroups: [""]
resources: ["namespaces", "services", "endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["route.openshift.io"]
resources: ["routes"]
verbs: ["list", "watch"]
- apiGroups: ["route.openshift.io"]
resources: ["routes/status"]
verbs: ["get", "patch", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: openshift-router
roleRef:
apiGroup: ""
kind: ClusterRole
name: openshift-router
subjects:
- kind: ServiceAccount
namespace: default
name: router
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: openshift-router-auth-delegator
roleRef:
apiGroup: ""
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
namespace: default
name: router
EOF配置 serviceaccount 文件
cat > router/serviceaccount.yaml <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: router
namespace: default
EOF配置 crd 文件
cat > router/crd.yaml <<EOF
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name must match the spec fields below, and be in the form: <plural>.<group>
name: routes.route.openshift.io
namespace: default
spec:
group name to use for REST API: /apis/<group>/<version>
group: route.openshift.io
list of versions supported by this CustomResourceDefinition
versions:
- name: v1
Each version can be enabled/disabled by Served flag.
served: true
One and only one version must be marked as the storage version.
storage: true
either Namespaced or Cluster
scope: Namespaced
subresources:
enable spec/status
status: {}
names:
plural name to be used in the URL: /apis/<group>/<version>/<plural>
plural: routes
singular name to be used as an alias on the CLI and for display
singular: route
kind is normally the CamelCased singular type. Your resource manifests use this.
kind: Route
additionalPrinterColumns:
- name: Host
type: string
JSONPath: .status.ingress[0].host
- name: Admitted
type: string
JSONPath: .status.ingress[0].conditions[?(@.type=="Admitted")].status
- name: Service
type: string
JSONPath: .spec.to.name
- name: TLS
type: string
JSONPath: .spec.tls.type
EOF配置 router 安装文件
cat > router/deployment.yaml <<EOF
---
apiVersion: apps/v1
kind: DaemonSet Deployment DaemonSet
metadata:
name: router
namespace: default
labels:
k8s-app: router
spec:
replicas: 3 如果是 DaemonSet，可以不设置副本数量。
selector:
matchLabels:
k8s-app: router
template:
metadata:
labels:
k8s-app: router
spec:
serviceAccountName: router
nodeSelector:
node-role.kubernetes.io/infra: "true"
containers:
- env:
- name: ROUTER_LISTEN_ADDR
value: 0.0.0.0:1936
- name: ROUTER_METRICS_TYPE
value: haproxy
- name: ROUTER_SERVICE_HTTPS_PORT
value: "443"
- name: ROUTER_SERVICE_HTTP_PORT
value: "80"
- name: ROUTER_THREADS
value: "4"
image: openshift/origin-haproxy-router:v4.0.0
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
host: localhost
path: /healthz
port: 1936
initialDelaySeconds: 10
name: router
ports:
- containerPort: 80
- containerPort: 443
- containerPort: 1936
name: stats
protocol: TCP
readinessProbe:
httpGet:
host: localhost
path: healthz/ready
port: 1936
initialDelaySeconds: 10
resources:
requests:
cpu: 100m
memory: 256Mi
hostNetwork: true
EOF安装 router 服务
kubectl apply -f router/serviceaccount.yaml
kubectl apply -f router/rbac.yaml
kubectl apply -f router/deployment.yaml
kubectl apply -f router/crd.yaml检查端口监听
load blance 配置 TCP 80/443/1936 端口转发到 infra 节点服务器。
ansible infra -m shell -a 'netstat -lntp|grep -E "80|443|1936"'校验 route 服务
安装nginx
kubectl create deployment nginx --image nginx:alpine
kubectl expose deployment nginx --port=80创建 nginx route
kubectl apply -f - <<EOF
---
apiVersion: route.openshift.io/v1
kind: Route
metadata:
name: nginx
labels:
app: nginx
spec:
host: test.mokr.cn
to:
kind: Service
name: nginx
weight: 100
wildcardPolicy: None
EOF访问验证
curl -ik http://test.mokr.cn --resolve 'test.mokr.cn:80:10.10.34.89'
HTTP/1.1 200 OK
Server: nginx/1.17.9
Date: Tue, 24 Mar 2020 07:01:43 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 03 Mar 2020 17:36:53 GMT
ETag: "5e5e95b5-264"
Accept-Ranges: bytes
Set-Cookie: 3bc5f7a8aec67b74b33e81d956f57cb9=5ffd533ac952a924e0ec8b6f1c601703; path=/; HttpOnly
Cache-control: private
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
或者
curl -ik http://10.10.34.89 -H 'Host: test.mokr.cn' -v删除 router
kubectl delete -f router.yaml安装 kubernetes dashboard 2.0
curl -L https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta4/aio/deploy/recommended.yaml -o kubernetes-dashboard.yaml签发 https 访问证书
- Can’t load /root/.rnd into RNG
cd /root
openssl rand -writerand .rnddashboard 多域名配置
mkdir -p /certs
cp /etc/ssl/openssl.cnf /certs
vim /certs/openssl.cnf
...
[ req_distinguished_name ]
countryName = Country Name (2 letter code)
countryName_default = CN
countryName_min = 2
countryName_max = 2
stateOrProvinceName = State or Province Name (full name)
stateOrProvinceName_default = GuangDong
localityName = Locality Name (eg, city)
localityName_default = ShenZhen
0.organizationName = Organization Name (eg, company)
0.organizationName_default = MOKR Ltd
organizationalUnitName = Organizational Unit Name (eg, section)
organizationalUnitName_default = OPS
commonName = Common Name (e.g. server FQDN or YOUR name)
commonName_max = 64
emailAddress = Email Address
emailAddress_max = 64
emailAddress_default = seanzhau@gmail.com
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = dashboard.mokr.cn
DNS.2 = kubernetes-dashboard.kubernetes-dashboard.svc
DNS.3 = kubernetes-dashboard.kubernetes-dashboard.svc.cluster.local
IP.1 = 10.104.71.164
IP.2 = 10.10.34.89
...签发 dashboard 密钥
openssl req -nodes -newkey rsa:2048 \
-keyout /certs/dashboard.key \
-out /certs/dashboard.csr \
-subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Mokr LTD/OU=OPS" \
-config /certs/openssl.cnf \
-extensions v3_req签发 dashboard 证书
私钥签发
openssl x509 -req -sha256 -days 365 \
-in /certs/dashboard.csr \
-signkey /certs/dashboard.key \
-out /certs/dashboard.crt \
-extfile /certs/openssl.cnf \
-extensions v3_req
ca 签发
openssl x509 -req -days 365 \
-in /certs/dashboard.csr \
-CA /etc/kubernetes/pki/ca.crt \
-CAkey /etc/kubernetes/pki/ca.key \
-CAcreateserial \
-out /certs/dashboard.crt \
-extfile /certs/openssl.cnf \
-extensions v3_req验证 dashboard 证书
openssl x509 -text -noout -in /certs/dashboard.crt
Certificate:
...
X509v3 extensions:
X509v3 Basic Constraints:
CA:FALSE
X509v3 Key Usage:
Digital Signature, Non Repudiation, Key Encipherment
X509v3 Subject Alternative Name:
DNS:dashboard.mokr.cn, DNS:kubernetes-dashboard.kubernetes-dashboard.svc, DNS:kubernetes-dashboard.kubernetes-dashboard.svc.cluster.local, IP Address:10.104.71.164, IP Address:10.10.34.89
...部署 dashboard
kubectl apply -f kubernetes-dashboard.yaml为 dashboard 配置 secret 证书
kubectl create secret generic kubernetes-dashboard-certs --from-file=/certs/dashboard -n kubernetes-dashboard
可不配置
kubectl create secret tls kubernetes-dashboard-tls --cert=/certs/dashboard/dashboard.crt --key=/certs/dashboard/dashboard.key -n kubernetes-dashboard校验 dashboard
kubectl get pods,svc -n kubernetes-dashboard
NAME READY STATUS RESTARTS AGE
pod/dashboard-metrics-scraper-566cddb686-m5dzb 1/1 Running 0 22h
pod/kubernetes-dashboard-7b5bf5d559-dbrh2 1/1 Running 0 22h
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/dashboard-metrics-scraper ClusterIP 10.107.240.229 <none> 8000/TCP 22h
service/kubernetes-dashboard ClusterIP 10.96.188.203 <none> 443/TCP 22h创建 dashboard route
kubectl apply -f - <<EOF
---
apiVersion: route.openshift.io/v1
kind: Route
metadata:
name: kubernetes-dashboard
labels:
app: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
host: dashboard.mokr.cn
tls:
insecureEdgeTerminationPolicy: Redirect ## None Allow Redirect
termination: reencrypt ## edge passthrough reencrypt
to:
kind: Service
name: kubernetes-dashboard
weight: 100
wildcardPolicy: None
EOFcurl 访问 dashboard
curl -ik https://dashboard.mokr.cn --resolve 'dashboard.mokr.cn:443:10.10.34.89'或者
curl -ik https://10.10.34.89 -H 'Host: dashboard.mokr.cn'创建 dashboard 用户
创建用户
kubectl create serviceaccount seanzhau -n kubernetes-dashboard
添加用户授权，配置集群 admin 权限
## ex: kubectl create clusterrolebinding cluster-admin --clusterrole=cluster-admin --user=user1 --user=user2 --group=group1
kubectl create clusterrolebinding dashboard-admin --clusterrole cluster-admin --serviceaccount kubernetes-dashboard:seanzhau
或者
kubectl apply -f - <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: seanzhau
namespace: kubernetes-dashboard
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: dashboard-admin
namespace: kubernetes-dashboard
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: seanzhau
namespace: kubernetes-dashboard
EOF查看 secret 信息
查看 secret 私钥或者 ca 信息
kubectl -n kubernetes-dashboard get secret -o jsonpath='{range .items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="seanzhau")].data}{.ca\.crt}{end}' | base64 -d
查看用户 token
kubectl -n kubernetes-dashboard get secret -o jsonpath='{range .items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="seanzhau")].data}{.token}{end}' | base64 -d浏览器访问
浏览器打开 https://dashboard.mokr.cn ，选择 Token ，输入获取的用户 token。
使用 kubeconfig 登录
下载 /etc/kubernetes/admin.conf 到本地，修改配置文件，在最底部添加 token 字段。
vim admin.conf
...
users:
- name: kubernetes-admin
user:
client-certificate-data: xxxxxxxxxxxxxxxxxxxxx
client-key-data: xxxxxxxxxxxxxxxxxxxxx
token: xxxxxxxxxxxxxxxxxxxxx安装 ceph 存储
ceph ansible 安装
git clone -b stable-5.0 https://github.com/ceph/ceph-ansible.git
cd ceph-ansible安装 pip
apt install -y python-pip配置 pip 源
mkdir -p ~/.pip
cat > ~/.pip/pip.conf <<EOF
[global]
index-url = https://mirrors.aliyun.com/pypi/simple/
[install]
trusted-host=mirrors.aliyun.com
EOF安装 ansible 以及依赖
pip install -r requirements.txt配置 ansible 资产文件
cat > dummy-ansible-hosts <<EOF
[mons]
10.10.34.29
10.10.34.31
10.10.34.42
[osds]
10.10.34.29
10.10.34.31
10.10.34.42
[mdss]
10.10.34.29
10.10.34.31
10.10.34.42
[rgws]
10.10.34.29
10.10.34.31
10.10.34.42
[mgrs]
10.10.34.29
10.10.34.31
10.10.34.42
[grafana-server]
10.10.34.29
[clients]
10.10.34.29
10.10.34.31
10.10.34.42
EOF配置 ceph 安装文件
docker 安装
cp site-container.yml.sample site-container.yml
二进制文件安装
cp site.yml.sample site.yml
cat > group_vars/all.yml <<EOF
---
cluster: ceph
generate_fsid: true
journal_size: 512
ceph_origin: repository
ceph_repository: community
docker 安装
ceph_docker_image: "ceph/daemon"
ceph_docker_image_tag: latest-octopus
containerized_deployment: true
ceph_docker_registry: docker.io
ceph_stable_repo: "{{ ceph_mirror }}/debian-{{ ceph_stable_release }}"
二进制安装
ceph_mirror: http://mirrors.aliyun.com/ceph
ceph_stable_key: http://mirrors.aliyun.com/ceph/keys/release.asc
ceph_stable_release: octopus
public_network: "10.10.34.0/24"
cluster_network: "10.10.34.0/24"
monitor_interface: eth0
osd_objectstore: bluestore
dmcrypt: false
devices:
- /dev/vdb
- /dev/vdc
radosgw_interface: eth0
radosgw_address: 0.0.0.0
radosgw_address_block: "10.10.34.0/24"
grafana_admin_user: seanzhau
grafana_admin_password: iPaaS2020
dashboard_enabled: True
dashboard_admin_user: seanzhau
dashboard_admin_password: iPaaS2020
EOF安装 ceph 集群
docker 安装
ansible-playbook -i dummy-ansible-hosts site-container.yml
二进制文件安装
ansible-playbook -i dummy-ansible-hosts site.yml非集群节点安装 ceph 客户端
apt install -y ceph-common配置客户端
配置 ceph.conf
cat > /etc/ceph/ceph.conf <<EOF
[global]
mon host = 10.10.34.29,10.10.34.31,10.10.34.42
EOF
从 ceph 集群复制 admin key
scp 10.10.34.29:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring查看集群状态
ceph -s
cluster:
id: 5ae77bc9-1831-42df-8ef2-956499ce685b
health: HEALTH_WARN
1 pools have too few placement groups
services:
mon: 3 daemons, quorum ceph01,ceph02,ceph03 (age 15m)
mgr: ceph01(active, since 14m), standbys: ceph02, ceph03
mds: cephfs:1 {0=ceph01=up:active} 2 up:standby
osd: 6 osds: 6 up (since 19m), 6 in (since 19m)
rgw: 3 daemons active (ceph01.rgw0, ceph02.rgw0, ceph03.rgw0)
task status:
scrub status:
mds.ceph01: idle
data:
pools: 7 pools, 121 pgs
objects: 215 objects, 11 KiB
usage: 6.1 GiB used, 2.9 TiB / 2.9 TiB avail
pgs: 121 active+clean查看磁盘状态
ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 2.92978 root default
-3 0.97659 host ceph01
0 hdd 0.48830 osd.0 up 1.00000 1.00000
3 hdd 0.48830 osd.3 up 1.00000 1.00000
-7 0.97659 host ceph02
2 hdd 0.48830 osd.2 up 1.00000 1.00000
5 hdd 0.48830 osd.5 up 1.00000 1.00000
-5 0.97659 host ceph03
1 hdd 0.48830 osd.1 up 1.00000 1.00000
4 hdd 0.48830 osd.4 up 1.00000 1.00000挂载 ceph
mount.ceph 10.10.34.29,10.10.34.31,10.10.34.42:/ /mnt/ -o name=admin,secret=admin-password secretfile=/root/admin.secret
df -h /mnt/
Filesystem Size Used Avail Use% Mounted on
10.10.34.29,10.10.34.31,10.10.34.42:/ 948G 0 948G 0% /mnt删除集群
docker 卸载
ansible-playbook -i dummy-ansible-hosts infrastructure-playbooks/purge-container-cluster.yml
二进制文件卸载
ansible-playbook -i dummy-ansible-hosts infrastructure-playbooks/purge-cluster.yml删除磁盘分区信息
RuntimeError: Unable to use device, already a member of LVM: /dev/vdb
ansible -i dummy-ansible-hosts osds -m shell -a 'wipefs -a /dev/vdb --force'配置 provisioner
下载 ceph images
docker pull quay.io/external_storage/rbd-provisioner:latest
docker pull quay.io/external_storage/cephfs-provisioner:latest创建 ceph namespace
kubectl create namespace ceph安装 cephfs
生成 cephfs admin 密码
ceph auth get-key client.admin > /tmp/key创建 cephfs secret
kubectl create secret generic cephfs-secret-admin --from-file=/tmp/key --namespace=ceph --type=kubernetes.io/cephfs创建 cephfs 安装文件
mkdir -p storageClass/cephfs重新构建 image
如果有权限问题，可以按操作重新制造 image。
cat > storageClass/cephfs/Dockerfile <<EOF
FROM quay.io/external_storage/cephfs-provisioner:latest
USER root
RUN sed -i 's|0o755|0o777|g' /usr/lib/python2.7/site-packages/ceph_volume_client.py
EOF
docker build -t quay.io/external_storage/cephfs-provisioner:new storageClass/cephfs/创建 cephfs deployment 安装文件
cat > storageClass/cephfs/deployment.yaml <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: cephfs-provisioner
namespace: ceph
spec:
replicas: 1
selector:
matchLabels:
app: cephfs-provisioner
strategy:
type: Recreate
template:
metadata:
labels:
app: cephfs-provisioner
spec:
nodeSelector:
node-role.kubernetes.io/infra: "true"
containers:
- name: cephfs-provisioner
image: "quay.io/external_storage/cephfs-provisioner:latest"
imagePullPolicy: IfNotPresent
env:
- name: PROVISIONER_NAME
value: ceph.com/cephfs
- name: PROVISIONER_SECRET_NAMESPACE
value: ceph
command:
- "/usr/local/bin/cephfs-provisioner"
args:
- "-id=cephfs-provisioner-1"
serviceAccount: cephfs-provisioner
EOF创建 cephfs serviceaccount 安装文件
cat > storageClass/cephfs/serviceaccount.yaml <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cephfs-provisioner
namespace: ceph
EOF创建 cephfs rbac 安装文件
cat > storageClass/cephfs/rbac.yaml <<EOF
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: cephfs-provisioner
namespace: ceph
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "get", "delete"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: cephfs-provisioner
namespace: ceph
subjects:
- kind: ServiceAccount
name: cephfs-provisioner
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: cephfs-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-provisioner
namespace: ceph
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
- apiGroups: [""]
resources: ["services"]
resourceNames: ["kube-dns","coredns"]
verbs: ["list", "get"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-provisioner
subjects:
- kind: ServiceAccount
name: cephfs-provisioner
namespace: ceph
roleRef:
kind: ClusterRole
name: cephfs-provisioner
apiGroup: rbac.authorization.k8s.io
EOF创建 cephfs storageclass 安装文件
cat > storageClass/cephfs/storageclass.yaml <<EOF
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: cephfs-provisioner
provisioner: ceph.com/cephfs
parameters:
monitors: "10.10.34.29:6789,10.10.34.31:6789,10.10.34.42:6789"
adminId: admin
adminSecretName: cephfs-secret-admin
adminSecretNamespace: ceph
claimRoot: /storage
EOF安装 cephfs provisioner
kubectl apply -f storageClass/cephfs/serviceaccount.yaml
kubectl apply -f storageClass/cephfs/rbac.yaml
kubectl apply -f storageClass/cephfs/deployment.yaml
kubectl apply -f storageClass/cephfs/storageclass.yaml查看 cephfs provisioner
kubectl get storageclasses cephfs-provisioner
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
cephfs-provisioner ceph.com/cephfs Delete Immediate false 17s创建 cephfs PVC
cat > storageClass/cephfs/pvc.yaml <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cephfs-test
spec:
storageClassName: cephfs-provisioner
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5Gi
EOF
kubectl apply -f storageClass/cephfs/pvc.yaml -n kube-system
kubectl apply -f storageClass/cephfs/pvc.yaml -n ceph
kubectl apply -f storageClass/cephfs/pvc.yaml -n default验证 cephfs PVC
kubectl get pvc --all-namespaces
NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ceph cephfs-test Bound pvc-ae5ff9f8-4032-4690-9104-eeb563d45a1e 5Gi RWX cephfs-provisioner 8s
default cephfs-test Bound pvc-85d47294-50a0-4f6c-b9ca-7a676cf3c34e 5Gi RWX cephfs-provisioner 5s
kube-system cephfs-test Bound pvc-759c1b48-df22-4b72-ab22-490a2f3ebd5c 5Gi RWX cephfs-provisioner 13s创建 rbd
生成 rbd admin 密码
ceph auth get-key client.admin > /tmp/key创建 admin secret
kubectl create secret generic rbd-secret-admin --from-file=/tmp/key --namespace=ceph --type=kubernetes.io/rbd创建 pool secret
ceph osd pool create kube 8 8
ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
ceph auth get-key client.kube > /tmp/key
kubectl create secret generic kube-secret --from-file=/tmp/key --namespace=ceph --type=kubernetes.io/rbd创建 rbd 目录
mkdir -p storageClass/rbd创建 rbd deployment 安装文件
cat > storageClass/rbd/deployment.yaml <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rbd-provisioner
namespace: ceph
spec:
replicas: 1
selector:
matchLabels:
app: rbd-provisioner
strategy:
type: Recreate
template:
metadata:
labels:
app: rbd-provisioner
spec:
nodeSelector:
node-role.kubernetes.io/infra: "true"
containers:
- name: rbd-provisioner
image: "quay.io/external_storage/rbd-provisioner:latest"
imagePullPolicy: IfNotPresent
env:
- name: PROVISIONER_NAME
value: ceph.com/rbd
serviceAccount: rbd-provisioner
EOF创建 rbd serviceaccount 安装文件
cat > storageClass/rbd/serviceaccount.yaml <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rbd-provisioner
namespace: ceph
EOF创建 rbd rbac 安装文件
cat > storageClass/rbd/rbac.yaml <<EOF
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: rbd-provisioner
namespace: ceph
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rbd-provisioner
namespace: ceph
subjects:
- kind: ServiceAccount
name: rbd-provisioner
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rbd-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-provisioner
namespace: ceph
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
- apiGroups: [""]
resources: ["services"]
resourceNames: ["kube-dns","coredns"]
verbs: ["list", "get"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: rbd-provisioner
subjects:
- kind: ServiceAccount
name: rbd-provisioner
namespace: ceph
roleRef:
kind: ClusterRole
name: rbd-provisioner
apiGroup: rbac.authorization.k8s.io
EOF创建 rbd storageclass 安装文件
cat > storageClass/rbd/storageclass.yaml <<EOF
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rbd-provisioner
provisioner: ceph.com/rbd
parameters:
monitors: "10.10.34.29:6789,10.10.34.31:6789,10.10.34.42:6789"
pool: kube
adminId: admin
adminSecretNamespace: ceph
adminSecretName: rbd-secret-admin
userId: kube
userSecretNamespace: ceph
userSecretName: ceph-secret
imageFormat: "2"
imageFeatures: layering
EOF安装 rbd provisioner
kubectl apply -f storageClass/rbd/serviceaccount.yaml
kubectl apply -f storageClass/rbd/rbac.yaml
kubectl apply -f storageClass/rbd/deployment.yaml
kubectl apply -f storageClass/rbd/storageclass.yaml查看 rbd provisioner
kubectl get storageclasses rbd-provisioner
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
rbd-provisioner ceph.com/rbd Delete Immediate false 7s创建 rbd PVC
cat > storageClass/rbd/pvc.yaml <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rbd-test
spec:
accessModes:
- ReadWriteOnce
storageClassName: rbd-provisioner
resources:
requests:
storage: 10Gi
EOF
kubectl apply -f storageClass/rbd/pvc.yaml -n kube-system
kubectl apply -f storageClass/rbd/pvc.yaml -n ceph
kubectl apply -f storageClass/rbd/pvc.yaml -n default验证 rbd PVC
kubectl get pvc --all-namespaces
NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ceph rbd-test Bound pvc-4d85ad48-b24c-43b6-a7c2-89aec9c661a7 10Gi RWO rbd-provisioner 19s
default rbd-test Bound pvc-9d93ccbd-b6ed-42fe-8f57-78fbff88baeb 10Gi RWO rbd-provisioner 11s
kube-system rbd-test Bound pvc-5887166f-d9b4-4d40-b419-2e37ac81901d 10Gi RWO rbd-provisioner 31s对象存储
s3 存储
配置 ceph s3 账号，保留 access_key 和 secret_key
radosgw-admin user create --uid="testuser" --display-name="test user"
{
"user_id": "testuser",
"display_name": "test user",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"subusers": [],
"keys": [
{
"user": "testuser",
"access_key": "60DD3SXN58LIRC06ILQE",
"secret_key": "zgFnP3lL3uPF3WsCeQtnESKvgPYHnW75cSJfkYeZ"
}
],
"swift_keys": [],
"caps": [],
"op_mask": "read, write, delete",
"default_placement": "",
"default_storage_class": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"temp_url_keys": [],
"type": "rgw",
"mfa_ids": []
}
安装 s3 python 库
pip install boto
创建 s3 测试文件，使用上面创建的账号。
cat > s3.py <<EOF
from boto.s3.connection import S3Connection
from boto.s3.connection import OrdinaryCallingFormat
add radosgw user in ceph cluster
radosgw-admin user create --uid=rgwuser --subuser=rgwuser:swift --display-name=s3 --access=full
conn = S3Connection(aws_access_key_id='60DD3SXN58LIRC06ILQE',
aws_secret_access_key='zgFnP3lL3uPF3WsCeQtnESKvgPYHnW75cSJfkYeZ',
host='10.10.34.29',
port=8080,
is_secure=False,
calling_format=OrdinaryCallingFormat())
create_bucket = conn.create_bucket('cephfs')
buckets_list = conn.get_all_buckets()
for bucket in buckets_list:
print('%s\t%s' % (bucket.name, bucket.creation_date))
delete_bucket = conn.delete_bucket('cephfs')
EOF
测试验证
python s3.py
cephfs 2020-04-10T08:16:15.647Zswift
创建账号
radosgw-admin subuser create --uid=testuser --subuser=testuser:swift --access=full
{
"user_id": "testuser",
"display_name": "test user",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"subusers": [
{
"id": "testuser:swift",
"permissions": "full-control"
}
],
"keys": [
{
"user": "testuser",
"access_key": "60DD3SXN58LIRC06ILQE",
"secret_key": "zgFnP3lL3uPF3WsCeQtnESKvgPYHnW75cSJfkYeZ"
}
],
"swift_keys": [
{
"user": "testuser:swift",
"secret_key": "b59Aw59eiStnhibukAm7fN3vQRtTU2BwNio2dHnl"
}
],
"caps": [],
"op_mask": "read, write, delete",
"default_placement": "",
"default_storage_class": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"temp_url_keys": [],
"type": "rgw",
"mfa_ids": []
}
重置密码
radosgw-admin key create --subuser=testuser:swift --key-type=swift --gen-secret
安装 swift 客户端
pip install python-swiftclient
测试
swift -A http://10.10.34.29:8080/auth/1.0 -U testuser:swift -K 'b59Aw59eiStnhibukAm7fN3vQRtTU2BwNio2dHnl' list安装 istio
使用 demo 安装
- 参考
https://istio.io/docs/setup/additional-setup/config-profiles/
istioctl manifest apply --set profile=demo --set values.gateways.istio-ingressgateway.type=NodePort自定义参数安装
- 参数
https://istio.io/zh/docs/reference/config/installation-options/
istioctl manifest apply \
--namespace istio-system \
--set values.global.hub=docker.io/istio \
--set values.global.tag=1.5.0 \
--set values.global.logging.level="default:info" \
--set values.global.imagePullPolicy=IfNotPresent \
--set values.global.k8sIngress.enabled=false \
--set values.global.k8sIngress.enableHttps=false \
--set values.global.proxy.image=proxyv2 \
--set values.global.proxy.clusterDomain="cluster.local" \
--set values.global.proxy.privileged=true \
--set values.global.proxy.enableCoreDump=true \
--set values.global.proxy.autoInject=enabled \
--set values.global.proxy.tracer=zipkin \
--set values.global.controlPlaneSecurityEnabled=true \
--set values.global.mtls.enabled=false \
--set values.global.outboundTrafficPolicy.mode=ALLOW_ANY \
--set values.global.sds.enabled=false \
--set values.certmanager.enabled=false \
--set values.galley.enabled=true \
--set values.gateways.enabled=true \
--set values.gateways.istio-ingressgateway.enabled=true \
--set values.gateways.istio-ingressgateway.type=NodePort NodePort, ClusterIP or LoadBalancer \
--set values.gateways.istio-egressgateway.enabled=true \
--set values.grafana.enabled=true \
--set values.istio_cni.enabled=false \
--set values.istiocoredns.enabled=false \
--set values.kiali.enabled=true \
--set values.kiali.createDemoSecret=true \
--set values.kiali.hub=docker.io/kiali \
--set values.mixer.policy.enabled=false \
--set values.mixer.policy.autoscaleEnabled=true \
--set values.nodeagent.enabled=false \
--set values.pilot.enabled=true \
--set values.pilot.sidecar=true \
--set values.prometheus.enabled=true \
--set values.security.enabled=true \
--set values.sidecarInjectorWebhook.enabled=true \
--set values.sidecarInjectorWebhook.enableNamespacesByDefault=true \
--set values.tracing.enabled=true \
--set values.tracing.jaeger.hub=docker.io/jaegertracing \
--set values.tracing.jaeger.tag=1.14查看 ingress 端口监听情况
kubectl get svc -n istio-system | grep ingress
istio-ingressgateway NodePort 10.105.16.129 <none> 15020:30796/TCP,80:32179/TCP,443:31133/TCP,15029:31338/TCP,15030:30980/TCP,15031:31675/TCP,15032:32296/TCP,31400:31821/TCP,15443:32679/TCP 30s
ansible ins -m shell -a 'netstat -lntp | grep -E "32179|31133"'
10.10.34.92 | CHANGED | rc=0 >>
tcp6 0 0 :::32179 :::* LISTEN 21780/kube-proxy
tcp6 0 0 :::31133 :::* LISTEN 21780/kube-proxy
10.10.34.94 | CHANGED | rc=0 >>
tcp6 0 0 :::32179 :::* LISTEN 3093/kube-proxy
tcp6 0 0 :::31133 :::* LISTEN 3093/kube-proxy
10.10.34.100 | CHANGED | rc=0 >>
tcp6 0 0 :::32179 :::* LISTEN 14998/kube-proxy
tcp6 0 0 :::31133 :::* LISTEN 14998/kube-proxy
10.10.34.99 | CHANGED | rc=0 >>
tcp6 0 0 :::31133 :::* LISTEN 3031/kube-proxy
tcp6 0 0 :::32179 :::* LISTEN 3031/kube-proxy
10.10.34.95 | CHANGED | rc=0 >>
tcp6 0 0 :::32179 :::* LISTEN 23912/kube-proxy
tcp6 0 0 :::31133 :::* LISTEN 23912/kube-proxy
10.10.34.96 | CHANGED | rc=0 >>
tcp6 0 0 :::32179 :::* LISTEN 26951/kube-proxy
tcp6 0 0 :::31133 :::* LISTEN 26951/kube-proxy
10.10.34.93 | CHANGED | rc=0 >>
tcp6 0 0 :::31133 :::* LISTEN 31622/kube-proxy
tcp6 0 0 :::32179 :::* LISTEN 31622/kube-proxy
10.10.34.97 | CHANGED | rc=0 >>
tcp6 0 0 :::32179 :::* LISTEN 4244/kube-proxy
tcp6 0 0 :::31133 :::* LISTEN 4244/kube-proxy
10.10.34.98 | CHANGED | rc=0 >>
tcp6 0 0 :::31133 :::* LISTEN 8077/kube-proxy
tcp6 0 0 :::32179 :::* LISTEN 8077/kube-proxy配置 load blance
- LB 80 端口的 TCP 协议请求转发到集群 node 节点的 32179 端口
- LB 443 端口的 TCP 协议请求转发到集群 node 节点的 31133 端口
应用测试
创建 namespace 并打开自动注入
kubectl create namespace book
kubectl label namespace book istio-injection=enabled创建 demo
kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml -n book
kubectl get services -n book
kubectl get pods -n book
kubectl exec -n book \
-it $(kubectl get pod -n book -l app=ratings -o jsonpath='{.items[0].metadata.name}') -c ratings \
-- curl productpage:9080/productpage | grep -o "<title>.*</title>"创建 http gateway
kubectl apply -f - <<EOF
--
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: book
namespace: book
spec:
hosts:
- "book.mokr.cn"
gateways:
- istio-system/ingressgateway
http:
- match:
- uri:
exact: /productpage
- uri:
prefix: /static
- uri:
exact: /login
- uri:
exact: /logout
- uri:
prefix: /api/v1/products
route:
- destination:
host: productpage
port:
number: 9080
EOF验证 http gateway
curl -ik http://book.mokr.cn --resolve 'book.mokr.cn:443:10.10.34.89' -v创建 https gateway
创建根证书
openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout mokr.cn.key -out mokr.cn.crt -subj "/C=CN/ST=Guangdong/L=Shenzhen/O=mokr/OU=ops/CN=mokr.cn"
创建域名 key
openssl req -out book.mokr.cn.csr -newkey rsa:2048 -nodes -keyout book.mokr.cn.key -subj "/C=CN/ST=Guangdong/L=Shenzhen/O=mokr/OU=ops/CN=book.mokr.cn"
创建域名证书
openssl x509 -req -days 365 -CA mokr.cn.crt -CAkey mokr.cn.key -set_serial 0 -in book.mokr.cn.csr -out book.mokr.cn.crt
创建 tls secret
kubectl create secret tls ingressgateway-book-certs --key book.mokr.cn.key --cert book.mokr.cn.crt -n istio-system
创建 https gateway
kubectl apply -f - <<EOF
--
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
name: book-gateway
namespace: istio-system
spec:
selector:
istio: ingressgateway
servers:
- port:
number: 80
name: http
protocol: HTTP
hosts:
- book.mokr.cn
tls:
httpsRedirect: true
- port:
number: 443
name: https
protocol: HTTPS HTTP | HTTPS | GRPC | HTTP2 | MONGO | TCP | TLS
tls:
mode: SIMPLE PASSTHROUGH | SIMPLE | MUTUAL | AUTO_PASSTHROUGH | ISTIO_MUTUAL
credentialName: ingressgateway-book-certs
hosts:
- book.mokr.cn
EOF
创建 https VirtualService
kubectl apply -f - <<EOF
--
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: book
namespace: book
spec:
hosts:
- "book.mokr.cn"
gateways:
- istio-system/book-gateway
http:
- match:
- uri:
exact: /productpage
- uri:
prefix: /static
- uri:
exact: /login
- uri:
exact: /logout
- uri:
prefix: /api/v1/products
route:
- destination:
host: productpage
port:
number: 9080
EOF查看安装情况
kubectl get gateway -n book
NAME AGE
bookinfo-gateway 23s
kubectl get virtualservice -n book
NAME GATEWAYS HOSTS AGE
bookinfo [bookinfo-gateway] [book.mokr.cn] 36s验证 https gateway
curl -ik https://book.mokr.cn --resolve 'book.mokr.cn:443:10.10.34.89' -v删除 demo
kubectl delete -f samples/bookinfo/platform/kube/bookinfo.yaml -n book
kubectl delete namespace book --force --grace-period=0删除 istioctl 集群
istioctl manifest generate | kubectl delete -f -部署 registry
创建 pvc 部署文件
cat > registry/pvc.yaml <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: registry-storage
spec:
storageClassName: cephfs-provisioner
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10Gi
EOF创建 service 部署文件
cat > registry/service.yaml <<EOF
---
apiVersion: v1
kind: Service
metadata:
name: docker-registry
namespace: default
spec:
ports:
- port: 5000
protocol: TCP
targetPort: 5000
selector:
k8s-app: docker-registry
sessionAffinity: None
type: ClusterIP
EOF创建 registry 部署文件
cat > registry/deployment.yaml <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
k8s-app: docker-registry
name: docker-registry
namespace: default
spec:
replicas: 1
selector:
matchLabels:
k8s-app: docker-registry
template:
metadata:
labels:
k8s-app: docker-registry
spec:
nodeSelector:
node-role.kubernetes.io/infra: "true"
containers:
- env:
- name: REGISTRY_HTTP_ADDR
value: :5000
- name: REGISTRY_HTTP_TLS_CERTIFICATE
value: /etc/secrets/tls.crt
- name: REGISTRY_HTTP_TLS_KEY
value: /etc/secrets/tls.key
- name: REGISTRY_STORAGE_DELETE_ENABLED
value: "true"
name: registry
image: registry:2
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5000
protocol: TCP
resources:
requests:
cpu: 100m
memory: 256Mi
volumeMounts:
- name: registry-data
mountPath: /var/lib/registry
- name: registry-certificates
mountPath: /etc/secrets
serviceAccountName: docker-registry
volumes:
- name: registry-data
persistentVolumeClaim:
claimName: registry-storage
- name: registry-certificates
secret:
defaultMode: 420
secretName: registry-certificates
EOF创建 service
kubectl apply -f registry/service.yaml
kubectl get svc docker-registry -o custom-columns=clusterIP:.spec.clusterIP配置多域名
mkdir -p registry/
cp /etc/ssl/openssl.cnf registry/
vim registry/openssl.cnf
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = docker-registry.mokr.cn
DNS.2 = docker-registry.default.svc
DNS.3 = docker-registry.default.svc.cluster.local签发 registry 密钥
openssl req -nodes -newkey rsa:2048 \
-keyout registry/docker-registry.key \
-out registry/docker-registry.csr \
-subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Mokr LTD/OU=OPS" \
-config registry/openssl.cnf \
-extensions v3_req签发 registry 证书
私钥签发
openssl x509 -req -sha256 -days 365 \
-in registry/docker-registry.csr \
-signkey registry/docker-registry.key \
-out registry/docker-registry.crt \
-extfile registry/openssl.cnf \
-extensions v3_req
ca 签发
openssl x509 -req -days 365 \
-in registry/docker-registry.csr \
-CA /etc/kubernetes/pki/ca.crt \
-CAkey /etc/kubernetes/pki/ca.key \
-CAcreateserial \
-out registry/docker-registry.crt \
-extfile registry/openssl.cnf \
-extensions v3_req验证 registry 证书
openssl x509 -text -noout -in registry/docker-registry.crt
Certificate:
...
X509v3 extensions:
X509v3 Basic Constraints:
CA:FALSE
X509v3 Key Usage:
Digital Signature, Non Repudiation, Key Encipherment
X509v3 Subject Alternative Name:
DNS:docker-registry.mokr.cn, DNS:docker-registry.default.svc, DNS:docker-registry.default.svc.cluster.local
...为 registry 配置 secret 证书
kubectl create secret tls registry-certificates --cert=registry/docker-registry.crt --key=registry/docker-registry.key创建 registry
kubectl apply -f registry/pvc.yaml
kubectl apply -f registry/deployment.yaml创建 registry route
cat > registry/docker-registry-route.yaml <<EOF
kind: Route
apiVersion: route.openshift.io/v1
metadata:
name: docker-registry
labels:
app: docker-registry
spec:
host: docker-registry.mokr.cn
tls:
insecureEdgeTerminationPolicy: Redirect ## None Allow Redirect
termination: passthrough ## edge passthrough reencrypt
to:
kind: Service
name: docker-registry
weight: 100
wildcardPolicy: None
EOF
kubectl apply -f registry/docker-registry-route.yaml配置 docker
vim /etc/docker/daemon.json
{
...
"registry-mirrors": [
"https://docker-registry.mokr.cn",
"https://docker-registry.default.svc:5000",
"https://docker-registry.default.svc.cluster.local:5000"
],
"insecure-registries": [
"docker-registry.mokr.cn",
"docker-registry.default.svc:5000",
"docker-registry.default.svc.cluster.local:5000"
],
...
}
systemctl restart docker
docker info
...
Insecure Registries:
docker-registry.default.svc.cluster.local:5000
docker-registry.default.svc:5000
docker-registry.mokr.cn
127.0.0.0/8
Registry Mirrors:
https://docker-registry.mokr.cn/
https://docker-registry.default.svc:5000/
https://docker-registry.default.svc.cluster.local:5000/
Live Restore Enabled: false下载 image 测试
docker pull busybox:latest
docker tag busybox:latest docker-registry.default.svc:5000/busybox:latest
docker tag busybox:latest docker-registry.default.svc.cluster.local:5000/busybox:latest
docker tag busybox:latest docker-registry.mokr.cn/default/busybox:latest
docker push docker-registry.default.svc:5000/busybox:latest
docker push docker-registry.default.svc.cluster.local:5000/busybox:latest
docker push docker-registry.mokr.cn/default/busybox:latest验证 registry
curl -ik https://docker-registry.mokr.cn/v2/_catalog --resolve 'docker-registry.mokr.cn:443:10.10.34.89' -v
* Added docker-registry.mokr.cn:443:10.10.34.89 to DNS cache
* Hostname docker-registry.mokr.cn was found in DNS cache
* Trying 10.10.34.89...
* TCP_NODELAY set
* Connected to docker-registry.mokr.cn (10.10.34.89) port 443 (0)
* ALPN, offering h2
* ALPN, offering http/1.1
* successfully set certificate verify locations:
* CAfile: /etc/ssl/certs/ca-certificates.crt
CApath: /etc/ssl/certs
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
* TLSv1.3 (IN), TLS handshake, Server hello (2):
* TLSv1.2 (IN), TLS handshake, Certificate (11):
* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
* TLSv1.2 (IN), TLS handshake, Server finished (14):
* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
* TLSv1.2 (OUT), TLS change cipher, Client hello (1):
* TLSv1.2 (OUT), TLS handshake, Finished (20):
* TLSv1.2 (IN), TLS handshake, Finished (20):
* SSL connection using TLSv1.2 / ECDHE-RSA-AES128-GCM-SHA256
* ALPN, server accepted to use h2
* Server certificate:
* subject: C=CN; ST=GuangDong; L=ShenZhen; O=Mokr LTD; OU=OPS
* start date: Apr 11 15:05:20 2020 GMT
* expire date: Apr 11 15:05:20 2021 GMT
* issuer: CN=kubernetes
* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
* Using HTTP2, server supports multi-use
* Connection state changed (HTTP/2 confirmed)
* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0
* Using Stream ID: 1 (easy handle 0x556a671559a0)
> GET /v2/_catalog HTTP/2
> Host: docker-registry.mokr.cn
> User-Agent: curl/7.58.0
> Accept: */*
>
* Connection state changed (MAX_CONCURRENT_STREAMS updated)!
< HTTP/2 200
HTTP/2 200
< content-type: application/json; charset=utf-8
content-type: application/json; charset=utf-8
< docker-distribution-api-version: registry/2.0
docker-distribution-api-version: registry/2.0
< x-content-type-options: nosniff
x-content-type-options: nosniff
< content-length: 47
content-length: 47
< date: Sat, 11 Apr 2020 15:57:22 GMT
date: Sat, 11 Apr 2020 15:57:22 GMT
<
{"repositories":["busybox","default/busybox"]}
* Connection 0 to host docker-registry.mokr.cn left intact清理 registry images
登录 docker-registry
kubectl exec -it $(kubectl get pods -l k8s-app=docker-registry -o jsonpath={.items[0].metadata.name}) sh
/ registry garbage-collect /etc/docker/registry/config.yml --delete-untagged=true安装部署 cert-manager
下载安装脚本
mkdir -p cert-manager
curl -L https://github.com/jetstack/cert-manager/releases/download/v0.14.1/cert-manager.yaml -o cert-manager/cert-manager.yaml
sed -i 's/namespace: "cert-manager"/namespace: kube-system/g' cert-manager/cert-manager.yaml
sed -i 's/namespace: cert-manager/namespace: kube-system/g' cert-manager/cert-manager.yaml
sed -i 's/secret: "cert-manager/secret: kube-system/g' cert-manager/cert-manager.yaml
sed -i 's/secret: cert-manager/secret: kube-system/g' cert-manager/cert-manager.yaml
注释 5878 - 5882 行，关闭
sed -i '5878,5882s/^//g' cert-manager/cert-manager.yaml执行安装
kubectl apply -f cert-manager/cert-manager.yaml检查安装
kubectl get pods -n kube-system | grep cert-manager
cert-manager-557f8d6944-zr5jh 1/1 Running 0 2m1s
cert-manager-cainjector-6999bdd97-nfsrn 1/1 Running 0 2m1s
cert-manager-webhook-c579c9f47-g6rd5 1/1 Running 0 2m1s
kubectl get crd -o custom-columns=crd-name:.metadata.name|grep cert-manager
certificaterequests.cert-manager.io
certificates.cert-manager.io
challenges.acme.cert-manager.io
clusterissuers.cert-manager.io
issuers.cert-manager.io
orders.acme.cert-manager.io本博客所有文章除特别声明外，均采用: 署名-非商业性使用-禁止演绎 4.0 国际协议，转载请保留原文链接及作者。
