kubernetes ubuntu安装部署

安装基础环境

配置系统hosts文件

 cat >> /etc/hosts <<EOF

10.10.34.92   ukm01
10.10.34.93   ukm02
10.10.34.94   ukm03
10.10.34.95   uki01
10.10.34.96   uki02
10.10.34.97   uki03
10.10.34.98   ukn01
10.10.34.99   ukn02
10.10.34.100  ukn03
EOF

配置主机名

hostnamectl set-hostname ukm01

配置sshkey

创建sshkey

 ssh-keygen -t dsa -b 1024 -C "seanzhau@mokr.cn"
Generating public/private dsa key pair.
Enter file in which to save the key (/root/.ssh/id_dsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_dsa.
Your public key has been saved in /root/.ssh/id_dsa.pub.
The key fingerprint is:
SHA256:LaO+CdmwvXlHaIPXvZf0s/XjKxPjVUdtvoHC0PCGEUs seanzhau@mokr.cn
The key's randomart image is:
+---[DSA 1024]----+
|         E+     .|
|        ..=.    +|
|         ooo  .+ |
|         ..o . .+|
|    .  .So...   =|
|     *..=oo .o.o |
|    + +o o  .o+o.|
|     o +. . .+oo+|
|      *o .   .++*|
+----[SHA256]-----+

分发key

 ssh-copy-id ukm01
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_dsa.pub"
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys

Number of key(s) added: 1
...

 ssh-copy-id ukn03
...

校验

 ssh ukm01
Welcome to Ubuntu 18.04.3 LTS (GNU/Linux 4.15.0-88-generic x86_64)

 * Documentation:  https://help.ubuntu.com
 * Management:     https://landscape.canonical.com
 * Support:        https://ubuntu.com/advantage

 System information disabled due to load higher than 4.0
...

 ssh ukn03
...

配置阿里云 repo

 mv /etc/apt/sources.list /etc/apt/sources.list.bak

 cat > /etc/apt/sources.list <<EOF
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse

deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse

EOF

安装基础包

ubuntu 18.04 默认未安装python,需要每台服务器手动安装。

apt update && apt install -y software-properties-common apt-transport-https ca-certificates gnupg2 python curl

安装 ansible

只需在 master 节点安装

apt-add-repository --yes --update ppa:ansible/ansible

apt install -y ansible

配置 ansible.cfg

 cat > /etc/ansible/ansible.cfg <<EOF
[defaults]
log_path = /var/log/ansible.log

forks = 20
host_key_checking = False
retry_files_enabled = False
deprecation_warnings = False
nocows = True
remote_user = root
roles_path = roles/
gathering = smart
fact_caching = jsonfile
fact_caching_connection = /etc/ansible/facts
fact_caching_timeout = 600
callback_whitelist = profile_tasks
inventory_ignore_extensions = secrets.py, .pyc, .cfg, .crt, .ini
timeout = 30

[inventory]
unparsed_is_failed=true

[ssh_connection]
pipelining = True
ssh_args = -o ControlMaster=auto -o ControlPersist=600s
timeout = 10
control_path = %(directory)s/%%h-%%r
EOF

配置 inventory hosts

 cat > /etc/ansible/hosts <<EOF
[ins]
10.10.34.92  hostname='ukm01'
10.10.34.93  hostname='ukm02'
10.10.34.94  hostname='ukm03'
10.10.34.95  hostname='uki01'
10.10.34.96  hostname='uki02'
10.10.34.97  hostname='uki03'
10.10.34.98  hostname='ukn01'
10.10.34.99  hostname='ukn02'
10.10.34.100 hostname='ukn03' ansible_ssh_user='root' ansible_ssh_pass='xxxxxxx'

[ans]
10.10.34.92

[master]
10.10.34.9[2:4]

[infra]
10.10.34.9[5:7]

[worker]
10.10.34.9[8:9]
10.10.34.100
EOF

验证 ansible 配置

 ansible ins -m ping
10.10.34.92 | SUCCESS => {
    "ansible_facts": {
        "discovered_interpreter_python": "/usr/bin/python"
    },
    "changed": false,
    "ping": "pong"
}
...

集群节点绑定主机名

 ansible ins:\!ans -m shell -a 'cat >> /etc/hosts <<EOF
10.10.34.92   ukm01
10.10.34.93   ukm02
10.10.34.94   ukm03
10.10.34.95   uki01
10.10.34.96   uki02
10.10.34.97   uki03
10.10.34.98   ukn01
10.10.34.99   ukn02
10.10.34.100  ukn03
EOF'

 ansible ins:\!ans -m shell -a 'cat /etc/hosts'

分发配置

分发私钥
ansible master -m copy -a 'src=/root/.ssh/id_rsa dest=/root/.ssh/id_rsa mode=0600 owner=root'

分发 ansible 配置
ansible master -m copy -a 'src=/etc/ansible/hosts dest=/etc/ansible/hosts'

ansible master -m copy -a 'src=/etc/ansible/ansible.cfg dest=/etc/ansible/ansible.cfg'

安装 Docker CE

添加 Docker GPG key

ansible ins -m shell -a 'curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -'

配置 Docker 源

  • 动态识别操作系统版本 add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
ansible ins -m shell -a 'add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu bionic stable"'

安装 Docker

ansible ins -m shell -a 'apt update && apt install -y docker-ce docker-ce-cli containerd.io'

配置 docker daemon

 cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m",
    "max-file": "5"
  },
  "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn"],
  "storage-driver": "overlay2"
}
EOF

分发 daemon.json

ansible ins -m copy -a 'src=/etc/docker/daemon.json dest=/etc/docker/daemon.json'

ansible ins -m shell -a 'mkdir -p /etc/systemd/system/docker.service.d'

重启 docker 服务

ansible ins -m shell -a 'systemctl daemon-reload'

ansible ins -m shell -a 'systemctl restart docker'

ansible ins -m shell -a 'docker info|grep -i cgroup'
Cgroup Driver: systemd

daemon.json 配置详解

 cat /etc/docker/daemon.json
{
  "authorization-plugins": [],
  "data-root": "",
  "dns": [],
  "dns-opts": [],
  "dns-search": [],
  "exec-opts": [],
  "exec-root": "",
  "experimental": false,
  "features": {},
  "storage-driver": "",
  "storage-opts": [],
  "labels": [],
  "live-restore": true,
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file":"5",
    "labels": "somelabel",
    "env": "os,customer"
  },
  "mtu": 0,
  "pidfile": "",
  "cluster-store": "",
  "cluster-store-opts": {},
  "cluster-advertise": "",
  "max-concurrent-downloads": 3,
  "max-concurrent-uploads": 5,
  "default-shm-size": "64M",
  "shutdown-timeout": 15,
  "debug": true,
  "hosts": [],
  "log-level": "",
  "tls": true,
  "tlsverify": true,
  "tlscacert": "",
  "tlscert": "",
  "tlskey": "",
  "swarm-default-advertise-addr": "",
  "api-cors-header": "",
  "selinux-enabled": false,
  "userns-remap": "",
  "group": "",
  "cgroup-parent": "",
  "default-ulimits": {
    "nofile": {
      "Name": "nofile",
      "Hard": 64000,
      "Soft": 64000
    }
  },
  "init": false,
  "init-path": "/usr/libexec/docker-init",
  "ipv6": false,
  "iptables": false,
  "ip-forward": false,
  "ip-masq": false,
  "userland-proxy": false,
  "userland-proxy-path": "/usr/libexec/docker-proxy",
  "ip": "0.0.0.0",
  "bridge": "",
  "bip": "",
  "fixed-cidr": "",
  "fixed-cidr-v6": "",
  "default-gateway": "",
  "default-gateway-v6": "",
  "icc": false,
  "raw-logs": false,
  "allow-nondistributable-artifacts": [],
  "registry-mirrors": [],
  "seccomp-profile": "",
  "insecure-registries": [],
  "no-new-privileges": false,
  "default-runtime": "runc",
  "oom-score-adjust": -500,
  "node-generic-resources": ["NVIDIA-GPU=UUID1", "NVIDIA-GPU=UUID2"],
  "runtimes": {
    "cc-runtime": {
      "path": "/usr/bin/cc-runtime"
    },
    "custom": {
      "path": "/usr/local/bin/my-runc-replacement",
      "runtimeArgs": [
        "--debug"
      ]
    }
  },
  "default-address-pools":[
    {"base":"172.80.0.0/16","size":24},
    {"base":"172.90.0.0/16","size":24}
  ]
}

格式化查看 images

  • .ID Image ID
  • .Repository Image 仓库地址和名称
  • .Tag Image 标签
  • .Digest Image hash 值信息
  • .CreatedSince Image 创建到现在的时间
  • .CreatedAt Image 创建的具体时间
  • .Size Image 大小
docker images --format "table {{ .Repository }} \t {{ .Tag }} \t {{ .ID }} \t {{ .CreatedSince }} \t {{ .CreatedAt }} \t {{ .Size }}" --digests

docker pull 过程

  • Docker 客户端发送 REPOSITORY 和 TAG(或者 DIGEST)给 Registry;
  • Registry 根据 REPOSITORY 和 TAG(或者 DIGEST)找到相应 image manifest;
  • Registry 将 manifest 返回给 Docker 客户端;
  • Docker 客户端根据 manifest 读取 image 的 digest(sha256) , sha256 就是 IMAGE ID;
  • Docker 客户端根据 ID 查找本地是否存在对应的 image;
  • 如果 image 存在则创建对应的 image REPOSITORY 和 TAG;
  • 如果 image 不存在,则继续请求 Registry 获取 image 的配置文件;
  • 根据 image 配置文件中的 diff_ids 在本地找对应的 layer 是否存在;
  • 如果 layer 存在,则返回 Already exists;
  • 如果 layer 不存在,则根据 manifest 中 layer 的 sha256 和 media type 去服务器下载对应的 layer;
  • 下载解压之后,校验 tar 包的 sha256 是否和 Image Config 中的 diff_id 一致。

image本地存放信息

 查看 image DIGEST 信息
 docker images --digests
REPOSITORY                    TAG                 DIGEST                                                                    IMAGE ID            CREATED             SIZE
busybox                       latest              sha256:a8cf7ff6367c2afa2a90acd081b484cbded349a7076e7bdf37a05279f276bc12   be5888e67be6        2 weeks ago         1.22MB

 如果是其他的文件系统的话,名字会是其他的,比如btrfs、overlay2、devicemapper等。
 cat /var/lib/docker/image/overlay2/repositories.json | python -m json.tool
{
    "Repositories": {
        "busybox": {
            "busybox:latest": "sha256:be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a",
            "busybox@sha256:a8cf7ff6367c2afa2a90acd081b484cbded349a7076e7bdf37a05279f276bc12": "sha256:be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a"
        }
    }
}

 校验 image config 的 sha256 值
 sha256sum /var/lib/docker/image/overlay2/imagedb/content/sha256/be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a
be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a  /var/lib/docker/image/overlay2/imagedb/content/sha256/be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a

 查看 image config
 cat /var/lib/docker/image/overlay2/imagedb/content/sha256/be5888e67be651f1fbb59006f0fd791b44ed3fceaa6323ab4e37d5928874345a | python -m json.tool
{
    "architecture": "amd64",
    "config": {
        "ArgsEscaped": true,
        "AttachStderr": false,
        "AttachStdin": false,
        "AttachStdout": false,
        "Cmd": [
            "sh"
        ],
        "Domainname": "",
        "Entrypoint": null,
        "Env": [
            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
        ],
        "Hostname": "",
        "Image": "sha256:b0acc7ebf5092fcdd0fe097448529147e6619bd051f03ccf25b29bcae87e783f",
        "Labels": null,
        "OnBuild": null,
        "OpenStdin": false,
        "StdinOnce": false,
        "Tty": false,
        "User": "",
        "Volumes": null,
        "WorkingDir": ""
    },
    "container": "f7e67f16a539f8bbf53aae18cdb5f8c53e6a56930e7660010d9396ae77f7acfa",
    "container_config": {
        "ArgsEscaped": true,
        "AttachStderr": false,
        "AttachStdin": false,
        "AttachStdout": false,
        "Cmd": [
            "/bin/sh",
            "-c",
            "(nop) ",
            "CMD [\"sh\"]"
        ],
        "Domainname": "",
        "Entrypoint": null,
        "Env": [
            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
        ],
        "Hostname": "f7e67f16a539",
        "Image": "sha256:b0acc7ebf5092fcdd0fe097448529147e6619bd051f03ccf25b29bcae87e783f",
        "Labels": {},
        "OnBuild": null,
        "OpenStdin": false,
        "StdinOnce": false,
        "Tty": false,
        "User": "",
        "Volumes": null,
        "WorkingDir": ""
    },
    "created": "2020-04-14T19:19:53.590635493Z",
    "docker_version": "18.09.7",
    "history": [
        {
            "created": "2020-04-14T19:19:53.444488372Z",
            "created_by": "/bin/sh -c (nop) ADD file:09a89925137e1b768ef1f0e7d1d7325eb2b4f1a0895b3aa8dfc98b0c75f3f507 in / "
        },
        {
            "created": "2020-04-14T19:19:53.590635493Z",
            "created_by": "/bin/sh -c (nop)  CMD [\"sh\"]",
            "empty_layer": true
        }
    ],
    "os": "linux",
    "rootfs": {
        "diff_ids": [
            "sha256:5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab"
        ],
        "type": "layers"
    }
}

 layer diff_id 和 digest 的对应关系
 diffid-by-digest: 存放 digest 到 diffid 的对应关系
 v2metadata-by-diffid: 存放 diffid 到 digest 的对应关系
 tree /var/lib/docker/image/overlay2/distribution/
/var/lib/docker/image/overlay2/distribution/
├── diffid-by-digest
│   └── sha256
│       ├── e2334dd9fee4b77e48a8f2d793904118a3acf26f1f2e72a3d79c6cae993e07f0
│       └── ec237e1426f0de497da8c0951d5db1d2b7f155d207cfc0df02f3fb0508a3036a
└── v2metadata-by-diffid
    └── sha256
        ├── 5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab
        └── f53db92a80829925f8ec12396ab68569d7a769d9ab94983b86dbe6b26ae3d1fa

 根据 diffid 获取 digest
~ cat /var/lib/docker/image/overlay2/distribution/v2metadata-by-diffid/sha256/5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab | python -m json.tool
[
    {
        "Digest": "sha256:e2334dd9fee4b77e48a8f2d793904118a3acf26f1f2e72a3d79c6cae993e07f0",
        "HMAC": "",
        "SourceRepository": "docker.io/library/busybox"
    }
]

 根据 digest 获取 diffid
 cat /var/lib/docker/image/overlay2/distribution/diffid-by-digest/sha256/e2334dd9fee4b77e48a8f2d793904118a3acf26f1f2e72a3d79c6cae993e07f0
sha256:5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab

 根据 digest 查看 layer 元数据
 cd /var/lib/docker/image/overlay2/layerdb/sha256/5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab/ && ls -l
total 24
-rw-r--r-- 1 root root    64 Apr 30 15:21 cache-id
-rw-r--r-- 1 root root    71 Apr 30 15:21 diff
-rw-r--r-- 1 root root     7 Apr 30 15:21 size
-rw-r--r-- 1 root root 12159 Apr 30 15:21 tar-split.json.gz

 cache-id 是 docker 下载 layer 的时候随机生成 uuid,指向真正存放 layer 文件的位置。
 cat cache-id
486d3aacd2ee026b1e3db99b23e32a9f60d0972f7224fd53657a41943c871872

 diff 文件存放 layer 的 diffid
 cat diff
sha256:5b0d2d635df829f65d0ffb45eab2c3124a470c4f385d6602bda0c21c5248bcab

 size 当前 layer 的大小,单位是字节。
 tar-split.json.gz 是 layer 的 split 文件,通过这个文件可以还原 layer 的 tar 包,在 docker save 导出 image 时会用到。

 layer 数据,image 存储数据,使用 cache-id 访问。
 cd /var/lib/docker/overlay2/486d3aacd2ee026b1e3db99b23e32a9f60d0972f7224fd53657a41943c871872
 tree -d diff/
diff/
├── bin
├── dev
├── etc
│   └── network
│       ├── if-down.d
│       ├── if-post-down.d
│       ├── if-pre-up.d
│       └── if-up.d
├── home
├── root
├── tmp
├── usr
│   └── sbin
└── var
    ├── spool
    │   └── mail
    └── www

 manifest 存储对 config 和 layer 的 sha256 + media type 描述,目的就是为了下载 config 和 layer。下载完成后,manifest 的使命就完成了,所以在本地没有存储 manifest 文件。

docker 空间管理

 查看空间
 docker system df
TYPE                TOTAL               ACTIVE              SIZE                RECLAIMABLE
Images              11                  9                   1.084GB             202.9MB (18%)
Containers          18                  12                  10.61kB             0B (0%)
Local Volumes       0                   0                   0B                  0B
Build Cache         0                   0                   0B                  0B

 清理文件系统
 docker system prune
WARNING! This will remove:
  - all stopped containers
  - all networks not used by at least one container
  - all dangling images
  - all dangling build cache

Are you sure you want to continue? [y/N]

 强制清理,忽略依赖关系。
 docker system prune -a -f

 清理所有状态为 <none> 的 image
 docker image prune
WARNING! This will remove all images without at least one container associated to them.
Are you sure you want to continue? [y/N]

 强制清理 image,忽略依赖关系。
 docker image prune -a
WARNING! This will remove all images without at least one container associated to them.
Are you sure you want to continue? [y/N]

 清理无用数据卷
 docker volume ls
 docker volume prune
WARNING! This will remove all local volumes not used by at least one container.
Are you sure you want to continue? [y/N]

 清理所有停止的容器
 docker container prune
WARNING! This will remove all stopped containers.
Are you sure you want to continue? [y/N]

配置系统

内核关闭 swap

ansible ins -m shell -a 'echo "vm.swappiness = 0" >> /etc/sysctl.conf && sysctl -p'

零时关闭 swap

ansible ins -m shell -a 'swapoff -a'

系统 fstab 关闭配置

ansible ins -m shell -a "sed -i 's/^[^#].*swap*/#&/g'  /etc/fstab && chattr +i /etc/fstab"
  • WARNING: No swap limit support
sed -i '/GRUB_CMDLINE_LINUX=/cGRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0 console=tty0 console=ttyS0,115200 cgroup_enable=memory swapaccount=1"' /etc/default/grub

ansible ins -m copy -a 'src=/etc/default/grub dest=/etc/default/grub'

ansible ins -m shell -a 'update-grub && shutdown -r +1 "reboot for turnoff swap"'

安装 kubernetes

  • 激活旧版本的防火墙配置(19年之后的操作系统需要修改)
apt install -y iptables arptables ebtables

update-alternatives --set iptables /usr/sbin/iptables-legacy

update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy

update-alternatives --set arptables /usr/sbin/arptables-legacy

update-alternatives --set ebtables /usr/sbin/ebtables-legacy

配置 kubernetes repo

批量配置
ansible ins -m shell -a 'curl -fsSL https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -'

ansible ins -m shell -a 'add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main"'

安装 kubernetes 基础工具

查看当前支持的版本信息
apt-cache policy kubeadm

ansible ins -m shell -a 'apt update && apt install -y kubelet=1.17.3-00 kubeadm=1.17.3-00 kubectl=1.17.3-00'

配置 driver

 cat > /etc/default/kubelet <<EOF
KUBELET_EXTRA_ARGS=--cgroup-driver=systemd
EOF

 cat > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf <<EOF
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cgroup-driver=systemd"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# 这是 "kubeadm init" 和 "kubeadm join" 运行时生成的文件,动态地填充 KUBELET_KUBEADM_ARGS 变量
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# 这是一个文件,用户在不得已下可以将其用作替代 kubelet args。
# 用户最好使用 .NodeRegistration.KubeletExtraArgs 对象在配置文件中替代。
# KUBELET_EXTRA_ARGS 应该从此文件中获取。
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS

EOF

ansible ins -m copy -a 'src=/etc/default/kubelet dest=/etc/default/kubelet'
ansible ins -m copy -a 'src=/etc/systemd/system/kubelet.service.d/10-kubeadm.conf dest=/etc/systemd/system/kubelet.service.d/10-kubeadm.conf'

重启 kubelet

ansible ins -m shell -a 'systemctl daemon-reload && systemctl restart kubelet'

开启 ipvs 支持

配置内核支持 IP 转发

ansible ins -m shell -a 'grep net.ipv4.ip_forward /etc/sysctl.conf && sed -i "/net.ipv4.ip_forward/c\net.ipv4.ip_forward=1" /etc/sysctl.conf || echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf'
ansible ins -m shell -a 'grep net.bridge.bridge-nf-call-iptables /etc/sysctl.conf && sed -i "/net.bridge.bridge-nf-call-iptables/c\net.bridge.bridge-nf-call-iptables=1" /etc/sysctl.conf || echo "net.bridge.bridge-nf-call-iptables=1" >> /etc/sysctl.conf'
ansible ins -m shell -a 'grep net.bridge.bridge-nf-call-ip6tables /etc/sysctl.conf && sed -i "/net.bridge.bridge-nf-call-ip6tables/c\net.bridge.bridge-nf-call-ip6tables=1" /etc/sysctl.conf || echo "net.bridge.bridge-nf-call-ip6tables=1" >> /etc/sysctl.conf'

ansible ins -m shell -a 'sysctl -p'

配置内核加载 ipvs 模块

 cat > /tmp/ipvs.modules <<EOF
!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack_ipv4"
for kernel_module in \${ipvs_modules}; do
    /sbin/modinfo -F filename \${kernel_module} > /dev/null 2>&1
    if [ $? -eq 0 ]; then
        /sbin/modprobe \${kernel_module}
    fi
done
EOF

 ansible ins -m copy -a 'src=/tmp/ipvs.modules dest=/tmp/ipvs.modules mode=0755'
 ansible ins -m shell -a '/tmp/ipvs.modules'

 验证 ipvs 支持
 ansible ins -m shell -a 'lsmod | grep ip_vs'

安装 ipvsadm

ansible ins -m yum -a 'name=ipvsadm state=present'

创建 kubernetes 集群

ukm01 上执行

  • 内网负载均衡地址 10.10.34.89
 配置文件创建,可以修改较多的参数。
 cat > kubeadm-config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 0.0.0.0
  bindPort: 8443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master=""
  kubeletExtraArgs:
    cgroup-driver: "systemd"
  ignorePreflightErrors:
  - IsPrivilegedUser
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
controlPlaneEndpoint: 10.10.34.89:8443
certificatesDir: /etc/kubernetes/pki
clusterName: shenzhen
apiServer:
  timeoutForControlPlane: 5m0s
  extraArgs:
    authorization-mode: "Node,RBAC"
  certSANs:
  - "10.10.34.89"
  - "10.10.34.92"
  - "10.10.34.93"
  - "10.10.34.94"
  - "113.108.71.77"
  - "ukm01"
  - "ukm02"
  - "ukm03"
  - "kubernetes"
  - "kubernetes.default"
  - "kubernetes.default.svc"
  - "kubernetes.default.svc.cluster"
  - "kubernetes.default.svc.cluster.local"
controllerManager:
  extraArgs:
    "node-cidr-mask-size": "20"
scheduler:
  extraArgs:
    address: "0.0.0.0"
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
    extraArgs:
      listen-client-urls: "http://10.100.0.1:2379"
    serverCertSANs:
    -  "ec2-10-100-0-1.compute-1.amazonaws.com"
    peerCertSANs:
    - "10.100.0.1"
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kubernetesVersion: v1.18.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.100.0.0/16
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
clusterCIDR:
mode: ipvs   定义代理模式: userspace、 iptables 或 ipvs 。默认使用 iptables ,大集群建议使用 ipvs。
ipvs:
  scheduler: wlc
  syncPeriod: 30s
  minSyncPeriod: 5s
  tcpTimeout: 0s
  tcpFinTimeout: 0s
  udpTimeout: 0s
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
EOF

 验证 kubeadm-config 配置
 kubeadm init --config=kubeadm-config.yaml --upload-certs

 创建集群
 kubeadm init --config=kubeadm-config.yaml --upload-certs

 参数化创建集群
 kubeadm init \
  --apiserver-bind-port 8443 \
  --control-plane-endpoint "10.10.34.89:8443" \
  --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers \
  --kubernetes-version=v1.17.3 \
  --upload-certs \
  --pod-network-cidr=10.100.0.0/16 \
  --service-cidr=10.96.0.0/12 \

...
kubeadm join 10.10.34.89:8443 --token 7nf8wp.cefgemzgn5xcdmln \
    --discovery-token-ca-cert-hash sha256:36b328e0052c7f8e7cac826d70b21bef0cdc4e0505fddc8200c52b8b01605da3 \
    --control-plane --certificate-key bd9fc1600d2db4f2b323a17b2b7be4f4234d7ed8705bc64976dcd43d929a24a8
...
kubeadm join 10.10.34.89:8443 --token 7nf8wp.cefgemzgn5xcdmln \
    --discovery-token-ca-cert-hash sha256:36b328e0052c7f8e7cac826d70b21bef0cdc4e0505fddc8200c52b8b01605da3

kubeadm init 常用参数

--apiserver-advertise-address string
API 服务器所公布的其正在监听的 IP 地址。如果未设置,则使用默认网络接口。
--apiserver-bind-port int32     默认值:6443
API 服务器绑定的端口。
--apiserver-cert-extra-sans stringSlice
用于 API Server 服务证书的可选附加主题备用名称(SAN)。可以是 IP 地址和 DNS 名称。
--cert-dir string     默认值:"/etc/kubernetes/pki"
保存和存储证书的路径。
--certificate-key string
用于加密 kubeadm-certs Secret 中的控制平面证书的密钥。
--config string
kubeadm 配置文件的路径。
--control-plane-endpoint string
为控制平面指定一个稳定的 IP 地址或 DNS 名称。
--cri-socket string
要连接的 CRI 套接字的路径。如果为空,则 kubeadm 将尝试自动检测此值;仅当安装了多个 CRI 或具有非标准 CRI 插槽时,才使用此选项。
--dry-run
不要应用任何更改;只是输出将要执行的操作。
-k, --experimental-kustomize string
用于存储 kustomize 为静态 pod 清单所提供的补丁的路径。
--feature-gates string
一组用来描述各种功能特性的键值(key=value)对。选项是:
IPv6DualStack=true|false (ALPHA - default=false)
-h, --help
init 操作的帮助命令
--ignore-preflight-errors stringSlice
错误将显示为警告的检查列表;例如:'IsPrivilegedUser,Swap'。取值为 'all' 时将忽略检查中的所有错误。
--image-repository string     默认值:"k8s.gcr.io"
选择用于拉取控制平面镜像的容器仓库
--kubernetes-version string     默认值:"stable-1"
为控制平面选择一个特定的 Kubernetes 版本。
--node-name string
指定节点的名称。
--pod-network-cidr string
指明 pod 网络可以使用的 IP 地址段。如果设置了这个参数,控制平面将会为每一个节点自动分配 CIDRs。
--service-cidr string     默认值:"10.96.0.0/12"
为服务的虚拟 IP 地址另外指定 IP 地址段
--service-dns-domain string     默认值:"cluster.local"
为服务另外指定域名,例如:"myorg.internal"。
--skip-certificate-key-print
不要打印用于加密控制平面证书的密钥。
--skip-phases stringSlice
要跳过的阶段列表
--skip-token-print
跳过打印 'kubeadm init' 生成的默认引导令牌。
--token string
这个令牌用于建立控制平面节点与工作节点间的双向通信。格式为 [a-z0-9]{6}\.[a-z0-9]{16} - 示例:abcdef.0123456789abcdef
--token-ttl duration     默认值:24h0m0s
令牌被自动删除之前的持续时间(例如 1 s,2 m,3 h)。如果设置为 '0',则令牌将永不过期
--upload-certs
将控制平面证书上传到 kubeadm-certs Secret。

master 节点扩容

在 ukm02 ukm03 上执行

ansible master:\!ans -m shell -a 'kubeadm join 10.10.34.89:8443 \
   --apiserver-bind-port 8443 \
   --control-plane \
   --token 7nf8wp.cefgemzgn5xcdmln \
   --discovery-token-ca-cert-hash sha256:36b328e0052c7f8e7cac826d70b21bef0cdc4e0505fddc8200c52b8b01605da3 \
   --control-plane --certificate-key bd9fc1600d2db4f2b323a17b2b7be4f4234d7ed8705bc64976dcd43d929a24a8'

kubeadm join 常用参数

--apiserver-advertise-address string
如果该节点托管一个新的控制平面实例,则 API 服务器将公布其正在侦听的 IP 地址。如果未设置,则使用默认网络接口。
--apiserver-bind-port int32     默认值: 6443
如果节点应该托管新的控制平面实例,则为 API 服务器要绑定的端口。
--certificate-key string
使用此密钥可以解密由 init 上传的证书 secret。
--config string
kubeadm 配置文件的路径。
--control-plane
在此节点上创建一个新的控制平面实例
--cri-socket string
要连接的 CRI 套接字的路径。如果为空,则 kubeadm 将尝试自动检测此值;仅当安装了多个 CRI 或具有非标准 CRI 插槽时,才使用此选项。
--discovery-file string
对于基于文件的发现,给出用于加载集群信息的文件或者 URL。
--discovery-token string
对于基于令牌的发现,该令牌用于验证从 API 服务器获取的集群信息。
--discovery-token-ca-cert-hash stringSlice
对基于令牌的发现,验证根 CA 公钥是否与此哈希匹配 (格式: "<type>:<value>")。
--discovery-token-unsafe-skip-ca-verification
对于基于令牌的发现,允许在未关联 --discovery-token-ca-cert-hash 参数的情况下添加节点。
-k, --experimental-kustomize string
用于存储 kustomize 为静态 pod 清单所提供的补丁的路径。
-h, --help
join 操作的帮助命令
--ignore-preflight-errors stringSlice
错误将显示为警告的检查列表;例如:'IsPrivilegedUser,Swap'。取值为 'all' 时将忽略检查中的所有错误。
--node-name string
指定节点的名称
--skip-phases stringSlice
要跳过的阶段列表
--tls-bootstrap-token string
指定在加入节点时用于临时通过 Kubernetes 控制平面进行身份验证的令牌。
--token string
如果未提供这些值,则将它们用于 discovery-token 令牌和 tls-bootstrap 令牌。

配置 master 管理命令

ansible master -m shell -a 'echo "source <(kubectl completion bash)" >> ~/.bashrc'

ansible master -m shell -a 'mkdir -p /root/.kube'

ansible master -m copy -a 'src=/etc/kubernetes/admin.conf dest=/root/.kube/config mode=0600 owner=root remote_src=yes'

验证 master

 kubectl get nodes -l node-role.kubernetes.io/master
NAME    STATUS     ROLES    AGE     VERSION
ukm01   NotReady   master   3m26s   v1.17.3
ukm02   NotReady   master   1m56s   v1.17.3
ukm03   NotReady   master   2m14s   v1.17.3

 kubectl get pods --all-namespaces
NAMESPACE     NAME                            READY   STATUS    RESTARTS   AGE
kube-system   coredns-7f9c544f75-h9hmm        0/1     Pending   0          3m55s
kube-system   coredns-7f9c544f75-k9pb7        0/1     Pending   0          3m55s
kube-system   etcd-ukm01                      1/1     Running   1          20s
kube-system   etcd-ukm02                      1/1     Running   0          40s
kube-system   etcd-ukm03                      1/1     Running   0          20s
kube-system   kube-apiserver-ukm01            1/1     Running   0          20s
kube-system   kube-apiserver-ukm02            1/1     Running   0          20s
kube-system   kube-apiserver-ukm03            1/1     Running   0          20s
kube-system   kube-controller-manager-ukm01   1/1     Running   3          5m17s
kube-system   kube-controller-manager-ukm02   1/1     Running   0          74s
kube-system   kube-controller-manager-ukm03   1/1     Running   2          3m27s
kube-system   kube-proxy-4fkkx                1/1     Running   0          3m27s
kube-system   kube-proxy-5vjpc                1/1     Running   0          2m9s
kube-system   kube-proxy-plx2x                1/1     Running   0          3m55s
kube-system   kube-scheduler-ukm01            1/1     Running   2          5m17s
kube-system   kube-scheduler-ukm02            1/1     Running   0          75s
kube-system   kube-scheduler-ukm03            1/1     Running   0          3m26s

确认 kube-proxy 启用了 ipvs 模式

 kubectl get configmaps kube-proxy -n kube-system -o yaml
...
    iptables:
      masqueradeAll: false
      masqueradeBit: null
      minSyncPeriod: 0s
      syncPeriod: 0s
    ipvs:
      excludeCIDRs: null
      minSyncPeriod: 5s
      scheduler: wlc
      strictARP: false
      syncPeriod: 30s
      tcpFinTimeout: 0s
      tcpTimeout: 0s
      udpTimeout: 0s
    kind: KubeProxyConfiguration
    metricsBindAddress: ""
    mode: ipvs
...

查看 ipvs 转发情况

 ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.96.0.1:443 wlc
  -> 10.10.34.92:8443             Masq    1      0          0
  -> 10.10.34.93:8443             Masq    1      0          0
  -> 10.10.34.94:8443             Masq    1      0          0
TCP  10.96.0.10:53 wlc
TCP  10.96.0.10:9153 wlc
UDP  10.96.0.10:53 wlc

查看网卡信息

 ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether fa:16:3e:cf:28:7f brd ff:ff:ff:ff:ff:ff
    inet 10.10.34.92/24 brd 10.10.34.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fecf:287f/64 scope link
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
    link/ether 02:42:81:5a:e2:7c brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
       valid_lft forever preferred_lft forever
    inet6 fe80::42:81ff:fe5a:e27c/64 scope link
       valid_lft forever preferred_lft forever
6: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1440 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
46: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN group default
    link/ether 6a:10:8f:81:3e:61 brd ff:ff:ff:ff:ff:ff
    inet 10.96.0.10/32 brd 10.96.0.10 scope global kube-ipvs0
       valid_lft forever preferred_lft forever
    inet 10.96.0.1/32 brd 10.96.0.1 scope global kube-ipvs0
       valid_lft forever preferred_lft forever

更新 etcd 集群配置

ansible master -m shell -a 'sed -i "/--initial-cluster=/c\    - --initial-cluster=ukm01=https://10.10.34.92:2380,ukm02=https://10.10.34.93:2380,ukm03=https://10.10.34.94:2380" /etc/kubernetes/manifests/etcd.yaml'

apiserver 默认访问本地的 etcd 节点,如果需要访问集群所有节点可以修改配置。
ansible master -m shell -a 'sed -i "/- --etcd-servers/c\    - --etcd-servers=https://10.10.34.92:2379,https://10.10.34.93:2379,https://10.10.34.94:2379" /etc/kubernetes/manifests/kube-apiserver.yaml'

查看 etcd 成员状态

成员管理命令:

  • add - 添加集群成员
  • list - 查看集群成员信息
  • promote - 提升集群成员权限
  • remove - 从集群中删除成员ID
  • update - 更新集群成员信息
 kubectl exec -n kube-system -it $(kubectl get pod  -n kube-system -l component=etcd -o jsonpath='{.items[0].metadata.name}') sh

 或者下载 etcdctl
 curl -L -O https://github.com/etcd-io/etcd/releases/download/v3.4.3/etcd-v3.4.3-linux-amd64.tar.gz
 tar xzvf etcd-v3.4.3-linux-amd64.tar.gz
 mv etcd-v3.4.3-linux-amd64/etcd* /usr/local/bin/

 ETCDCTL=3 etcdctl --write-out=table \
  --cert /etc/kubernetes/pki/etcd/peer.crt \
  --key /etc/kubernetes/pki/etcd/peer.key \
  --cacert /etc/kubernetes/pki/etcd/ca.crt \
  --endpoints https://ukm01:2379,https://ukm02:2379,https://ukm03:2379 \
  member list

+------------------+---------+-------+--------------------------+--------------------------+------------+
|        ID        | STATUS  | NAME  |        PEER ADDRS        |       CLIENT ADDRS       | IS LEARNER |
+------------------+---------+-------+--------------------------+--------------------------+------------+
| 7a212ff0c04f1e82 | started | ukm02 | https://10.10.34.93:2380 | https://10.10.34.93:2379 |      false |
| ae758756c0c6b09c | started | ukm01 | https://10.10.34.92:2380 | https://10.10.34.92:2379 |      false |
| ff90415307bdc341 | started | ukm03 | https://10.10.34.94:2380 | https://10.10.34.94:2379 |      false |
+------------------+---------+-------+--------------------------+--------------------------+------------+

验证 etcd 集群状态

 ETCDCTL=3 etcdctl --cluster --write-out=table \
  --cert /etc/kubernetes/pki/etcd/peer.crt \
  --key /etc/kubernetes/pki/etcd/peer.key \
  --cacert /etc/kubernetes/pki/etcd/ca.crt \
  --endpoints https://ukm01:2379,https://ukm02:2379,https://ukm03:2379 \
  endpoint status

+--------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|         ENDPOINT         |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+--------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.10.34.94:2379 | 4677521bc1465d7e |   3.4.3 |  1.5 MB |     false |      false |         9 |       1671 |               1671 |        |
| https://10.10.34.92:2379 | ae758756c0c6b09c |   3.4.3 |  1.6 MB |     false |      false |         9 |       1671 |               1671 |        |
| https://10.10.34.93:2379 | ff3c0a769521e2c2 |   3.4.3 |  1.5 MB |      true |      false |         9 |       1671 |               1671 |        |
+--------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

查看 etcd 节点健康状况

 ETCDCTL=3 etcdctl --cluster --write-out=table \
  --cert /etc/kubernetes/pki/etcd/peer.crt \
  --key /etc/kubernetes/pki/etcd/peer.key \
  --cacert /etc/kubernetes/pki/etcd/ca.crt \
  --endpoints https://ukm01:2379,https://ukm02:2379,https://ukm03:2379 \
  endpoint health

+--------------------------+--------+-------------+-------+
|         ENDPOINT         | HEALTH |    TOOK     | ERROR |
+--------------------------+--------+-------------+-------+
| https://10.10.34.93:2379 |   true | 18.384799ms |       |
| https://10.10.34.94:2379 |   true | 18.649393ms |       |
| https://10.10.34.92:2379 |   true | 18.831792ms |       |
+--------------------------+--------+-------------+-------+

增加 node 节点

在 uki01 uki02 uki03 ukn01 ukn02 ukn03 上执行

ansible infra:worker -m shell -a 'kubeadm join 10.10.34.89:8443 \
   --token 7nf8wp.cefgemzgn5xcdmln \
   --discovery-token-ca-cert-hash sha256:36b328e0052c7f8e7cac826d70b21bef0cdc4e0505fddc8200c52b8b01605da3'

SDN

安装 calico

curl -L https://docs.projectcalico.org/v3.11/manifests/calico.yaml -o calico.yaml

kubectl apply -f calico.yaml

查看服务状态

  • node 节点处于 Ready 状态
  • 每个节点上都有一个 pod:calico-node
 kubectl get nodes
NAME    STATUS   ROLES    AGE     VERSION
uki01   Ready    infra    5m      v1.17.3
uki02   Ready    infra    5m      v1.17.3
uki03   Ready    infra    5m      v1.17.3
ukm01   Ready    master   8m      v1.17.3
ukm02   Ready    master   8m      v1.17.3
ukm03   Ready    master   8m      v1.17.3
ukn01   Ready    worker   5m      v1.17.3
ukn02   Ready    worker   5m      v1.17.3
ukn03   Ready    worker   5m      v1.17.3

 watch kubectl get pods -n kube-system -o wide
NAME                                       READY   STATUS    RESTARTS   AGE   IP                NODE    NOMINATED NODE   READINESS GATES
...
calico-kube-controllers-5b644bc49c-7xjzr   1/1     Running   0          30s   192.168.144.138   uki01   <none>           <none>
calico-node-57rns                          1/1     Running   0          30s   10.10.34.98       ukn01   <none>           <none>
calico-node-6p8z7                          1/1     Running   0          30s   10.10.34.100      ukn03   <none>           <none>
calico-node-77jwm                          1/1     Running   0          30s   172.17.0.1        ukn02   <none>           <none>
calico-node-c4gxf                          1/1     Running   0          30s   10.10.34.95       uki01   <none>           <none>
calico-node-gdb5j                          1/1     Running   0          30s   10.10.34.96       uki02   <none>           <none>
calico-node-gmzmf                          1/1     Running   0          30s   10.10.34.93       ukm02   <none>           <none>
calico-node-jdd2f                          1/1     Running   0          30s   10.10.34.97       uki03   <none>           <none>
calico-node-mrr49                          1/1     Running   0          30s   10.10.34.92       ukm01   <none>           <none>
calico-node-w5b7r                          1/1     Running   0          30s   10.10.34.94       ukm03   <none>           <none>
...

 kubectl get ippool -o wide
NAME                  AGE
default-ipv4-ippool   2m

验证 calico

 下载 calicoctl
 curl -L -o /usr/local/bin/calicoctl https://github.com/projectcalico/calicoctl/releases/download/v3.11.2/calicoctl-linux-amd64
 chmod +x /usr/local/bin/calicoctl

 使用 kubernetes api 存储数据
 mkdir -p /etc/calico
 cat > /etc/calico/calicoctl.cfg <<EOF
apiVersion: projectcalico.org/v3
kind: CalicoAPIConfig
metadata:
spec:
  datastoreType: kubernetes
  kubeconfig: /etc/kubernetes/admin.conf   使用 admin 配置
  k8sAPIEndpoint: https://10.10.34.89:8443
  k8sCertFile: /etc/kubernetes/pki/apiserver-kubelet-client.crt
  k8sKeyFile: /etc/kubernetes/pki/apiserver-kubelet-client.key
  k8sCAFile: /etc/kubernetes/pki/ca.crt
EOF

 使用独立的 etcd 存储数据
 mkdir -p /etc/calico
 cat > /etc/calico/calicoctl.cfg <<EOF
apiVersion: projectcalico.org/v3
kind: CalicoAPIConfig
metadata:
spec:
  datastoreType: etcdv3
  etcdEndpoints: https://10.10.34.92:2379,https://10.10.34.93:2379,https://10.10.34.94:2379
  etcdKeyFile: /etc/kubernetes/pki/etcd/peer.key
  etcdCertFile: /etc/kubernetes/pki/etcd/peer.crt
  etcdCACertFile: /etc/kubernetes/pki/etcd/ca.crt
EOF

 查看 node
 calicoctl node status
Calico process is running.

IPv4 BGP status
+--------------+-------------------+-------+------------+-------------+
| PEER ADDRESS |     PEER TYPE     | STATE |   SINCE    |    INFO     |
+--------------+-------------------+-------+------------+-------------+
| 10.10.34.95  | node-to-node mesh | up    | 2020-04-11 | Established |
| 10.10.34.96  | node-to-node mesh | up    | 2020-04-11 | Established |
| 10.10.34.97  | node-to-node mesh | up    | 2020-04-11 | Established |
| 10.10.34.93  | node-to-node mesh | up    | 2020-04-11 | Established |
| 10.10.34.94  | node-to-node mesh | up    | 2020-04-11 | Established |
| 10.10.34.98  | node-to-node mesh | up    | 2020-04-11 | Established |
| 10.10.34.99  | node-to-node mesh | up    | 2020-04-11 | Established |
| 10.10.34.100 | node-to-node mesh | up    | 2020-04-11 | Established |
+--------------+-------------------+-------+------------+-------------+

IPv6 BGP status
No IPv6 peers found.


 calicoctl get node -o wide
NAME    ASN       IPV4              IPV6
uki01   (64512)   10.10.34.95/24
uki02   (64512)   10.10.34.96/24
uki03   (64512)   10.10.34.97/24
ukm01   (64512)   10.10.34.92/24
ukm02   (64512)   10.10.34.93/24
ukm03   (64512)   10.10.34.94/24
ukn01   (64512)   10.10.34.98/24
ukn02   (64512)   10.10.34.99/24
ukn03   (64512)   10.10.34.100/24

 查看 ippool
 calicoctl get ippool -o wide
NAME                  CIDR             NAT    IPIPMODE   VXLANMODE   DISABLED   SELECTOR
default-ipv4-ippool   192.168.0.0/16   true   Always     Never       false      all()

 查看 ip 状态
 calicoctl ipam show
+----------+----------------+-----------+------------+--------------+
| GROUPING |      CIDR      | IPS TOTAL | IPS IN USE |   IPS FREE   |
+----------+----------------+-----------+------------+--------------+
| IP Pool  | 192.168.0.0/16 |     65536 | 5  (0%)    | 65531 (100%) |
+----------+----------------+-----------+------------+--------------+

 查看已分配的服务节点信息
 calicoctl get workloadendpoints -o wide -n kube-system
NAMESPACE     NAME                                                          WORKLOAD                                   NODE    NETWORKS             INTERFACE         PROFILES                                                  NATS
kube-system   ukn02-k8s-calico--kube--controllers--75d56dfc47--xfdp9-eth0   calico-kube-controllers-75d56dfc47-xfdp9   ukn02   192.168.33.65/32     calif1d8798ea15   kns.kube-system,ksa.kube-system.calico-kube-controllers
kube-system   uki02-k8s-coredns--546565776c--mkx9n-eth0                     coredns-546565776c-mkx9n                   uki02   192.168.225.1/32     calib399f7859db   kns.kube-system,ksa.kube-system.coredns
kube-system   ukn03-k8s-coredns--546565776c--v9wgw-eth0                     coredns-546565776c-v9wgw                   ukn03   192.168.188.129/32   cali4fffc74a7b2   kns.kube-system,ksa.kube-system.coredns

calico 网络资源

官方文档: https://docs.projectcalico.org/reference/resources/overview

  • BGPConfiguration
---
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
  name: default
spec:
  logSeverityScreen: Info
  nodeToNodeMeshEnabled: true
  asNumber: 63400
  serviceClusterIPs:
  - cidr: 10.96.0.0/12
  serviceExternalIPs:
  - cidr: 104.244.42.129/32
  - cidr: 172.217.3.0/24
  • BGPPeer
---
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
  name: some.name
spec:
  node: rack1-host1
  peerIP: 192.168.1.1
  asNumber: 63400
  • FelixConfiguration
---
apiVersion: projectcalico.org/v3
kind: FelixConfiguration
metadata:
  name: default
spec:
  ipv6Support: false
  ipipMTU: 1400
  chainInsertMode: Append
  • GlobalNetworkPolicy
---
apiVersion: projectcalico.org/v3
kind: GlobalNetworkPolicy
metadata:
  name: allow-tcp-6379
spec:
  selector: role == 'database'
  types:
  - Ingress
  - Egress
  ingress:
  - action: Allow
    metadata:
      annotations:
        from: frontend
        to: database
    protocol: TCP
    source:
      selector: role == 'frontend'
    destination:
      ports:
      - 6379
  egress:
  - action: Allow
  • GlobalNetworkSet
---
apiVersion: projectcalico.org/v3
kind: GlobalNetworkSet
metadata:
  name: a-name-for-the-set
  labels:
    role: external-database
spec:
  nets:
  - 198.51.100.0/28
  - 203.0.113.0/24
  • HostEndpoint
---
apiVersion: projectcalico.org/v3
kind: HostEndpoint
metadata:
  name: some.name
  labels:
    type: production
spec:
  interfaceName: eth0
  node: myhost
  expectedIPs:
  - 192.168.0.1
  - 192.168.0.2
  profiles:
  - profile1
  - profile2
  ports:
  - name: some-port
    port: 1234
    protocol: TCP
  - name: another-port
    port: 5432
    protocol: UDP
  • IPPool
---
apiVersion: projectcalico.org/v3
kind: IPPool
metadata:
  name: my.ippool-1
spec:
  cidr: 10.1.0.0/16
  ipipMode: CrossSubnet
  natOutgoing: true
  disabled: false
  nodeSelector: all()
  • NetworkPolicy
---
apiVersion: projectcalico.org/v3
kind: NetworkPolicy
metadata:
  name: allow-tcp-6379
  namespace: production
spec:
  selector: role == 'database'
  types:
  - Ingress
  - Egress
  ingress:
  - action: Allow
    metadata:
      annotations:
        from: frontend
        to: database
    protocol: TCP
    source:
      selector: role == 'frontend'
    destination:
      ports:
      - 6379
  egress:
  - action: Allow
  • NetworkSet
---
apiVersion: projectcalico.org/v3
kind: NetworkSet
metadata:
  name: external-database
  namespace: staging
  labels:
    role: db
spec:
  nets:
  - 198.51.100.0/28
  - 203.0.113.0/24
  • Node
---
apiVersion: projectcalico.org/v3
kind: Node
metadata:
  name: node-hostname
spec:
  bgp:
    asNumber: 64512
    ipv4Address: 10.244.0.1/24
    ipv6Address: 2001:db8:85a3::8a2e:370:7334/120
    ipv4IPIPTunnelAddr: 192.168.0.1
  • Profile
---
apiVersion: projectcalico.org/v3
kind: Profile
metadata:
  name: dev-apps
spec:
  ingress:
  - action: Deny
    source:
      nets:
      - 10.0.20.0/24
  - action: Allow
    source:
      selector: stage == 'development'
  egress:
  - action: Allow
  labelsToApply:
    stage: development
  • WorkloadEndpoint
---
apiVersion: projectcalico.org/v3
kind: WorkloadEndpoint
metadata:
  name: node1-k8s-my--nginx--b1337a-eth0
  namespace: default
  labels:
    app: frontend
    projectcalico.org/namespace: default
    projectcalico.org/orchestrator: k8s
spec:
  node: node1
  orchestrator: k8s
  endpoint: eth0
  containerID: 1337495556942031415926535
  pod: my-nginx-b1337a
  endpoint: eth0
  interfaceName: cali0ef24ba
  mac: ca:fe:1d:52:bb:e9
  ipNetworks:
  - 192.168.0.0/32
  profiles:
  - profile1
  ports:
  - name: some-port
    port: 1234
    protocol: TCP
  - name: another-port
    port: 5432
    protocol: UDP

卸载 calico

kubectl delete -f calico.yaml

删除 calico 配置

ansible ins -m shell -a 'rm -rf /etc/cni /opt/cni/ /var/lib/calico/'

集群校验

查看集群信息

 kubectl config get-clusters
NAME
shenzhen

 kubectl cluster-info
Kubernetes master is running at https://10.10.34.89:8443
KubeDNS is running at https://10.10.34.89:8443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy

查看 namespace

 kubectl get namespaces
NAME              STATUS   AGE
default           Active   15m
kube-node-lease   Active   15m
kube-public       Active   15m
kube-system       Active   15m

查看节点状态

 kubectl get nodes
NAME    STATUS     ROLES    AGE    VERSION
uki01   NotReady   <none>   7m     v1.17.3
uki02   NotReady   <none>   7m    v1.17.3
uki03   NotReady   <none>   7m    v1.17.3
ukm01   NotReady   master   10m    v1.17.3
ukm02   NotReady   master   14m    v1.17.3
ukm03   NotReady   master   13m    v1.17.3
ukn01   NotReady   <none>   7m     v1.17.3
ukn02   NotReady   <none>   7m     v1.17.3
ukn03   NotReady   <none>   7m     v1.17.3

添加 infra 标签

 kubectl label node uki01 uki02 uki03 node-role.kubernetes.io/infra=true
node/uki01 labeled
node/uki02 labeled
node/uki03 labeled

 kubectl get nodes -l node-role.kubernetes.io/infra
NAME    STATUS      ROLES   AGE   VERSION
uki01   NotReady    infra   8m    v1.17.3
uki02   NotReady    infra   8m    v1.17.3
uki03   NotReady    infra   8m    v1.17.3

添加 worker 标签

 kubectl label node ukn01 ukn02 ukn03 node-role.kubernetes.io/worker=true
node/ukn01 labeled
node/ukn02 labeled
node/ukn03 labeled

 kubectl get nodes -l node-role.kubernetes.io/worker
NAME    STATUS      ROLES    AGE   VERSION
ukn01   NotReady    worker   8m    v1.17.3
ukn02   NotReady    worker   8m    v1.17.3
ukn03   NotReady    worker   8m    v1.17.3

查看所有节点标签

 kubectl get nodes --show-labels
NAME    STATUS     ROLES    AGE     VERSION   LABELS
uki01   NotReady   infra    10m     v1.17.3   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=uki01,kubernetes.io/os=linux,node-role.kubernetes.io/infra=true
uki02   NotReady   infra    10m     v1.17.3   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=uki02,kubernetes.io/os=linux,node-role.kubernetes.io/infra=true
uki03   NotReady   infra    10m     v1.17.3   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=uki03,kubernetes.io/os=linux,node-role.kubernetes.io/infra=true
ukm01   NotReady   master   19m     v1.17.3   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukm01,kubernetes.io/os=linux,node-role.kubernetes.io/master=
ukm02   NotReady   master   17m     v1.17.3   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukm02,kubernetes.io/os=linux,node-role.kubernetes.io/master=
ukm03   NotReady   master   15m      v1.17.3   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukm03,kubernetes.io/os=linux,node-role.kubernetes.io/master=
ukn01   NotReady   worker   10m     v1.17.3   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukn01,kubernetes.io/os=linux,node-role.kubernetes.io/worker=true
ukn02   NotReady   worker   10m     v1.17.3   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukn02,kubernetes.io/os=linux,node-role.kubernetes.io/worker=true
ukn03   NotReady   worker   10m     v1.17.3   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=ukn03,kubernetes.io/os=linux,node-role.kubernetes.io/worker=true

删除节点标签

kubectl label node uki01 uki02 uki03 node-role.kubernetes.io/infra-

kubectl 简介

基础命令:
  create        从文件或 stdin 创建资源
  expose        为 deployment、pod 创建 Service
  run           使用 image 去创建一个服务
  set           更新resource ,比如更新env环境变量,image,resources 资源限制,selector  subject等。

管理命令:
  explain       查看资源定义(文档)。如 kubectl explain replicaset
  get           最基本的对象查询命令。如 kubectl get nodes/pods/deploy/rs/ns/secret等等, 加-o wide查看详细信息,-o yaml 或-o json 输出具体格式。
  edit          编辑资源配置,完成对象的更新。
  delete        删除指定资源,支持文件名、资源名、label selector。

部署命令:
  rollout       Deployment, Daemonset的升级过程管理(查看状态status、操作历史history、暂停升级、恢复升级、回滚等)
  scale         修改Deployment, ReplicaSet, Replication Controller, Job的实例数,实现一个副本集的手工扩展
  autoscale     为Deploy, RS, RC配置自动伸缩规则(依赖heapster和hpa)

集群管理命令:
  certificate   修改证书资源
  cluster-info  查看集群信息
  top           查看资源占用率 (CPU/Memory/Storage)
  cordon        标记节点为不可调度状态
  uncordon      标记节点为可调度状态
  drain         驱逐节点上的 pod ,准备下线维护
  taint         设置节点污点隔离标记

故障诊断命令:
  describe      查看资源详情
  logs          查看 pod 内容器的日志
  attach        Attach 到 pod 内的一个容器
  exec          进入指定容器执行命令
  port-forward  为 pod 创建本地端口映射
  proxy         为 Kubernetes API server 创建代理
  cp            容器内外/容器间文件拷贝
  auth          授权校验

高级命令:
  diff          比较当前版本和要部署的版本
  apply         从文件或stdin创建/更新资源
  patch         使用strategic merge patch语法更新对象的某些字段
  replace       从文件或stdin更新资源
  wait          实验功能:等待一种或多种资源的特定条件。
  convert       在不同API版本之间转换对象定义
  kustomize     从指定目录或远程URL构建Kustomization目标。

配置命令:
  label         给资源设置label
  annotate      给资源设置annotation
  completion    获取shell自动补全脚本 (bash or zsh)

其他命令:
  alpha         Alpha中功能的命令
  api-resources 打印当前 kubernetes API 所支持的命令资源
  api-versions  打印当前 kubernetes API 所支持的 api 接口以及版本信息
  config        修改kubectl配置(kubeconfig文件),如context。
  plugin        提供用于与插件交互的插件程序。
  version       打印当前 kubernetes 的服务端和客户端版本

调度策略 Scheduling Policies

kube-scheduler 给一个 pod 做调度选择包含两个步骤:

  • 过滤 Filtering
    过滤策略会检查候选 Node 是否满足 Pod 的创建需求;
    在过滤之后得出一个 Node 列表,里面包含了所有可调度节点;
    如果为空,代表这个 Pod 不可调度。

  • 打分 Scoring
    根据当前启用的打分规则,调度器会给每一个可调度节点进行打分。

过滤策略

  • PodFitsHostPorts:检查 Pod 请求的端口是否空闲(网络协议类型)。
  • PodFitsHost:检查 Pod 是否通过其主机名指定了特定的 Node。
  • PodFitsResources:检查节点空闲资源(例如 CPU 和内存)是否满足 Pod 的要求。
  • PodMatchNodeSelector:检查 Pod label 是否与 Node label 匹配。
  • NoVolumeZoneConflict:检查 Volume 存储卷是否可用。
  • NoDiskConflict:检查节点 Disk 磁盘是否冲突。
  • MaxCSIVolumeCount:检查节点挂载的 CSI 卷是否超出最大限制。
  • CheckNodeMemoryPressure:节点正在报告内存压力,并且没有配置的异常,则不会在此处安排 Pod。
  • CheckNodePIDPressure:节点正在报告进程 PID 稀缺,并且没有配置的异常,则不会在此处安排 Pod。
  • CheckNodeDiskPressure:节点正在报告存储压力,并且没有配置的异常,则不会在此处安排Pod。
  • CheckNodeCondition:检查节点的文件系统、网络状态或者 kubelet 是否准备好运行 Pod。
  • PodToleratesNodeTaints:检查 Pod 的 Tolerates 是否可以容忍节点的 Taints。
  • CheckVolumeBinding:检查 PVC 状态是否满足 Pod 创建需求。

打分策略

  • SelectorSpreadPriority:将属于同一 StatefulSet、 ReplicaSet 或 Service 的 Pod 分散在不同的主机 。
  • InterPodAffinityPriority:根据 Pod 亲和条目打分,匹配到给定节点的条目权重相加,结果值越大的节点得分越高。
  • LeastRequestedPriority:计算Pods需要的CPU和内存在当前节点可用资源的百分比,具有最小百分比的节点就是最优。
  • MostRequestedPriority:适用动态伸缩集群环境,会优先调度pod到使用率最高的主机节点。当伸缩集群时,就会腾出空闲机器,从而可以停机处理。
  • RequestedToCapacityRatioPriority:为节点上每个资源占用比例设定得分值,给资源打分函数在打分时使用。
  • BalancedResourceAllocation:优选那些使得资源利用率更为均衡的节点。
  • NodePreferAvoidPodsPriority:根据节点是否包含 scheduler.alpha.kubernetes.io/preferAvoidPods 来计算其优先级,将两个不同 Pod 运行在不同的节点上。
  • NodeAffinityPriority:基于 PreferredDuringSchedulingIgnoredDuringExecution 来进行节点亲和调度。
  • TaintTolerationPriority:基于 Pod 中对节点的污点容忍程度进行优先级评估,这个策略能够调整待选节点的排名。
  • ImageLocalityPriority:已经拥有 Pod 需要的 image 的节点会有较高的优先级。
  • ServiceSpreadingPriority:确保归属于同一给 Service 的 Pod 调度到不同的节点上,确保节点宕机之后 Service 也具有很强容灾能力。
  • EqualPriority:将所有的 Node 设置成相同的权重为 1。
  • EvenPodsSpreadPriority:实现首选的Pod拓扑扩展约束。

调度配置 Scheduling Profiles

官方文档 https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/

每次调度一个Pod的尝试都分为两个阶段,调度周期 Scheduling Cycle 和 绑定周期 Binding Cycle。

  • 调度周期为Pod选择一个节点,并且绑定周期将该决定应用于集群。调度周期和绑定周期一起被称为“调度上下文”。
  • 调度周期是串行运行的,而绑定周期可能是同时运行的。
  • 如果确定Pod不可调度或存在内部错误,则可以中止调度或绑定周期。Pod将返回队列并重试。

扩展点 Extension points

QueueSort - 对 Pod 的待调度队列进行排序,比较两个 Pod 谁更优先调度,同一时间点只能有一个 QueueSort 插件生效。

调度周期 Scheduling Cycle

  • PreFilter - 对 Pod 条件信息进行预处理,如果 PreFilter 返回了 error 则调度过程终止。
  • Filter - 标记不符合调度条件的
  • PreScore - 对 Pod 进行预评分,为 Score 插件使用提供可共享的状态。如果PreScore插件返回错误,则调度周期将中止。
  • Score - 对符合过滤条件的节点进行计分排名,在 NormalizeScore 阶段之后,调度程序将根据权重合并所有的节点分值。
  • NormalizeScore - 在调度程序计算节点的最终排名之前修改分数。
  • Reserve - 在符合条件的节点上为 Pod 保留资源。
  • Permit - 用于阻止或者延迟 Pod 与节点的绑定。
    • approve - 批准
    • deny - 拒绝
    • wait - 等待

绑定周期 Binding Cycle

  • PreBind - 用于执行绑定 Pod 之前所需的任何工作。如果任何 PreBind 插件返回错误,则 Pod 被拒绝并返回到调度队列。
  • Bind - 用于 Pod 绑定。如果绑定插件选择处理 Pod,则会跳过其余的绑定插件。
  • PostBind - 成功绑定Pod后,将调用后绑定插件。绑定周期到此结束,可以用来清理关联的资源。

Unreserve - 如果节点为 Pod 预留了资源,Pod 又在被绑定过程中被拒绝绑定,则执行 unreserve 释放为 Pod 预留的资源。

调度插件 Scheduling plugins

默认情况下启用以下插件,实现这些扩展点中的一个或多个:

  • DefaultTopologySpread:基于 Service、ReplicaSets 和 StatefulSets 分散部署。扩展点:PreScore, Score。
  • ImageLocality:倾向于已经具有 Pod 运行的 image 的节点。扩展点:Score。
  • TaintToleration:实施污点和宽容。扩展点:Filter,Prescore,Score。
  • NodeName:Pod 配置的节点与节点是否匹配。扩展点:Filter。
  • NodePorts:节点端口是否被占用。扩展点:PreFilter,Filter。
  • NodePreferAvoidPods:根据节点 annotation 进行评分 scheduler.alpha.kubernetes.io/preferAvoidPods。扩展点:Score。
  • NodeAffinity:节点的亲和与反亲和。扩展点:Filter,Score。
  • PodTopologySpread:实现 Pod 拓扑传播。扩展点:PreFilter,Filter,PreScore,Score。
  • NodeUnschedulable:过滤掉 .spec.unschedulable 为 true 的节点。扩展点:Filter。
  • NodeResourcesFit:检查节点是否具有 Pod 请求的资源。扩展点:PreFilter,Filter。
  • NodeResourcesBallancedAllocation:使用资源利用均衡的节点。扩展点:Score。
  • NodeResourcesLeastAllocated:使用资源分配少的节点。扩展点:Score。
  • VolumeBinding:节点是否具有或可以绑定所请求卷。扩展点:Filter。
  • VolumeRestrictions:节点挂载的卷是否满足限制条件。扩展点:Filter。
  • VolumeZone:检查卷是否满足任何区域要求。扩展点:Filter。
  • NodeVolumeLimits:检查节点是否可以符合 CSI 限制。扩展点:Filter。
  • EBSLimits:检查节点是否满足 AWS EBS 限制。扩展点:Filter。
  • GCEPDLimits:检查节点是否满足 GCP-PD 限制。扩展点:Filter。
  • AzureDiskLimits:检查节点是否满足 Azure Disk 限制。扩展点:Filter。
  • InterPodAffinity:基于 Pod 亲和与反亲和。扩展点:PreFilter,Filter,PreScore,Score。
  • PrioritySort:基于默认优先级的排序。扩展点:QueueSort。
  • DefaultBinder:基于默认的绑定机制。扩展点:Bind。

您还可以通过组件配置API启用以下默认未启用的插件:

  • NodeResourcesMostAllocated:倾向于资源分配高的节点。扩展点:Score。
  • RequestedToCapacityRatio:基于节点的资源可用性比例。扩展点:Score。
  • NodeResourceLimits:支持满足 Pod 资源限制的节点。扩展点:PreScore,Score。
  • CinderVolume:检查节点是否可以满足 OpenStack Cinder 限制。扩展点:Filter。
  • NodeLabel:根据节点的 label 进行评分。扩展点:Filter,Score。
  • ServiceAffinity:检查 Service label 匹配的节点,该插件有利于在节点之间部署服务。扩展点:PreFilter,Filter,Score。

亲和 Affinity 与 反亲和 Anti-Affinity

亲和规则

  • requiredDuringSchedulingRequiredDuringExecution
    该规则还未正式上线;
    在调度期间必须满足亲和或者反亲和规则,如果不能满足规则,则 pod 不能被调度到对应的主机上;
    在之后的运行过程中,系统会持续监规则是否满足。

  • RequiredDuringSchedulingIgnoredDuringExecution
    在调度期间必须满足亲和或者反亲和规则,如果不能满足规则,则 pod 不能被调度到对应的主机上;
    在之后的运行过程中,系统不再检查这些规则是否满足。

  • PreferredDuringSchedulingIgnoredDuringExecution
    在调度期间尽量满足亲和或者反亲和规则,如果不能满足规则,pod 也有可能被调度到对应的主机上;
    在之后的运行过程中,系统不再检查这些规则是否满足。

nodeAffinity

nodeAffinity - 节点亲和,使用场景 :

  • 将服务的所有Pod部署到指定的符合标签规则的主机上。
  • 将服务的所有Pod部署到除部分主机外的其他主机上。
  • 支持的操作符: In,NotIn,Exists,DoesNotExist,Gt,Lt。

节点亲和:

apiVersion: v1
kind: Pod
metadata:
  name: with-node-affinity
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: kubernetes.io/e2e-az-name
            operator: In
            values:
            - e2e-az1
            - e2e-az2
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 1
        preference:
          matchExpressions:
          - key: another-node-label-key
            operator: In
            values:
            - another-node-label-value
  containers:
  - name: with-node-affinity
    image: k8s.gcr.io/pause:2.0

podAffinity 与 podAntiAffinity

注意:
Pod 间亲和与反亲和需要大量的处理,这可能会显著减慢大规模集群中的调度。我们不建议在超过数百个节点的集群中使用它们。

podAffinity - pod 亲和,使用场景 :

  • 将某一特定的 pod 部署在同一拓扑域中,不用指定具体的拓扑域。
  • 为了减少关联的2个服务之间的网络延迟(或其它原因),将他们部署在同一拓扑域中。
  • 支持的操作符: In,NotIn,Exists,DoesNotExist。

podAntiAffinity - pod 反亲和,使用场景 :

  • 将一个服务的 pod 分散在不同的主机或者拓扑域中,提高服务本身的稳定性。
  • 为某一个 pod 提供节点的资源独占权限。
  • 把可能会相互影响的 pod 分散在不同的主机上。
  • 支持的操作符: In,NotIn,Exists,DoesNotExist。

pod 亲和与反亲和:

apiVersion: v1
kind: Pod
metadata:
  name: with-pod-affinity
spec:
  affinity:
    podAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchExpressions:
          - key: security
            operator: In
            values:
            - S1
        topologyKey: failure-domain.beta.kubernetes.io/zone
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          labelSelector:
            matchExpressions:
            - key: security
              operator: In
              values:
              - S2
          topologyKey: failure-domain.beta.kubernetes.io/zone
  containers:
  - name: with-pod-affinity
    image: k8s.gcr.io/pause:2.0

污点驱逐 Taint

  • NoSchedule:表示k8s将不会将Pod调度到具有该污点的Node上
  • PreferNoSchedule:表示k8s将尽量避免将Pod调度到具有该污点的Node上
  • NoExecute:表示k8s将不会将Pod调度到具有该污点的Node上,同时会将Node上已经存在的Pod驱逐出去
删除污点,开启 master 节点 pod 调度功能。
kubectl taint node ukm03 node-role.kubernetes.io/master=:NoSchedule-

添加污点,关闭 master 节点 pod 调度功能。
kubectl taint node ukm03 node-role.kubernetes.io/master=:NoSchedule

容忍 Toleration

设置了污点的节点,Pod 将在一定程度上不会被调度到节点上。 通过设置 pod 容忍(Toleration),将 pod 调度到存在污点的Node上。通过在Pod的spec中设置tolerations字段,给Pod设置上容忍点:

tolerations:
- key: "key1"
  operator: "Equal"
  value: "value1"
  effect: "NoSchedule"
  tolerationSeconds: 3600
- key: "key1"
  operator: "Equal"
  value: "value1"
  effect: "NoExecute"
- key: "key2"
  operator: "Exists"
  effect: "NoSchedule"
  • key、vaule、effect要与Node上设置的taint保持一致。
  • 当不指定key值时,表示容忍所有的污点key;当不指定effect值时,表示容忍所有的污点作用。
  • operator的值为Exists将会忽略value值。
  • tolerationSeconds用于描述当Pod需要被驱逐时可以在Pod上继续保留运行的时间。

create 和 apply

kubectl create 命令可创建新资源。 因此再次运行该命令,则会抛出错误,因为资源名称在名称空间中应该是唯一的。

 kubectl create -f test.yml
pod/myapp-pod created

 kubectl create -f test.yml
Error from server (AlreadyExists): error when creating "pod.xml": pods "myapp-pod" already exists

kubectl apply 命令将配置应用于资源。 如果资源不在那里,那么它将被创建。 apply 命令可以二次运行,因为它只是应用配置。 在这种情况下,配置没有改变, 所以 pod 没有改变。

 kubectl apply -f test.yml
pod/myapp-pod created

 kubectl apply -f test.yml
pod/myapp-pod unchanged

kubernetes 控制器

ReplicationController

  • 确保在任何时候都有特定数量的 pod 副本处于运行状态;
  • 类似于进程管理器, ReplicationController 监控跨多个节点的多个 pod;
  • 通常缩写为 rc ,通过 replicas 控制副本的数量。
---
apiVersion: apps/v1
kind: ReplicationController
metadata:
...

ReplicaSet

  • ReplicaSet 是下一代的 ReplicationController;
  • ReplicaSet 和 ReplicationController 的唯一区别是选择器的支持,ReplicaSet 支持新的基于集合的选择器需求,而 Replication Controller 仅支持基于相等选择器的需求。
  • 通常缩写为 rs 。
---
apiVersion: apps/v1
kind: ReplicaSet
metadata:
...

Deployment

  • Deployment 管理和描述 Pods 和 ReplicaSets 部署方式;
  • 清理较旧的、不再需要的 ReplicaSets 。
  • 如果 Deployment 的当前状态不稳定,回滚到较早的 Deployment 版本。
---
apiVersion: apps/v1
kind: Deployment
metadata:
...

StatefulSet

  • 是用来管理有状态应用的对象;
  • 稳定的、唯一的网络标识符;
  • 稳定的、持久的存储;
  • 有序的、优雅的部署和缩放;
  • 有序的、自动的滚动更新。
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
...

DaemonSet

  • DaemonSet 确保全部或者某个 label 的节点上运行一个 Pod 的实例;
  • 在每个节点上运行集群存储 DaemonSet,例如 glusterd、ceph;
  • 在每个节点上运行日志收集 DaemonSet,例如 fluentd、logstash;
  • 在每个节点上运行监控 DaemonSet,例如 Prometheus Node Exporter;
  • 在每个 infra 节点上面运行一个 ingress 服务。
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
...

Job

  • Job负责处理任务,即仅执行一次的任务,它保证批处理任务的一个或多个Pod成功结束;
  • 运行示例作业
  • 编写工作规范
  • 处理Pod和容器故障
  • 作业终止和清理
  • 自动清理完成的作业
  • CronJob
---
apiVersion: batch/v1
kind: Job
metadata:
  name: pi
spec:
  template:
    spec:
      containers:
      - name: pi
        image: perl
        command: ["perl",  "-Mbignum=bpi", "-wle", "print bpi(2000)"]
      restartPolicy: Never
  backoffLimit: 4

CronJob

  • 执行基于时间调度的任务,类似于 linux 系统的 crontab 任务。
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: hello
spec:
  schedule: "*/1 * * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: hello
            image: busybox
            args:
            - /bin/sh
            - -c
            - date; echo Hello from the Kubernetes cluster
          restartPolicy: OnFailure

备份 images

 cat images.sh
!/bin/bash

imagePath="/root/images"
mkdir -p ${imagePath}

save_images() {
    docker images|awk '!/REPOSITORY/ && !/<none>/ && !/docker-registry.default.svc/ && !/jenkins/{print $1,$2}' > /tmp/images.list
    while read Name Version
    do
        fileName=$(echo $Name | awk -F\/ '{print $NF}')
        if [ ! -f "${imagePath}/${fileName}_${Version}.tgz" ];then
            echo "save image $Name:$Version"
            docker save $Name:$Version | gzip > ${imagePath}/${fileName}_${Version}.tgz
        fi
    done < /tmp/images.list
}

load_images() {
    for images in $(ls ${imagePath}/*.tgz)
    do
        echo "load image $images"
        docker load < $images
    done
}

push_images() {
    newRegistry="https://docker-registry-default.pase-hrx-stg1.zhi-niao.com/"

    docker images|awk '!/REPOSITORY/ && !/<none>/{print $1,$2}' > /tmp/images.list
    while read Name Version
    do
        fileName=$(echo $Name | awk -F/ 'sub($1,"")')
        echo "${Name}:${Version} | new tag | ${newRegistry}${fileName}:${Version}"
        docker tag ${Name}:${Version} ${newRegistry}${fileName}:${Version}
        docker push ${newRegistry}${fileName}:${Version}
        docker rmi ${newRegistry}${fileName}:${Version}
    done < /tmp/images.list
}

case $1 in
    save|s)
        save_images
        ;;
    load|l)
        load_images
        ;;
    push|p)
        push_images
        ;;
    *)
        echo "Usage: $0 {save|load}"
        ;;
esac

 分发 images.sh
 ansible master:worker:infra -m copy -a 'src=images.sh dest=/root/images.sh mode=0755'

 ansible master:worker:infra -m shell -a './images.sh save'
 for host in ukm02 ukm03 uki01 uki02 uki03 ukn01 ukn02 ukn03;do rsync -vzrtopg --progress -e ssh ${host}:/root/images/* /root/images/;done

切换 namespace

kubectl config set-context --current --namespace=kube-system

获取加入集群 token

 kubeadm token list
TOKEN                     TTL         EXPIRES                     USAGES                   DESCRIPTION                                                EXTRA GROUPS
gnc91d.aav2px3kb3021uxl   8h          2020-02-20T20:20:00+08:00   authentication,signing   The default bootstrap token generated by 'kubeadm init'.   system:bootstrappers:kubeadm:default-node-token

获取加入集群 token hash

 openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
ae56bc3c9cabaaf04878be16436348419a0772342d5cc71c371fb5a79157b5a8

重新生成 kubeadm-certs

  • kubeadm-certs 密钥和解密密钥会在两个小时后失效
 kubeadm init phase upload-certs --upload-certs
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
4379fa5fdc142d5e98d01a290b8490fc9a169e2a4c3a94695ec3745573b22fb3

查看 kubernetes 证书状态

 kubeadm alpha certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Apr 22, 2021 09:03 UTC   364d                                    no
apiserver                  Apr 22, 2021 09:03 UTC   364d            ca                      no
apiserver-etcd-client      Apr 22, 2021 09:03 UTC   364d            etcd-ca                 no
apiserver-kubelet-client   Apr 22, 2021 09:03 UTC   364d            ca                      no
controller-manager.conf    Apr 22, 2021 09:03 UTC   364d                                    no
etcd-healthcheck-client    Apr 22, 2021 09:03 UTC   364d            etcd-ca                 no
etcd-peer                  Apr 22, 2021 09:03 UTC   364d            etcd-ca                 no
etcd-server                Apr 22, 2021 09:03 UTC   364d            etcd-ca                 no
front-proxy-client         Apr 22, 2021 09:03 UTC   364d            front-proxy-ca          no
scheduler.conf             Apr 22, 2021 09:03 UTC   364d                                    no

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Apr 14, 2030 11:18 UTC   9y              no
etcd-ca                 Apr 14, 2030 11:18 UTC   9y              no
front-proxy-ca          Apr 14, 2030 11:18 UTC   9y              no

重新签发所有 kubernetes 证书

 kubeadm alpha certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'

certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed

更新证书签名

 创建更新配置
 cat > ca-sign.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
apiServer:
  certSANs:
  - "10.10.34.89"
  - "10.10.34.92"
  - "10.10.34.93"
  - "10.10.34.94"
  - "113.108.71.77"
  - "kubernetes"
  - "kubernetes.default"
  - "kubernetes.default.svc"
  - "kubernetes.default.svc.cluster"
  - "kubernetes.default.svc.cluster.local"
controllerManager:
  extraArgs:
    cluster-signing-cert-file: /etc/kubernetes/pki/ca.crt
    cluster-signing-key-file: /etc/kubernetes/pki/ca.key
EOF

 更新 kubernetes 配置
 kubeadm config upload from-file --config=ca-sign.yaml

 确认更新配置生效
 kubeadm config view
apiServer:
  certSANs:
  - 10.10.34.89
  - 10.10.34.92
  - 10.10.34.93
  - 10.10.34.94
  - 113.108.71.77
  - kubernetes
  - kubernetes.default
  - kubernetes.default.svc
  - kubernetes.default.svc.cluster
  - kubernetes.default.svc.cluster.local
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager:
  extraArgs:
    cluster-signing-cert-file: /etc/kubernetes/pki/ca.crt
    cluster-signing-key-file: /etc/kubernetes/pki/ca.key
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.18.2
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
scheduler: {}

 删除原 apiserver 证书
 rm -rf /etc/kubernetes/pki/apiserver.*

 重新生成 apiserver 证书
 kubeadm init phase certs apiserver --config=ca-sign.yaml
W0429 15:59:26.534066   10196 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [ukm01 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local ukm01 ukm02 ukm03 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 10.10.34.92 10.10.34.89 10.10.34.92 10.10.34.93 10.10.34.94 113.108.71.77]


 确认 apiserver 证书更新情况
 openssl x509 -text -noout -in /etc/kubernetes/pki/apiserver.crt
...
        X509v3 extensions:
            X509v3 Key Usage: critical
                Digital Signature, Key Encipherment
            X509v3 Extended Key Usage:
                TLS Web Server Authentication
            X509v3 Subject Alternative Name:
                DNS:ukm01, DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster.local, DNS:ukm01, DNS:ukm02, DNS:ukm03, DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster, DNS:kubernetes.default.svc.cluster.local, IP Address:10.96.0.1, IP Address:10.10.34.92, IP Address:10.10.34.89, IP Address:10.10.34.92, IP Address:10.10.34.93, IP Address:10.10.34.94, IP Address:113.108.71.77
...

 更新所有证书
 kubeadm alpha certs renew all --config=ca-sign.yaml
W0429 15:50:57.792294    2944 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed

 通过 api 更新所有证书,需要执行确认命令。
 kubeadm alpha certs renew all --use-api --config=ca-sign.yaml &
...
[certs] Certificate request "kubeadm-cert-kubernetes-admin-8pvf8" created
...

 批准更新
 kubectl get csr | awk '!/Approved/ && !/NAME/{print "kubectl certificate approve "$1}' | bash
...
certificatesigningrequest.certificates.k8s.io/kubeadm-cert-kubernetes-admin-8pvf8 approved
...

 查看证书更新状态
 kubectl get csr
NAME                                                AGE     SIGNERNAME                     REQUESTOR          CONDITION
kubeadm-cert-front-proxy-client-tq62p               101s    kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued
kubeadm-cert-kube-apiserver-etcd-client-rnnjn       2m15s   kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued
kubeadm-cert-kube-apiserver-fk62g                   2m16s   kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued
kubeadm-cert-kube-apiserver-kubelet-client-5cmpv    2m11s   kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued
kubeadm-cert-kube-etcd-healthcheck-client-7lcw9     118s    kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued
kubeadm-cert-kubernetes-admin-8pvf8                 3m24s   kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued
kubeadm-cert-system:kube-controller-manager-bgsmt   2m8s    kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued
kubeadm-cert-system:kube-scheduler-dhj6b            96s     kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued
kubeadm-cert-ukm01-2bwbg                            115s    kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued
kubeadm-cert-ukm01-ftrgn                            104s    kubernetes.io/legacy-unknown   kubernetes-admin   Approved,Issued

卸载 kubernetes 集群

先隔离再删除节点

kubectl drain uki01 uki02 uki03 ukm01 ukm02 ukm03 ukn01 ukn02 ukn03 --delete-local-data --force --ignore-daemonsets

kubectl delete node uki01 uki02 uki03 ukm01 ukm02 ukm03 ukn01 ukn02 ukn03

重置节点状态

ansible ins -m shell -a 'kubeadm reset --force'

ansible ins -m shell -a 'rm -rf /etc/cni /opt/cni/ /var/lib/calico/ ~/.kube/'

kubernetes 集群升级

检查 kubeadm 可用版本

apt update && apt-cache policy kubeadm

升级 kubeadm

建议一台一台升级,批量操作存在无法预估的错误。

apt-mark unhold kubeadm && \
 apt-get update && apt-get install -y kubeadm=1.18.0-00 && \
 apt-mark hold kubeadm

或者
apt-get update && \
 apt-get install -y --allow-change-held-packages kubeadm=1.18.0-00

验证 kubeadm 版本

 kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.0", GitCommit:"9e991415386e4cf155a24b1da15becaa390438d8", GitTreeState:"clean", BuildDate:"2020-03-25T14:56:30Z", GoVersion:"go1.13.8", Compiler:"gc", Platform:"linux/amd64"}

隔离 ukm01

 kubectl drain ukm01 --ignore-daemonsets
node/ukm01 cordoned
node/ukm01 drained

查看可升级版本

 kubeadm upgrade plan
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade] Fetching available versions to upgrade to
[upgrade/versions] Cluster version: v1.17.3
[upgrade/versions] kubeadm version: v1.18.0
[upgrade/versions] Latest stable version: v1.18.0
[upgrade/versions] Latest stable version: v1.18.0
[upgrade/versions] Latest version in the v1.17 series: v1.17.4
[upgrade/versions] Latest version in the v1.17 series: v1.17.4

Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
COMPONENT   CURRENT       AVAILABLE
Kubelet     9 x v1.17.3   v1.17.4

Upgrade to the latest version in the v1.17 series:

COMPONENT            CURRENT   AVAILABLE
API Server           v1.17.3   v1.17.4
Controller Manager   v1.17.3   v1.17.4
Scheduler            v1.17.3   v1.17.4
Kube Proxy           v1.17.3   v1.17.4
CoreDNS              1.6.5     1.6.7
Etcd                 3.4.3     3.4.3-0

You can now apply the upgrade by executing the following command:

  kubeadm upgrade apply v1.17.4

_____________________________________________________________________

Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
COMPONENT   CURRENT       AVAILABLE
Kubelet     9 x v1.17.3   v1.18.0

Upgrade to the latest stable version:

COMPONENT            CURRENT   AVAILABLE
API Server           v1.17.3   v1.18.0
Controller Manager   v1.17.3   v1.18.0
Scheduler            v1.17.3   v1.18.0
Kube Proxy           v1.17.3   v1.18.0
CoreDNS              1.6.5     1.6.7
Etcd                 3.4.3     3.4.3-0

You can now apply the upgrade by executing the following command:

  kubeadm upgrade apply v1.18.0

_____________________________________________________________________

升级 kubernetes

 kubeadm upgrade apply v1.18.0
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade/version] You have chosen to change the cluster version to "v1.18.0"
[upgrade/versions] Cluster version: v1.17.3
[upgrade/versions] kubeadm version: v1.18.0
[upgrade/confirm] Are you sure you want to proceed with the upgrade? [y/N]: y
[upgrade/prepull] Will prepull images for components [kube-apiserver kube-controller-manager kube-scheduler etcd]
[upgrade/prepull] Prepulling image for component etcd.
[upgrade/prepull] Prepulling image for component kube-apiserver.
[upgrade/prepull] Prepulling image for component kube-controller-manager.
[upgrade/prepull] Prepulling image for component kube-scheduler.
[apiclient] Found 0 Pods for label selector k8s-app=upgrade-prepull-etcd
[apiclient] Found 3 Pods for label selector k8s-app=upgrade-prepull-kube-controller-manager
[apiclient] Found 3 Pods for label selector k8s-app=upgrade-prepull-kube-apiserver
[apiclient] Found 0 Pods for label selector k8s-app=upgrade-prepull-kube-scheduler
[apiclient] Found 3 Pods for label selector k8s-app=upgrade-prepull-etcd
[apiclient] Found 3 Pods for label selector k8s-app=upgrade-prepull-kube-scheduler
[upgrade/prepull] Prepulled image for component etcd.
[upgrade/prepull] Prepulled image for component kube-scheduler.
[apiclient] Error getting Pods with label selector "k8s-app=upgrade-prepull-kube-apiserver" [etcdserver: request timed out]
[apiclient] Error getting Pods with label selector "k8s-app=upgrade-prepull-kube-controller-manager" [etcdserver: request timed out]
[upgrade/prepull] Prepulled image for component kube-controller-manager.
[upgrade/prepull] Prepulled image for component kube-apiserver.
[upgrade/prepull] Successfully prepulled the images for all the control plane components
[upgrade/apply] Upgrading your Static Pod-hosted control plane to version "v1.18.0"...
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-controller-manager-ukm01 hash: fdad41767b3b13675192402b9a840a5c
Static pod: kube-scheduler-ukm01 hash: 703c43ab97818f969f780a2cbf4d24b7
[upgrade/etcd] Upgrading to TLS for etcd
[upgrade/etcd] Non fatal issue encountered during upgrade: the desired etcd version for this Kubernetes version "v1.18.0" is "3.4.3-0", but the current etcd version is "3.4.3". Won't downgrade etcd, instead just continue
[upgrade/staticpods] Writing new Static Pod manifests to "/etc/kubernetes/tmp/kubeadm-upgraded-manifests739015179"
W0403 09:40:19.888673   21690 manifests.go:225] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[upgrade/staticpods] Preparing for "kube-apiserver" upgrade
[upgrade/staticpods] Renewing apiserver certificate
[upgrade/staticpods] Renewing apiserver-kubelet-client certificate
[upgrade/staticpods] Renewing front-proxy-client certificate
[upgrade/staticpods] Renewing apiserver-etcd-client certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/kube-apiserver.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2020-04-03-09-40-18/kube-apiserver.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: 7e7a0d385fce06636ac0fc223274d299
Static pod: kube-apiserver-ukm01 hash: f9ef3245bc9735275ed5508f8bbea7e1
[apiclient] Found 3 Pods for label selector component=kube-apiserver
[upgrade/staticpods] Component "kube-apiserver" upgraded successfully!
[upgrade/staticpods] Preparing for "kube-controller-manager" upgrade
[upgrade/staticpods] Renewing controller-manager.conf certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/kube-controller-manager.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2020-04-03-09-40-18/kube-controller-manager.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
Static pod: kube-controller-manager-ukm01 hash: fdad41767b3b13675192402b9a840a5c
Static pod: kube-controller-manager-ukm01 hash: 24e6dad5d81a4c5257a7b29861a48543
[apiclient] Found 3 Pods for label selector component=kube-controller-manager
[upgrade/staticpods] Component "kube-controller-manager" upgraded successfully!
[upgrade/staticpods] Preparing for "kube-scheduler" upgrade
[upgrade/staticpods] Renewing scheduler.conf certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/kube-scheduler.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2020-04-03-09-40-18/kube-scheduler.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
Static pod: kube-scheduler-ukm01 hash: 703c43ab97818f969f780a2cbf4d24b7
Static pod: kube-scheduler-ukm01 hash: b10b96b094242086df7de02b74f10de2
[apiclient] Found 3 Pods for label selector component=kube-scheduler
[upgrade/staticpods] Component "kube-scheduler" upgraded successfully!
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.18" in namespace kube-system with the configuration for the kubelets in the cluster
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.18.0". Enjoy!

[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.

取消隔离

 kubectl uncordon ukm01
node/ukm01 uncordoned

升级 kubectl 和 kubelet

apt-mark unhold kubelet kubectl && \
 apt-get update && apt-get install -y kubelet=1.18.0-00 kubectl=1.18.0-00 && \
 apt-mark hold kubelet kubectl

或者
apt-get update && \
 apt-get install -y --allow-change-held-packages kubelet=1.18.0-00 kubectl=1.18.0-00

升级其他master节点 kubeadm

ansible master:\!ans -m shell -a 'apt-mark unhold kubeadm && apt-get update && apt-get install -y kubeadm=1.18.0-00 && apt-mark hold kubeadm'
ansible master:\!ans -m shell -a 'apt-mark unhold kubelet kubectl && apt-get update && apt-get install -y kubelet=1.18.0-00 kubectl=1.18.0-00 && apt-mark hold kubelet kubectl'
ansible master:\!ans -m shell -a 'kubeadm version'

升级 master 节点 kubernetes

kubectl drain ukm02 --ignore-daemonsets
ssh ukm02
kubeadm upgrade node
exit
kubectl uncordon ukm02

升级 master 节点 kubectl 和 kubelet

ansible master -m shell -a 'apt-mark unhold kubelet kubectl && apt-get update && apt-get install -y kubelet=1.18.0-00 kubectl=1.18.0-00 && apt-mark hold kubelet kubectl'

重启 master 节点 kubelet

ansible master -m shell -a 'systemctl restart kubelet'

确认升级成功

 kubectl get nodes
NAME    STATUS   ROLES    AGE   VERSION
uki01   Ready    infra    8d    v1.17.3
uki02   Ready    infra    8d    v1.17.3
uki03   Ready    infra    8d    v1.17.3
ukm01   Ready    master   8d    v1.18.0
ukm02   Ready    master   8d    v1.18.0
ukm03   Ready    master   8d    v1.18.0
ukn01   Ready    worker   8d    v1.17.3
ukn02   Ready    worker   8d    v1.17.3
ukn03   Ready    worker   8d    v1.17.3

升级 infra 和 worker 节点 kubeadm

ansible infra:worker -m shell -a 'apt-mark unhold kubeadm && apt-get update && apt-get install -y kubeadm=1.18.0-00 && apt-mark hold kubeadm'

升级 infra 和 worker 节点 kubernetes

kubectl drain uki01 --ignore-daemonsets
ssh uki01
kubeadm upgrade node
exit
kubectl uncordon uki01

升级 infra 和 worker 节点 kubectl 和 kubelet

ansible infra:worker -m shell -a 'apt-mark unhold kubelet kubectl && apt-get update && apt-get install -y kubelet=1.18.0-00 kubectl=1.18.0-00 && apt-mark hold kubelet kubectl'

重启 infra 和 worker 节点 kubelet

ansible infra:worker -m shell -a 'systemctl restart kubelet'

检查 kubernetes 升级状态

 kubectl get nodes
NAME    STATUS   ROLES    AGE   VERSION
uki01   Ready    infra    8d    v1.18.0
uki02   Ready    infra    8d    v1.18.0
uki03   Ready    infra    8d    v1.18.0
ukm01   Ready    master   8d    v1.18.0
ukm02   Ready    master   8d    v1.18.0
ukm03   Ready    master   8d    v1.18.0
ukn01   Ready    worker   8d    v1.18.0
ukn02   Ready    worker   8d    v1.18.0
ukn03   Ready    worker   8d    v1.18.0

安装 router

router 是 openshift 开源的 ingress 服务,router 服务默认安装在 default namespace 下面。

配置 rbac 文件

 cat > router/rbac.yaml  <<EOF
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: openshift-router
  namespace: default
rules:
- apiGroups: [""]
  resources: ["namespaces", "services", "endpoints"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["route.openshift.io"]
  resources: ["routes"]
  verbs: ["list", "watch"]
- apiGroups: ["route.openshift.io"]
  resources: ["routes/status"]
  verbs: ["get", "patch", "update"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: openshift-router
roleRef:
  apiGroup: ""
  kind: ClusterRole
  name: openshift-router
subjects:
- kind: ServiceAccount
  namespace: default
  name: router

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: openshift-router-auth-delegator
roleRef:
  apiGroup: ""
  kind: ClusterRole
  name: system:auth-delegator
subjects:
- kind: ServiceAccount
  namespace: default
  name: router
EOF

配置 serviceaccount 文件

 cat > router/serviceaccount.yaml  <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: router
  namespace: default
EOF

配置 crd 文件

 cat > router/crd.yaml  <<EOF
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
   name must match the spec fields below, and be in the form: <plural>.<group>
  name: routes.route.openshift.io
  namespace: default
spec:
   group name to use for REST API: /apis/<group>/<version>
  group: route.openshift.io
   list of versions supported by this CustomResourceDefinition
  versions:
    - name: v1
       Each version can be enabled/disabled by Served flag.
      served: true
       One and only one version must be marked as the storage version.
      storage: true
   either Namespaced or Cluster
  scope: Namespaced
  subresources:
     enable spec/status
    status: {}
  names:
     plural name to be used in the URL: /apis/<group>/<version>/<plural>
    plural: routes
     singular name to be used as an alias on the CLI and for display
    singular: route
     kind is normally the CamelCased singular type. Your resource manifests use this.
    kind: Route
  additionalPrinterColumns:
  - name: Host
    type: string
    JSONPath: .status.ingress[0].host
  - name: Admitted
    type: string
    JSONPath: .status.ingress[0].conditions[?(@.type=="Admitted")].status
  - name: Service
    type: string
    JSONPath: .spec.to.name
  - name: TLS
    type: string
    JSONPath: .spec.tls.type
EOF

配置 router 安装文件

 cat > router/deployment.yaml  <<EOF
---
apiVersion: apps/v1
kind: DaemonSet  Deployment DaemonSet
metadata:
  name: router
  namespace: default
  labels:
    k8s-app: router
spec:
  replicas: 3  如果是 DaemonSet,可以不设置副本数量。
  selector:
    matchLabels:
      k8s-app: router
  template:
    metadata:
      labels:
        k8s-app: router
    spec:
      serviceAccountName: router
      nodeSelector:
        node-role.kubernetes.io/infra: "true"
      containers:
      - env:
        - name: ROUTER_LISTEN_ADDR
          value: 0.0.0.0:1936
        - name: ROUTER_METRICS_TYPE
          value: haproxy
        - name: ROUTER_SERVICE_HTTPS_PORT
          value: "443"
        - name: ROUTER_SERVICE_HTTP_PORT
          value: "80"
        - name: ROUTER_THREADS
          value: "4"
        image: openshift/origin-haproxy-router:v4.0.0
        imagePullPolicy: IfNotPresent
        livenessProbe:
          httpGet:
            host: localhost
            path: /healthz
            port: 1936
          initialDelaySeconds: 10
        name: router
        ports:
        - containerPort: 80
        - containerPort: 443
        - containerPort: 1936
          name: stats
          protocol: TCP
        readinessProbe:
          httpGet:
            host: localhost
            path: healthz/ready
            port: 1936
          initialDelaySeconds: 10
        resources:
          requests:
            cpu: 100m
            memory: 256Mi
      hostNetwork: true
EOF

安装 router 服务

kubectl apply -f router/serviceaccount.yaml

kubectl apply -f router/rbac.yaml

kubectl apply -f router/deployment.yaml

kubectl apply -f router/crd.yaml

检查端口监听

load blance 配置 TCP 80/443/1936 端口转发到 infra 节点服务器。

ansible infra -m shell -a 'netstat -lntp|grep -E "80|443|1936"'

校验 route 服务

安装nginx

kubectl create deployment nginx --image nginx:alpine
kubectl expose deployment nginx --port=80

创建 nginx route

 kubectl apply -f - <<EOF
---
apiVersion: route.openshift.io/v1
kind: Route
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  host: test.mokr.cn
  to:
    kind: Service
    name: nginx
    weight: 100
  wildcardPolicy: None
EOF

访问验证

 curl -ik http://test.mokr.cn --resolve 'test.mokr.cn:80:10.10.34.89'
HTTP/1.1 200 OK
Server: nginx/1.17.9
Date: Tue, 24 Mar 2020 07:01:43 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 03 Mar 2020 17:36:53 GMT
ETag: "5e5e95b5-264"
Accept-Ranges: bytes
Set-Cookie: 3bc5f7a8aec67b74b33e81d956f57cb9=5ffd533ac952a924e0ec8b6f1c601703; path=/; HttpOnly
Cache-control: private

<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

 或者
 curl -ik http://10.10.34.89 -H 'Host: test.mokr.cn' -v

删除 router

kubectl delete -f router.yaml

安装 kubernetes dashboard 2.0

curl -L https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta4/aio/deploy/recommended.yaml -o kubernetes-dashboard.yaml

签发 https 访问证书

  • Can’t load /root/.rnd into RNG
cd /root
openssl rand -writerand .rnd

dashboard 多域名配置

 mkdir -p /certs

 cp /etc/ssl/openssl.cnf /certs
 vim /certs/openssl.cnf
...
[ req_distinguished_name ]
countryName                     = Country Name (2 letter code)
countryName_default             = CN
countryName_min                 = 2
countryName_max                 = 2
stateOrProvinceName             = State or Province Name (full name)
stateOrProvinceName_default     = GuangDong
localityName                    = Locality Name (eg, city)
localityName_default            = ShenZhen
0.organizationName              = Organization Name (eg, company)
0.organizationName_default      = MOKR Ltd
organizationalUnitName          = Organizational Unit Name (eg, section)
organizationalUnitName_default  = OPS
commonName                      = Common Name (e.g. server FQDN or YOUR name)
commonName_max                  = 64
emailAddress                    = Email Address
emailAddress_max                = 64
emailAddress_default            = seanzhau@gmail.com

[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names

[alt_names]
DNS.1 = dashboard.mokr.cn
DNS.2 = kubernetes-dashboard.kubernetes-dashboard.svc
DNS.3 = kubernetes-dashboard.kubernetes-dashboard.svc.cluster.local
IP.1 = 10.104.71.164
IP.2 = 10.10.34.89
...

签发 dashboard 密钥

openssl req -nodes -newkey rsa:2048 \
 -keyout /certs/dashboard.key \
 -out /certs/dashboard.csr \
 -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Mokr LTD/OU=OPS" \
 -config /certs/openssl.cnf \
 -extensions v3_req

签发 dashboard 证书

私钥签发
openssl x509 -req -sha256 -days 365 \
 -in /certs/dashboard.csr \
 -signkey /certs/dashboard.key \
 -out /certs/dashboard.crt \
 -extfile /certs/openssl.cnf \
 -extensions v3_req

ca 签发
openssl x509 -req -days 365 \
 -in /certs/dashboard.csr \
 -CA /etc/kubernetes/pki/ca.crt \
 -CAkey /etc/kubernetes/pki/ca.key \
 -CAcreateserial \
 -out /certs/dashboard.crt \
 -extfile /certs/openssl.cnf \
 -extensions v3_req

验证 dashboard 证书

 openssl x509 -text -noout -in /certs/dashboard.crt
Certificate:
...
        X509v3 extensions:
            X509v3 Basic Constraints:
                CA:FALSE
            X509v3 Key Usage:
                Digital Signature, Non Repudiation, Key Encipherment
            X509v3 Subject Alternative Name:
                DNS:dashboard.mokr.cn, DNS:kubernetes-dashboard.kubernetes-dashboard.svc, DNS:kubernetes-dashboard.kubernetes-dashboard.svc.cluster.local, IP Address:10.104.71.164, IP Address:10.10.34.89
...

部署 dashboard

kubectl apply -f kubernetes-dashboard.yaml

为 dashboard 配置 secret 证书

kubectl create secret generic kubernetes-dashboard-certs --from-file=/certs/dashboard -n kubernetes-dashboard

可不配置
kubectl create secret tls kubernetes-dashboard-tls --cert=/certs/dashboard/dashboard.crt --key=/certs/dashboard/dashboard.key -n kubernetes-dashboard

校验 dashboard

 kubectl get pods,svc -n kubernetes-dashboard
NAME                                             READY   STATUS    RESTARTS   AGE
pod/dashboard-metrics-scraper-566cddb686-m5dzb   1/1     Running   0          22h
pod/kubernetes-dashboard-7b5bf5d559-dbrh2        1/1     Running   0          22h

NAME                                TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
service/dashboard-metrics-scraper   ClusterIP   10.107.240.229   <none>        8000/TCP   22h
service/kubernetes-dashboard        ClusterIP   10.96.188.203    <none>        443/TCP    22h

创建 dashboard route

 kubectl apply -f -  <<EOF
---
apiVersion: route.openshift.io/v1
kind: Route
metadata:
  name: kubernetes-dashboard
  labels:
    app: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  host: dashboard.mokr.cn
  tls:
    insecureEdgeTerminationPolicy: Redirect ## None Allow Redirect
    termination: reencrypt ## edge passthrough reencrypt
  to:
    kind: Service
    name: kubernetes-dashboard
    weight: 100
  wildcardPolicy: None
EOF

curl 访问 dashboard

curl -ik https://dashboard.mokr.cn --resolve 'dashboard.mokr.cn:443:10.10.34.89'

或者

curl -ik https://10.10.34.89 -H 'Host: dashboard.mokr.cn'

创建 dashboard 用户

 创建用户
 kubectl create serviceaccount seanzhau -n kubernetes-dashboard

 添加用户授权,配置集群 admin 权限
## ex: kubectl create clusterrolebinding cluster-admin --clusterrole=cluster-admin --user=user1 --user=user2 --group=group1
 kubectl create clusterrolebinding dashboard-admin --clusterrole cluster-admin --serviceaccount kubernetes-dashboard:seanzhau

 或者
 kubectl apply -f - <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: seanzhau
  namespace: kubernetes-dashboard
  labels:
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: dashboard-admin
  namespace: kubernetes-dashboard
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: seanzhau
  namespace: kubernetes-dashboard
EOF

查看 secret 信息

查看 secret 私钥或者 ca 信息
kubectl -n kubernetes-dashboard get secret -o jsonpath='{range .items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="seanzhau")].data}{.ca\.crt}{end}' | base64 -d

查看用户 token
kubectl -n kubernetes-dashboard get secret -o jsonpath='{range .items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="seanzhau")].data}{.token}{end}' | base64 -d

浏览器访问

浏览器打开 https://dashboard.mokr.cn ,选择 Token ,输入获取的用户 token。

使用 kubeconfig 登录

下载 /etc/kubernetes/admin.conf 到本地,修改配置文件,在最底部添加 token 字段。

 vim admin.conf
...
users:
- name: kubernetes-admin
  user:
    client-certificate-data: xxxxxxxxxxxxxxxxxxxxx
    client-key-data: xxxxxxxxxxxxxxxxxxxxx
    token: xxxxxxxxxxxxxxxxxxxxx

安装 ceph 存储

ceph ansible 安装

git clone -b stable-5.0 https://github.com/ceph/ceph-ansible.git
cd ceph-ansible

安装 pip

apt install -y python-pip

配置 pip 源

 mkdir -p ~/.pip
 cat > ~/.pip/pip.conf <<EOF
[global]
index-url = https://mirrors.aliyun.com/pypi/simple/

[install]
trusted-host=mirrors.aliyun.com
EOF

安装 ansible 以及依赖

pip install -r requirements.txt

配置 ansible 资产文件

 cat  > dummy-ansible-hosts <<EOF
[mons]
10.10.34.29
10.10.34.31
10.10.34.42

[osds]
10.10.34.29
10.10.34.31
10.10.34.42

[mdss]
10.10.34.29
10.10.34.31
10.10.34.42

[rgws]
10.10.34.29
10.10.34.31
10.10.34.42

[mgrs]
10.10.34.29
10.10.34.31
10.10.34.42

[grafana-server]
10.10.34.29

[clients]
10.10.34.29
10.10.34.31
10.10.34.42
EOF

配置 ceph 安装文件

 docker 安装
 cp site-container.yml.sample site-container.yml

 二进制文件安装
 cp site.yml.sample site.yml

 cat > group_vars/all.yml <<EOF
---
cluster: ceph
generate_fsid: true
journal_size: 512
ceph_origin: repository
ceph_repository: community

 docker 安装
ceph_docker_image: "ceph/daemon"
ceph_docker_image_tag: latest-octopus
containerized_deployment: true
ceph_docker_registry: docker.io
ceph_stable_repo: "{{ ceph_mirror }}/debian-{{ ceph_stable_release }}"

 二进制安装
ceph_mirror: http://mirrors.aliyun.com/ceph
ceph_stable_key: http://mirrors.aliyun.com/ceph/keys/release.asc
ceph_stable_release: octopus

public_network: "10.10.34.0/24"
cluster_network: "10.10.34.0/24"
monitor_interface: eth0
osd_objectstore: bluestore
dmcrypt: false
devices:
  - /dev/vdb
  - /dev/vdc
radosgw_interface: eth0
radosgw_address: 0.0.0.0
radosgw_address_block: "10.10.34.0/24"
grafana_admin_user: seanzhau
grafana_admin_password: iPaaS2020
dashboard_enabled: True
dashboard_admin_user: seanzhau
dashboard_admin_password: iPaaS2020
EOF

安装 ceph 集群

docker 安装
ansible-playbook -i dummy-ansible-hosts site-container.yml

二进制文件安装
ansible-playbook -i dummy-ansible-hosts site.yml

非集群节点安装 ceph 客户端

apt install -y ceph-common

配置客户端

 配置 ceph.conf
 cat > /etc/ceph/ceph.conf <<EOF
[global]
mon host = 10.10.34.29,10.10.34.31,10.10.34.42
EOF

 从 ceph 集群复制 admin key
 scp 10.10.34.29:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring

查看集群状态

ceph -s
 cluster:
   id:     5ae77bc9-1831-42df-8ef2-956499ce685b
   health: HEALTH_WARN
           1 pools have too few placement groups

 services:
   mon: 3 daemons, quorum ceph01,ceph02,ceph03 (age 15m)
   mgr: ceph01(active, since 14m), standbys: ceph02, ceph03
   mds: cephfs:1 {0=ceph01=up:active} 2 up:standby
   osd: 6 osds: 6 up (since 19m), 6 in (since 19m)
   rgw: 3 daemons active (ceph01.rgw0, ceph02.rgw0, ceph03.rgw0)

 task status:
   scrub status:
       mds.ceph01: idle

 data:
   pools:   7 pools, 121 pgs
   objects: 215 objects, 11 KiB
   usage:   6.1 GiB used, 2.9 TiB / 2.9 TiB avail
   pgs:     121 active+clean

查看磁盘状态

 ceph osd tree
ID  CLASS  WEIGHT   TYPE NAME                STATUS  REWEIGHT  PRI-AFF
-1         2.92978  root default
-3         0.97659      host ceph01
 0    hdd  0.48830          osd.0                up   1.00000  1.00000
 3    hdd  0.48830          osd.3                up   1.00000  1.00000
-7         0.97659      host ceph02
 2    hdd  0.48830          osd.2                up   1.00000  1.00000
 5    hdd  0.48830          osd.5                up   1.00000  1.00000
-5         0.97659      host ceph03
 1    hdd  0.48830          osd.1                up   1.00000  1.00000
 4    hdd  0.48830          osd.4                up   1.00000  1.00000

挂载 ceph

 mount.ceph 10.10.34.29,10.10.34.31,10.10.34.42:/ /mnt/ -o name=admin,secret=admin-password   secretfile=/root/admin.secret

 df -h /mnt/
Filesystem                             Size  Used Avail Use% Mounted on
10.10.34.29,10.10.34.31,10.10.34.42:/  948G     0  948G   0% /mnt

删除集群

docker 卸载
ansible-playbook -i dummy-ansible-hosts infrastructure-playbooks/purge-container-cluster.yml

二进制文件卸载
ansible-playbook -i dummy-ansible-hosts infrastructure-playbooks/purge-cluster.yml

删除磁盘分区信息

RuntimeError: Unable to use device, already a member of LVM: /dev/vdb

ansible -i dummy-ansible-hosts osds -m shell -a 'wipefs -a /dev/vdb --force'

配置 provisioner

下载 ceph images

docker pull quay.io/external_storage/rbd-provisioner:latest
docker pull quay.io/external_storage/cephfs-provisioner:latest

创建 ceph namespace

kubectl create namespace ceph

安装 cephfs

生成 cephfs admin 密码

ceph auth get-key client.admin > /tmp/key

创建 cephfs secret

kubectl create secret generic cephfs-secret-admin --from-file=/tmp/key --namespace=ceph --type=kubernetes.io/cephfs

创建 cephfs 安装文件

mkdir -p storageClass/cephfs

重新构建 image

如果有权限问题,可以按操作重新制造 image。

 cat > storageClass/cephfs/Dockerfile <<EOF
FROM quay.io/external_storage/cephfs-provisioner:latest

USER root

RUN sed -i 's|0o755|0o777|g' /usr/lib/python2.7/site-packages/ceph_volume_client.py
EOF

 docker build -t quay.io/external_storage/cephfs-provisioner:new storageClass/cephfs/

创建 cephfs deployment 安装文件

 cat > storageClass/cephfs/deployment.yaml <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cephfs-provisioner
  namespace: ceph
spec:
  replicas: 1
  selector:
    matchLabels:
      app: cephfs-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: cephfs-provisioner
    spec:
      nodeSelector:
        node-role.kubernetes.io/infra: "true"
      containers:
      - name: cephfs-provisioner
        image: "quay.io/external_storage/cephfs-provisioner:latest"
        imagePullPolicy: IfNotPresent
        env:
        - name: PROVISIONER_NAME
          value: ceph.com/cephfs
        - name: PROVISIONER_SECRET_NAMESPACE
          value: ceph
        command:
        - "/usr/local/bin/cephfs-provisioner"
        args:
        - "-id=cephfs-provisioner-1"
      serviceAccount: cephfs-provisioner
EOF

创建 cephfs serviceaccount 安装文件

 cat > storageClass/cephfs/serviceaccount.yaml <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: cephfs-provisioner
  namespace: ceph
EOF

创建 cephfs rbac 安装文件

 cat > storageClass/cephfs/rbac.yaml <<EOF
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: cephfs-provisioner
  namespace: ceph
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["create", "get", "delete"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: cephfs-provisioner
  namespace: ceph
subjects:
- kind: ServiceAccount
  name: cephfs-provisioner
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: cephfs-provisioner

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-provisioner
  namespace: ceph
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["kube-dns","coredns"]
    verbs: ["list", "get"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-provisioner
subjects:
  - kind: ServiceAccount
    name: cephfs-provisioner
    namespace: ceph
roleRef:
  kind: ClusterRole
  name: cephfs-provisioner
  apiGroup: rbac.authorization.k8s.io
EOF

创建 cephfs storageclass 安装文件

 cat > storageClass/cephfs/storageclass.yaml <<EOF
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: cephfs-provisioner
provisioner: ceph.com/cephfs
parameters:
    monitors: "10.10.34.29:6789,10.10.34.31:6789,10.10.34.42:6789"
    adminId: admin
    adminSecretName: cephfs-secret-admin
    adminSecretNamespace: ceph
    claimRoot: /storage
EOF

安装 cephfs provisioner

kubectl apply -f storageClass/cephfs/serviceaccount.yaml

kubectl apply -f storageClass/cephfs/rbac.yaml

kubectl apply -f storageClass/cephfs/deployment.yaml

kubectl apply -f storageClass/cephfs/storageclass.yaml

查看 cephfs provisioner

 kubectl get storageclasses cephfs-provisioner
NAME                 PROVISIONER       RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
cephfs-provisioner   ceph.com/cephfs   Delete          Immediate           false                  17s

创建 cephfs PVC

 cat > storageClass/cephfs/pvc.yaml <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: cephfs-test
spec:
  storageClassName: cephfs-provisioner
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 5Gi
EOF

 kubectl apply -f storageClass/cephfs/pvc.yaml -n kube-system
 kubectl apply -f storageClass/cephfs/pvc.yaml -n ceph
 kubectl apply -f storageClass/cephfs/pvc.yaml -n default

验证 cephfs PVC

 kubectl get pvc --all-namespaces
NAMESPACE     NAME          STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS         AGE
ceph          cephfs-test   Bound    pvc-ae5ff9f8-4032-4690-9104-eeb563d45a1e   5Gi        RWX            cephfs-provisioner   8s
default       cephfs-test   Bound    pvc-85d47294-50a0-4f6c-b9ca-7a676cf3c34e   5Gi        RWX            cephfs-provisioner   5s
kube-system   cephfs-test   Bound    pvc-759c1b48-df22-4b72-ab22-490a2f3ebd5c   5Gi        RWX            cephfs-provisioner   13s

创建 rbd

生成 rbd admin 密码

ceph auth get-key client.admin > /tmp/key

创建 admin secret

kubectl create secret generic rbd-secret-admin  --from-file=/tmp/key --namespace=ceph --type=kubernetes.io/rbd

创建 pool secret

ceph osd pool create kube 8 8

ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'

ceph auth get-key client.kube > /tmp/key

kubectl create secret generic kube-secret --from-file=/tmp/key --namespace=ceph --type=kubernetes.io/rbd

创建 rbd 目录

mkdir -p storageClass/rbd

创建 rbd deployment 安装文件

 cat > storageClass/rbd/deployment.yaml <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rbd-provisioner
  namespace: ceph
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rbd-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: rbd-provisioner
    spec:
      nodeSelector:
        node-role.kubernetes.io/infra: "true"
      containers:
      - name: rbd-provisioner
        image: "quay.io/external_storage/rbd-provisioner:latest"
        imagePullPolicy: IfNotPresent
        env:
        - name: PROVISIONER_NAME
          value: ceph.com/rbd
      serviceAccount: rbd-provisioner
EOF

创建 rbd serviceaccount 安装文件

 cat > storageClass/rbd/serviceaccount.yaml <<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rbd-provisioner
  namespace: ceph
EOF

创建 rbd rbac 安装文件

 cat > storageClass/rbd/rbac.yaml <<EOF
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: rbd-provisioner
  namespace: ceph
rules:
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get", "list", "watch", "create", "update", "patch"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: rbd-provisioner
  namespace: ceph
subjects:
- kind: ServiceAccount
  name: rbd-provisioner
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rbd-provisioner

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rbd-provisioner
  namespace: ceph
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["kube-dns","coredns"]
    verbs: ["list", "get"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: rbd-provisioner
subjects:
  - kind: ServiceAccount
    name: rbd-provisioner
    namespace: ceph
roleRef:
  kind: ClusterRole
  name: rbd-provisioner
  apiGroup: rbac.authorization.k8s.io
EOF

创建 rbd storageclass 安装文件

 cat > storageClass/rbd/storageclass.yaml <<EOF
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rbd-provisioner
provisioner: ceph.com/rbd
parameters:
  monitors: "10.10.34.29:6789,10.10.34.31:6789,10.10.34.42:6789"
  pool: kube
  adminId: admin
  adminSecretNamespace: ceph
  adminSecretName: rbd-secret-admin
  userId: kube
  userSecretNamespace: ceph
  userSecretName: ceph-secret
  imageFormat: "2"
  imageFeatures: layering
EOF

安装 rbd provisioner

kubectl apply -f storageClass/rbd/serviceaccount.yaml

kubectl apply -f storageClass/rbd/rbac.yaml

kubectl apply -f storageClass/rbd/deployment.yaml

kubectl apply -f storageClass/rbd/storageclass.yaml

查看 rbd provisioner

 kubectl get storageclasses rbd-provisioner
NAME              PROVISIONER    RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
rbd-provisioner   ceph.com/rbd   Delete          Immediate           false                  7s

创建 rbd PVC

 cat > storageClass/rbd/pvc.yaml <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: rbd-test
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: rbd-provisioner
  resources:
    requests:
      storage: 10Gi
EOF

 kubectl apply -f storageClass/rbd/pvc.yaml -n kube-system
 kubectl apply -f storageClass/rbd/pvc.yaml -n ceph
 kubectl apply -f storageClass/rbd/pvc.yaml -n default

验证 rbd PVC

 kubectl get pvc --all-namespaces
NAMESPACE     NAME          STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS         AGE
ceph          rbd-test      Bound    pvc-4d85ad48-b24c-43b6-a7c2-89aec9c661a7   10Gi       RWO            rbd-provisioner      19s
default       rbd-test      Bound    pvc-9d93ccbd-b6ed-42fe-8f57-78fbff88baeb   10Gi       RWO            rbd-provisioner      11s
kube-system   rbd-test      Bound    pvc-5887166f-d9b4-4d40-b419-2e37ac81901d   10Gi       RWO            rbd-provisioner      31s

对象存储

s3 存储

 配置 ceph s3 账号,保留 access_key 和 secret_key
 radosgw-admin user create --uid="testuser" --display-name="test user"
{
    "user_id": "testuser",
    "display_name": "test user",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "testuser",
            "access_key": "60DD3SXN58LIRC06ILQE",
            "secret_key": "zgFnP3lL3uPF3WsCeQtnESKvgPYHnW75cSJfkYeZ"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

 安装 s3 python 库
 pip install boto

 创建 s3 测试文件,使用上面创建的账号。
 cat > s3.py <<EOF
from boto.s3.connection import S3Connection
from boto.s3.connection import OrdinaryCallingFormat


 add radosgw user in ceph cluster
 radosgw-admin user create --uid=rgwuser --subuser=rgwuser:swift --display-name=s3 --access=full

conn = S3Connection(aws_access_key_id='60DD3SXN58LIRC06ILQE',
                    aws_secret_access_key='zgFnP3lL3uPF3WsCeQtnESKvgPYHnW75cSJfkYeZ',
                    host='10.10.34.29',
                    port=8080,
                    is_secure=False,
                    calling_format=OrdinaryCallingFormat())

create_bucket = conn.create_bucket('cephfs')

buckets_list = conn.get_all_buckets()
for bucket in buckets_list:
     print('%s\t%s' % (bucket.name, bucket.creation_date))

delete_bucket = conn.delete_bucket('cephfs')
EOF

 测试验证
 python s3.py
cephfs  2020-04-10T08:16:15.647Z

swift

 创建账号
 radosgw-admin subuser create --uid=testuser --subuser=testuser:swift --access=full
{
    "user_id": "testuser",
    "display_name": "test user",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [
        {
            "id": "testuser:swift",
            "permissions": "full-control"
        }
    ],
    "keys": [
        {
            "user": "testuser",
            "access_key": "60DD3SXN58LIRC06ILQE",
            "secret_key": "zgFnP3lL3uPF3WsCeQtnESKvgPYHnW75cSJfkYeZ"
        }
    ],
    "swift_keys": [
        {
            "user": "testuser:swift",
            "secret_key": "b59Aw59eiStnhibukAm7fN3vQRtTU2BwNio2dHnl"
        }
    ],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

 重置密码
 radosgw-admin key create --subuser=testuser:swift --key-type=swift --gen-secret

 安装 swift 客户端
 pip install python-swiftclient

 测试
 swift -A http://10.10.34.29:8080/auth/1.0 -U testuser:swift -K 'b59Aw59eiStnhibukAm7fN3vQRtTU2BwNio2dHnl' list

安装 istio

使用 demo 安装

  • 参考 https://istio.io/docs/setup/additional-setup/config-profiles/
istioctl manifest apply --set profile=demo --set values.gateways.istio-ingressgateway.type=NodePort

自定义参数安装

  • 参数 https://istio.io/zh/docs/reference/config/installation-options/
istioctl manifest apply \
 --namespace istio-system \
 --set values.global.hub=docker.io/istio \
 --set values.global.tag=1.5.0 \
 --set values.global.logging.level="default:info" \
 --set values.global.imagePullPolicy=IfNotPresent \
 --set values.global.k8sIngress.enabled=false \
 --set values.global.k8sIngress.enableHttps=false \
 --set values.global.proxy.image=proxyv2 \
 --set values.global.proxy.clusterDomain="cluster.local" \
 --set values.global.proxy.privileged=true \
 --set values.global.proxy.enableCoreDump=true \
 --set values.global.proxy.autoInject=enabled \
 --set values.global.proxy.tracer=zipkin \
 --set values.global.controlPlaneSecurityEnabled=true \
 --set values.global.mtls.enabled=false \
 --set values.global.outboundTrafficPolicy.mode=ALLOW_ANY \
 --set values.global.sds.enabled=false \
 --set values.certmanager.enabled=false \
 --set values.galley.enabled=true \
 --set values.gateways.enabled=true \
 --set values.gateways.istio-ingressgateway.enabled=true \
 --set values.gateways.istio-ingressgateway.type=NodePort  NodePort, ClusterIP or LoadBalancer \
 --set values.gateways.istio-egressgateway.enabled=true \
 --set values.grafana.enabled=true \
 --set values.istio_cni.enabled=false \
 --set values.istiocoredns.enabled=false \
 --set values.kiali.enabled=true \
 --set values.kiali.createDemoSecret=true \
 --set values.kiali.hub=docker.io/kiali \
 --set values.mixer.policy.enabled=false \
 --set values.mixer.policy.autoscaleEnabled=true \
 --set values.nodeagent.enabled=false \
 --set values.pilot.enabled=true \
 --set values.pilot.sidecar=true \
 --set values.prometheus.enabled=true \
 --set values.security.enabled=true \
 --set values.sidecarInjectorWebhook.enabled=true \
 --set values.sidecarInjectorWebhook.enableNamespacesByDefault=true \
 --set values.tracing.enabled=true \
 --set values.tracing.jaeger.hub=docker.io/jaegertracing \
 --set values.tracing.jaeger.tag=1.14

查看 ingress 端口监听情况

 kubectl get svc -n istio-system | grep ingress
istio-ingressgateway        NodePort    10.105.16.129    <none>        15020:30796/TCP,80:32179/TCP,443:31133/TCP,15029:31338/TCP,15030:30980/TCP,15031:31675/TCP,15032:32296/TCP,31400:31821/TCP,15443:32679/TCP   30s

 ansible ins -m shell -a 'netstat -lntp | grep -E "32179|31133"'
10.10.34.92 | CHANGED | rc=0 >>
tcp6       0      0 :::32179                :::*                    LISTEN      21780/kube-proxy
tcp6       0      0 :::31133                :::*                    LISTEN      21780/kube-proxy
10.10.34.94 | CHANGED | rc=0 >>
tcp6       0      0 :::32179                :::*                    LISTEN      3093/kube-proxy
tcp6       0      0 :::31133                :::*                    LISTEN      3093/kube-proxy
10.10.34.100 | CHANGED | rc=0 >>
tcp6       0      0 :::32179                :::*                    LISTEN      14998/kube-proxy
tcp6       0      0 :::31133                :::*                    LISTEN      14998/kube-proxy
10.10.34.99 | CHANGED | rc=0 >>
tcp6       0      0 :::31133                :::*                    LISTEN      3031/kube-proxy
tcp6       0      0 :::32179                :::*                    LISTEN      3031/kube-proxy
10.10.34.95 | CHANGED | rc=0 >>
tcp6       0      0 :::32179                :::*                    LISTEN      23912/kube-proxy
tcp6       0      0 :::31133                :::*                    LISTEN      23912/kube-proxy
10.10.34.96 | CHANGED | rc=0 >>
tcp6       0      0 :::32179                :::*                    LISTEN      26951/kube-proxy
tcp6       0      0 :::31133                :::*                    LISTEN      26951/kube-proxy
10.10.34.93 | CHANGED | rc=0 >>
tcp6       0      0 :::31133                :::*                    LISTEN      31622/kube-proxy
tcp6       0      0 :::32179                :::*                    LISTEN      31622/kube-proxy
10.10.34.97 | CHANGED | rc=0 >>
tcp6       0      0 :::32179                :::*                    LISTEN      4244/kube-proxy
tcp6       0      0 :::31133                :::*                    LISTEN      4244/kube-proxy
10.10.34.98 | CHANGED | rc=0 >>
tcp6       0      0 :::31133                :::*                    LISTEN      8077/kube-proxy
tcp6       0      0 :::32179                :::*                    LISTEN      8077/kube-proxy

配置 load blance

  • LB 80 端口的 TCP 协议请求转发到集群 node 节点的 32179 端口
  • LB 443 端口的 TCP 协议请求转发到集群 node 节点的 31133 端口

应用测试

创建 namespace 并打开自动注入

kubectl create namespace book

kubectl label namespace book istio-injection=enabled

创建 demo

kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml -n book

kubectl get services -n book

kubectl get pods -n book

kubectl exec -n book \
 -it $(kubectl get pod  -n book -l app=ratings -o jsonpath='{.items[0].metadata.name}') -c ratings \
 -- curl productpage:9080/productpage | grep -o "<title>.*</title>"

创建 http gateway

 kubectl apply -f - <<EOF
--
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: book
  namespace: book
spec:
  hosts:
  - "book.mokr.cn"
  gateways:
  - istio-system/ingressgateway
  http:
  - match:
    - uri:
        exact: /productpage
    - uri:
        prefix: /static
    - uri:
        exact: /login
    - uri:
        exact: /logout
    - uri:
        prefix: /api/v1/products
    route:
    - destination:
        host: productpage
        port:
          number: 9080
EOF

验证 http gateway

curl -ik http://book.mokr.cn --resolve 'book.mokr.cn:443:10.10.34.89' -v

创建 https gateway

 创建根证书
 openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout mokr.cn.key -out mokr.cn.crt -subj "/C=CN/ST=Guangdong/L=Shenzhen/O=mokr/OU=ops/CN=mokr.cn"

 创建域名 key
 openssl req -out book.mokr.cn.csr -newkey rsa:2048 -nodes -keyout book.mokr.cn.key -subj "/C=CN/ST=Guangdong/L=Shenzhen/O=mokr/OU=ops/CN=book.mokr.cn"

 创建域名证书
 openssl x509 -req -days 365 -CA mokr.cn.crt -CAkey mokr.cn.key -set_serial 0 -in book.mokr.cn.csr -out book.mokr.cn.crt

 创建 tls secret
 kubectl create secret tls ingressgateway-book-certs --key book.mokr.cn.key --cert book.mokr.cn.crt  -n istio-system

 创建 https gateway
 kubectl apply -f - <<EOF
--
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: book-gateway
  namespace: istio-system
spec:
  selector:
    istio: ingressgateway
  servers:
  - port:
      number: 80
      name: http
      protocol: HTTP
    hosts:
    - book.mokr.cn
    tls:
      httpsRedirect: true
  - port:
      number: 443
      name: https
      protocol: HTTPS   HTTP | HTTPS | GRPC | HTTP2 | MONGO | TCP | TLS
    tls:
      mode: SIMPLE    PASSTHROUGH | SIMPLE | MUTUAL | AUTO_PASSTHROUGH | ISTIO_MUTUAL
      credentialName: ingressgateway-book-certs
    hosts:
    - book.mokr.cn
EOF

 创建 https VirtualService
 kubectl apply -f - <<EOF
--
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: book
  namespace: book
spec:
  hosts:
  - "book.mokr.cn"
  gateways:
  - istio-system/book-gateway
  http:
  - match:
    - uri:
        exact: /productpage
    - uri:
        prefix: /static
    - uri:
        exact: /login
    - uri:
        exact: /logout
    - uri:
        prefix: /api/v1/products
    route:
    - destination:
        host: productpage
        port:
          number: 9080
EOF

查看安装情况

 kubectl get gateway -n book
NAME               AGE
bookinfo-gateway   23s

 kubectl get virtualservice -n book
NAME       GATEWAYS             HOSTS                       AGE
bookinfo   [bookinfo-gateway]   [book.mokr.cn]   36s

验证 https gateway

curl -ik https://book.mokr.cn --resolve 'book.mokr.cn:443:10.10.34.89' -v

删除 demo

kubectl delete -f samples/bookinfo/platform/kube/bookinfo.yaml -n book

kubectl delete namespace book --force --grace-period=0

删除 istioctl 集群

istioctl manifest generate | kubectl delete -f -

部署 registry

创建 pvc 部署文件

 cat > registry/pvc.yaml <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: registry-storage
spec:
  storageClassName: cephfs-provisioner
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
EOF

创建 service 部署文件

 cat > registry/service.yaml <<EOF
---
apiVersion: v1
kind: Service
metadata:
  name: docker-registry
  namespace: default
spec:
  ports:
  - port: 5000
    protocol: TCP
    targetPort: 5000
  selector:
    k8s-app: docker-registry
  sessionAffinity: None
  type: ClusterIP
EOF

创建 registry 部署文件

 cat > registry/deployment.yaml <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    k8s-app: docker-registry
  name: docker-registry
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: docker-registry
  template:
    metadata:
      labels:
        k8s-app: docker-registry
    spec:
      nodeSelector:
        node-role.kubernetes.io/infra: "true"
      containers:
      - env:
        - name: REGISTRY_HTTP_ADDR
          value: :5000
        - name: REGISTRY_HTTP_TLS_CERTIFICATE
          value: /etc/secrets/tls.crt
        - name: REGISTRY_HTTP_TLS_KEY
          value: /etc/secrets/tls.key
        - name: REGISTRY_STORAGE_DELETE_ENABLED
          value: "true"
        name: registry
        image: registry:2
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 5000
          protocol: TCP
        resources:
          requests:
            cpu: 100m
            memory: 256Mi
        volumeMounts:
        - name: registry-data
          mountPath: /var/lib/registry
        - name: registry-certificates
          mountPath: /etc/secrets
      serviceAccountName: docker-registry
      volumes:
      - name: registry-data
        persistentVolumeClaim:
          claimName: registry-storage
      - name: registry-certificates
        secret:
          defaultMode: 420
          secretName: registry-certificates
EOF

创建 service

kubectl apply -f registry/service.yaml

kubectl get svc docker-registry -o custom-columns=clusterIP:.spec.clusterIP

配置多域名

 mkdir -p registry/

 cp /etc/ssl/openssl.cnf registry/
 vim registry/openssl.cnf
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names

[alt_names]
DNS.1 = docker-registry.mokr.cn
DNS.2 = docker-registry.default.svc
DNS.3 = docker-registry.default.svc.cluster.local

签发 registry 密钥

openssl req -nodes -newkey rsa:2048 \
 -keyout registry/docker-registry.key \
 -out registry/docker-registry.csr \
 -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Mokr LTD/OU=OPS" \
 -config registry/openssl.cnf \
 -extensions v3_req

签发 registry 证书

私钥签发
openssl x509 -req -sha256 -days 365 \
 -in registry/docker-registry.csr \
 -signkey registry/docker-registry.key \
 -out registry/docker-registry.crt \
 -extfile registry/openssl.cnf \
 -extensions v3_req

ca 签发
openssl x509 -req -days 365 \
 -in registry/docker-registry.csr \
 -CA /etc/kubernetes/pki/ca.crt \
 -CAkey /etc/kubernetes/pki/ca.key \
 -CAcreateserial \
 -out registry/docker-registry.crt \
 -extfile registry/openssl.cnf \
 -extensions v3_req

验证 registry 证书

 openssl x509 -text -noout -in registry/docker-registry.crt
Certificate:
...
        X509v3 extensions:
            X509v3 Basic Constraints:
                CA:FALSE
            X509v3 Key Usage:
                Digital Signature, Non Repudiation, Key Encipherment
            X509v3 Subject Alternative Name:
                DNS:docker-registry.mokr.cn, DNS:docker-registry.default.svc, DNS:docker-registry.default.svc.cluster.local
...

为 registry 配置 secret 证书

kubectl create secret tls registry-certificates --cert=registry/docker-registry.crt --key=registry/docker-registry.key

创建 registry

kubectl apply -f registry/pvc.yaml

kubectl apply -f registry/deployment.yaml

创建 registry route

 cat > registry/docker-registry-route.yaml <<EOF
kind: Route
apiVersion: route.openshift.io/v1
metadata:
  name: docker-registry
  labels:
    app: docker-registry
spec:
  host: docker-registry.mokr.cn
  tls:
    insecureEdgeTerminationPolicy: Redirect ## None Allow Redirect
    termination: passthrough ## edge passthrough reencrypt
  to:
    kind: Service
    name: docker-registry
    weight: 100
  wildcardPolicy: None
EOF

 kubectl apply -f registry/docker-registry-route.yaml

配置 docker

 vim /etc/docker/daemon.json
{
...
  "registry-mirrors": [
    "https://docker-registry.mokr.cn",
    "https://docker-registry.default.svc:5000",
    "https://docker-registry.default.svc.cluster.local:5000"
  ],
  "insecure-registries": [
    "docker-registry.mokr.cn",
    "docker-registry.default.svc:5000",
    "docker-registry.default.svc.cluster.local:5000"
  ],
...
}

 systemctl restart docker

 docker info
...
 Insecure Registries:
  docker-registry.default.svc.cluster.local:5000
  docker-registry.default.svc:5000
  docker-registry.mokr.cn
  127.0.0.0/8
 Registry Mirrors:
  https://docker-registry.mokr.cn/
  https://docker-registry.default.svc:5000/
  https://docker-registry.default.svc.cluster.local:5000/
 Live Restore Enabled: false

下载 image 测试

docker pull busybox:latest

docker tag busybox:latest docker-registry.default.svc:5000/busybox:latest
docker tag busybox:latest docker-registry.default.svc.cluster.local:5000/busybox:latest
docker tag busybox:latest docker-registry.mokr.cn/default/busybox:latest

docker push docker-registry.default.svc:5000/busybox:latest
docker push docker-registry.default.svc.cluster.local:5000/busybox:latest
docker push docker-registry.mokr.cn/default/busybox:latest

验证 registry

 curl -ik https://docker-registry.mokr.cn/v2/_catalog --resolve 'docker-registry.mokr.cn:443:10.10.34.89' -v
* Added docker-registry.mokr.cn:443:10.10.34.89 to DNS cache
* Hostname docker-registry.mokr.cn was found in DNS cache
*   Trying 10.10.34.89...
* TCP_NODELAY set
* Connected to docker-registry.mokr.cn (10.10.34.89) port 443 (0)
* ALPN, offering h2
* ALPN, offering http/1.1
* successfully set certificate verify locations:
*   CAfile: /etc/ssl/certs/ca-certificates.crt
  CApath: /etc/ssl/certs
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
* TLSv1.3 (IN), TLS handshake, Server hello (2):
* TLSv1.2 (IN), TLS handshake, Certificate (11):
* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
* TLSv1.2 (IN), TLS handshake, Server finished (14):
* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
* TLSv1.2 (OUT), TLS change cipher, Client hello (1):
* TLSv1.2 (OUT), TLS handshake, Finished (20):
* TLSv1.2 (IN), TLS handshake, Finished (20):
* SSL connection using TLSv1.2 / ECDHE-RSA-AES128-GCM-SHA256
* ALPN, server accepted to use h2
* Server certificate:
*  subject: C=CN; ST=GuangDong; L=ShenZhen; O=Mokr LTD; OU=OPS
*  start date: Apr 11 15:05:20 2020 GMT
*  expire date: Apr 11 15:05:20 2021 GMT
*  issuer: CN=kubernetes
*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
* Using HTTP2, server supports multi-use
* Connection state changed (HTTP/2 confirmed)
* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0
* Using Stream ID: 1 (easy handle 0x556a671559a0)
> GET /v2/_catalog HTTP/2
> Host: docker-registry.mokr.cn
> User-Agent: curl/7.58.0
> Accept: */*
>
* Connection state changed (MAX_CONCURRENT_STREAMS updated)!
< HTTP/2 200
HTTP/2 200
< content-type: application/json; charset=utf-8
content-type: application/json; charset=utf-8
< docker-distribution-api-version: registry/2.0
docker-distribution-api-version: registry/2.0
< x-content-type-options: nosniff
x-content-type-options: nosniff
< content-length: 47
content-length: 47
< date: Sat, 11 Apr 2020 15:57:22 GMT
date: Sat, 11 Apr 2020 15:57:22 GMT

<
{"repositories":["busybox","default/busybox"]}
* Connection 0 to host docker-registry.mokr.cn left intact

清理 registry images

 登录 docker-registry
 kubectl exec -it $(kubectl get pods -l k8s-app=docker-registry -o jsonpath={.items[0].metadata.name}) sh

/  registry garbage-collect /etc/docker/registry/config.yml --delete-untagged=true

安装部署 cert-manager

下载安装脚本

mkdir -p cert-manager

curl -L https://github.com/jetstack/cert-manager/releases/download/v0.14.1/cert-manager.yaml -o cert-manager/cert-manager.yaml

sed -i 's/namespace: "cert-manager"/namespace: kube-system/g' cert-manager/cert-manager.yaml

sed -i 's/namespace: cert-manager/namespace: kube-system/g' cert-manager/cert-manager.yaml

sed -i 's/secret: "cert-manager/secret: kube-system/g' cert-manager/cert-manager.yaml

sed -i 's/secret: cert-manager/secret: kube-system/g' cert-manager/cert-manager.yaml

注释 5878 - 5882 行,关闭
sed -i '5878,5882s/^//g' cert-manager/cert-manager.yaml

执行安装

kubectl apply -f cert-manager/cert-manager.yaml

检查安装

 kubectl get pods -n kube-system | grep cert-manager
cert-manager-557f8d6944-zr5jh              1/1     Running   0          2m1s
cert-manager-cainjector-6999bdd97-nfsrn    1/1     Running   0          2m1s
cert-manager-webhook-c579c9f47-g6rd5       1/1     Running   0          2m1s

 kubectl get crd -o custom-columns=crd-name:.metadata.name|grep cert-manager
certificaterequests.cert-manager.io
certificates.cert-manager.io
challenges.acme.cert-manager.io
clusterissuers.cert-manager.io
issuers.cert-manager.io
orders.acme.cert-manager.io

本博客所有文章除特别声明外,均采用: 署名-非商业性使用-禁止演绎 4.0 国际协议,转载请保留原文链接及作者。

Ansible简明指北 上一篇
ansible 部署zk、kafka、nacos集群 下一篇

 目录


买个卤蛋,吃根冰棒