Kubernetes 集羣部署
使用 VMware 創建出三個虛機環境,其中一個作爲 K8S 集羣中的 Master 節點(高可用部署建議三個 Master 節點),另外兩個作爲 Work 節點(Work 節點可隨時動態加入集羣)
使用 Xshell 連接,後續都使用 shell 命令進行操作
注:需要在所有節點上執行初始化步驟
初始化之設置主機名
hostnamectl set-hostname master # 在 master 節點執行
hostnamectl set-hostname work1 # 在 work1 節點執行
hostnamectl set-hostname work2 # 在 work2 節點執行
添加 DNS 解析
cat >> /etc/hosts <<EOF
192.168.19.136 master
192.168.19.137 work1
192.168.19.138 work2
EOF
重新登陸後生效
初始化之關閉防火牆
關閉防火牆,清理防火牆規則,設置默認轉發策略
systemctl stop firewalld
systemctl disable firewalld
iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat
iptables -P FORWARD ACCEPT
初始化之關閉 swap 分區
關閉 swap 分區,否則 kubelet 會啓動失敗
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
初始化之關閉 SELinux
關閉 SELinux,否則 kubelet 掛載目錄時可能報錯 Permission denied
setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
image.png
初始化之安裝 Docker
使用 yum
安裝
sudo yum install -y yum-utils device-mapper-persistent-data lvm2
sudo yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sudo yum install -y docker-ce-18.09.9 docker-ce-cli-18.09.9 containerd.io
(如果是 root
用戶,請忽略)普通用戶需要設置成非 root
用戶也能執行 docker
,需要將用戶加入 docker
組(例如你的登錄用戶名是 togettoyou
)
sudo usermod -aG docker togettoyou # 需要重啓生效
配置阿里鏡像,進入 https://cr.console.aliyun.com/cn-hangzhou/instances/mirrors 申請專屬鏡像加速器,並設置 systemd
爲 Docker
的驅動程序
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["改爲你申請的加速器地址"],
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
重載 daemon、重啓 docker 、啓用 docker 自啓服務
sudo systemctl daemon-reload
sudo service docker restart
sudo systemctl enable docker.service
下載 Docker Compose
curl -L https://get.daocloud.io/docker/compose/releases/download/1.25.4/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
配置執行權限
sudo chmod +x /usr/local/bin/docker-compose
檢查是否安裝成功
docker -v
docker-compose -v
image.png
初始化之安裝 kubelet kubeadm kubectl
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
sudo yum install -y kubelet-1.16.7 kubeadm-1.16.7 kubectl-1.16.7
檢查是否安裝成功:
kubelet --version
kubeadm version
kubectl version
image.png
初始化之修改內核的運行參數
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
# 應用配置
sysctl --system
image.png
初始化之配置 kubelet
sudo mkdir -p /etc/systemd/system/kubelet.service.d
sudo tee /etc/systemd/system/kubelet.service.d/10-proxy-ipvs.conf <<-'EOF'
# 啓用 ipvs 相關內核模塊
[Service]
ExecStartPre=-/sbin/modprobe ip_vs
ExecStartPre=-/sbin/modprobe ip_vs_rr
ExecStartPre=-/sbin/modprobe ip_vs_wrr
ExecStartPre=-/sbin/modprobe ip_vs_sh
EOF
sudo systemctl daemon-reload
systemctl enable kubelet.service
image.png
初始化 master 節點:
sudo kubeadm init --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers \
--kubernetes-version v1.16.7 \
--pod-network-cidr 10.244.0.0/16 \
--v 5 \
--ignore-preflight-errors=all
如遇失敗,可以使用 sudo kubeadm reset
恢復重設後,根據錯誤提示解決後重新部署
根據提示,要使非 root
用戶可以運行 kubectl
,請運行以下命令 :
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
或者,如果你是 root
用戶,則可以運行 :
export KUBECONFIG=/etc/kubernetes/admin.conf
也可以永久加入到環境變量中:
sudo tee /etc/profile <<-'EOF'
export KUBECONFIG=/etc/kubernetes/admin.conf
EOF
source /etc/profile
flannel
網絡插件:
# 添加 flannel 網絡插件,需要在 kubeadm init 時設置 --pod-network-cidr=10.244.0.0/16
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.14.0-rc1/Documentation/kube-flannel.yml
網絡原因可能無法加載 flannel yml url
,可使用以下命令從標準輸入創建:
cat <<EOF | kubectl apply -f -
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp.flannel.unprivileged
annotations:
seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
privileged: false
volumes:
- configMap
- secret
- emptyDir
- hostPath
allowedHostPaths:
- pathPrefix: "/etc/cni/net.d"
- pathPrefix: "/etc/kube-flannel"
- pathPrefix: "/run/flannel"
readOnlyRootFilesystem: false
# Users and groups
runAsUser:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
fsGroup:
rule: RunAsAny
# Privilege Escalation
allowPrivilegeEscalation: false
defaultAllowPrivilegeEscalation: false
# Capabilities
allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
defaultAddCapabilities: []
requiredDropCapabilities: []
# Host namespaces
hostPID: false
hostIPC: false
hostNetwork: true
hostPorts:
- min: 0
max: 65535
# SELinux
seLinux:
# SELinux is unused in CaaSP
rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
rules:
- apiGroups: ['extensions']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
tier: node
app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
"Network": "10.244.0.0/16",
"Backend": {
"Type": "vxlan"
}
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel-ds
namespace: kube-system
labels:
tier: node
app: flannel
spec:
selector:
matchLabels:
app: flannel
template:
metadata:
labels:
tier: node
app: flannel
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
hostNetwork: true
priorityClassName: system-node-critical
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: flannel
initContainers:
- name: install-cni
image: quay.io/coreos/flannel:v0.14.0-rc1
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: quay.io/coreos/flannel:v0.14.0-rc1
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: run
mountPath: /run/flannel
- name: flannel-cfg
mountPath: /etc/kube-flannel/
volumes:
- name: run
hostPath:
path: /run/flannel
- name: cni
hostPath:
path: /etc/cni/net.d
- name: flannel-cfg
configMap:
name: kube-flannel-cfg
EOF
在 work1
和 work2
節點執行 master
kubeadm init
打印出的 kubeadm join
命令加入集羣
kubeadm join 192.168.19.136:6443 --token gtv9xk.e5n390w7ry70029t \
--discovery-token-ca-cert-hash sha256:e4a9199f2b6138251efc0b6d55eb7d1ef81287f522dae2ab1a6a90bba4973001
在 master
節點執行
kubectl get nodes -o wide
work
節點也想使用 kubectl
工具,需要將 master
節點中的 /etc/kubernetes/admin.conf
拷貝到 work
節點下
scp /etc/kubernetes/admin.conf root@work1:/etc/kubernetes/admin.conf
scp /etc/kubernetes/admin.conf root@work2:/etc/kubernetes/admin.conf
在集羣中測試部署 nginx 服務,在 master 節點運行以下命令創建
kubectl create namespace test-zjh # 創建命名空間
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1 # API版本
kind: Deployment # 資源對象類型爲 Deployment
metadata:
name: zjh-nginx # Deployment的名稱
namespace: test-zjh # 命名空間
spec:
selector: # 定位需要管理的 Pod,通過Pod的labels標籤定位
matchLabels:
app: test-nginx
replicas: 2 # 要部署的個數,k8s自動擴容分配
template: # 要部署的 Pod
metadata:
labels:
app: test-nginx
spec:
containers:
- name: web
image: nginx:latest
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: nginx-service # Service的名稱
namespace: test-zjh # 命名空間
spec:
ports:
- port: 80 # Service虛端口
targetPort: 80 # 容器端口
nodePort: 30088 # 暴露端口
selector: # 指定如何選擇 Pod
app: test-nginx
type: NodePort # 指定爲Node的IP地址
EOF
本文由 Readfog 進行 AMP 轉碼,版權歸原作者所有。
來源:https://mp.weixin.qq.com/s/7nlTJlnoVOgHHzcrvjOstA