kubeadm安裝kubernetes v1.11.3 HA多主高可用並啟用ipvs
摘要：
環境介紹：
系統版本：CentOS 7.5
核心：4.18.7-1.el7.elrepo.x86_64
Kubernetes: v1.11.3
Docker-ce: 18.06
Keepalived保證apiserever伺服器的IP高可用
Haproxy實現apiserver的負載均...
環境介紹：
系統版本：CentOS 7.5 核心：4.18.7-1.el7.elrepo.x86_64 Kubernetes: v1.11.3 Docker-ce: 18.06 Keepalived保證apiserever伺服器的IP高可用 Haproxy實現apiserver的負載均衡 master x3 && etcd x3 保證k8s叢集可用性 192.168.1.1master 192.168.1.2master2 192.168.1.3master3 192.168.1.4Keepalived + Haproxy 192.168.1.5Keepalived + Haproxy 192.168.1.6etcd1 192.168.1.7etcd2 192.168.1.8etcd3 192.168.1.9node1 192.168.1.10node2 192.168.1.100VIP、apiserver的地址 有道筆記原文：http://note.youdao.com/noteshare?id=cd79131892c3a5bdae220d6fd8013555&sub=0687104101804B26AC12AE423C7E13E6
-
ofollow,noindex" target="_blank">有道筆記原文
一、準備工作
為方便操作，所有操作均以root使用者執行
以下操作僅在kubernetes叢集節點執行即可
- 關閉selinux和防火牆
sed -ri 's#(SELINUX=).*#\1disabled#' /etc/selinux/config setenforce 0 systemctl disable firewalld systemctl stop firewalld
- 關閉swap
swapoff -a
- 配置轉發相關引數，否則可能會出錯
cat <<EOF >/etc/sysctl.d/k8s.conf net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 vm.swappiness=0 EOF sysctl --system
- 載入ipvs模組
cat << EOF > /etc/sysconfig/modules/ipvs.modules #!/bin/bash ipvs_modules_dir="/usr/lib/modules/\`uname -r\`/kernel/net/netfilter/ipvs" for i in \`ls \$ipvs_modules_dir | sed-r 's#(.*).ko.*#\1#'\`; do /sbin/modinfo -F filename \$i&> /dev/null if [ \$? -eq 0 ]; then /sbin/modprobe \$i fi done EOF chmod +x /etc/sysconfig/modules/ipvs.modules bash /etc/sysconfig/modules/ipvs.modules
- 安裝cfssl
#在master節點安裝即可！！！ wget -O /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 wget -O /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 wget -O /bin/cfssl-certinfohttps://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 for cfssl in `ls /bin/cfssl*`;do chmod +x $cfssl;done;
- 安裝kubernetes阿里雲映象
cat << EOF > /etc/yum.repos.d/kubernetes.repo [kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/ enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF yum install -ykubelet kubeadm kubectl
- 安裝docker，並幹掉docker0網橋
wget -O /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce
mkdir /etc/docker/
cat << EOF > /etc/docker/daemon.json
{"registry-mirrors": ["https://registry.docker-cn.com"],
"live-restore": true,
"default-shm-size": "128M",
"bridge": "none",
"max-concurrent-downloads": 10,
"oom-score-adjust": -1000,
"debug": false
}
EOF
#重啟docker
systemctl daemon-reload
systemctl enable docker
systemctl restart docker
#可以忽略這步，後面kubeadm可以指定從阿里雲的映象倉庫中下載所需的image
#設定docker代理，以下載k8s所需要的images
mkdir /etc/systemd/system/docker.service.d/
cat << EOF > /etc/systemd/system/docker.service.d/http-proxy.conf
[Service]
Environment="HTTP_PROXY=http://192.168.0.6:1080/" "HTTPS_PROXY=http://192.168.0.6:1080/ "
Environment="NO_PROXY=localhost,127.0.0.1,1ti39uv1.mirror.aliyuncs.com,acs-cn-hangzhou-mirror.oss-cn-hangzhou.aliyuncs.com"
EOF
#重啟docker
systemctl daemon-reload
systemctl restart docker
- 配置hosts檔案
#為所有節點配置hosts檔案 192.168.1.1master 192.168.1.2master2 192.168.1.3master3 192.168.1.4lb1 192.168.1.5lb2 192.168.1.6etcd1 192.168.1.7etcd2 192.168.1.8etcd3 192.168.1.9node1 192.168.1.10node2
二、配置etcd
- 配置etcd的證書
mkdir -pv $HOME/ssl && cd $HOME/ssl
cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
EOF
cat > etcd-ca-csr.json << EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Shenzhen",
"L": "Shenzhen",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
EOF
cat > etcd-csr.json << EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.1.6",
"192.168.1.7",
"192.168.1.8"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Shenzhen",
"L": "Shenzhen",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
EOF
#生成證書並複製證書至其他etcd節點
cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
mkdir -pv /etc/etcd/ssl
mkdir -pv /etc/kubernetes/pki/etcd
cp etcd*.pem /etc/etcd/ssl
cp etcd*.pem /etc/kubernetes/pki/etcd
scp -r /etc/etcd 192.168.1.6:/etc/
scp -r /etc/etcd 192.168.1.7:/etc/
scp -r /etc/etcd 192.168.1.8:/etc/
- etcd1主機啟動etcd
yum install -y etcd cat << EOF > /etc/etcd/etcd.conf #[Member] #ETCD_CORS="" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" #ETCD_WAL_DIR="" ETCD_LISTEN_PEER_URLS="https://192.168.1.6:2380" ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.6:2379" #ETCD_MAX_SNAPSHOTS="5" #ETCD_MAX_WALS="5" ETCD_NAME="etcd1" #ETCD_SNAPSHOT_COUNT="100000" #ETCD_HEARTBEAT_INTERVAL="100" #ETCD_ELECTION_TIMEOUT="1000" #ETCD_QUOTA_BACKEND_BYTES="0" #ETCD_MAX_REQUEST_BYTES="1572864" #ETCD_GRPC_KEEPALIVE_MIN_TIME="5s" #ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s" #ETCD_GRPC_KEEPALIVE_TIMEOUT="20s" # #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.6:2380" ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.6:2379" #ETCD_DISCOVERY="" #ETCD_DISCOVERY_FALLBACK="proxy" #ETCD_DISCOVERY_PROXY="" #ETCD_DISCOVERY_SRV="" ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.6:2380,etcd2=https://192.168.1.7:2380,etcd3=https://192.168.1.8:2380" ETCD_INITIAL_CLUSTER_TOKEN="BigBoss" #ETCD_INITIAL_CLUSTER_STATE="new" #ETCD_STRICT_RECONFIG_CHECK="true" #ETCD_ENABLE_V2="true" # #[Proxy] #ETCD_PROXY="off" #ETCD_PROXY_FAILURE_WAIT="5000" #ETCD_PROXY_REFRESH_INTERVAL="30000" #ETCD_PROXY_DIAL_TIMEOUT="1000" #ETCD_PROXY_WRITE_TIMEOUT="5000" #ETCD_PROXY_READ_TIMEOUT="0" # #[Security] ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem" ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" #ETCD_CLIENT_CERT_AUTH="false" ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem" #ETCD_AUTO_TLS="false" ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem" ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" #ETCD_PEER_CLIENT_CERT_AUTH="false" ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem" #ETCD_PEER_AUTO_TLS="false" # #[Logging] #ETCD_DEBUG="false" #ETCD_LOG_PACKAGE_LEVELS="" #ETCD_LOG_OUTPUT="default" # #[Unsafe] #ETCD_FORCE_NEW_CLUSTER="false" # #[Version] #ETCD_VERSION="false" #ETCD_AUTO_COMPACTION_RETENTION="0" # #[Profiling] #ETCD_ENABLE_PPROF="false" #ETCD_METRICS="basic" # #[Auth] #ETCD_AUTH_TOKEN="simple" EOF chown -R etcd.etcd /etc/etcd systemctl enable etcd systemctl start etcd
- etcd2主機啟動etcd
yum install -y etcd cat << EOF > /etc/etcd/etcd.conf #[Member] #ETCD_CORS="" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" #ETCD_WAL_DIR="" ETCD_LISTEN_PEER_URLS="https://192.168.1.7:2380" ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.7:2379" #ETCD_MAX_SNAPSHOTS="5" #ETCD_MAX_WALS="5" ETCD_NAME="etcd2" #ETCD_SNAPSHOT_COUNT="100000" #ETCD_HEARTBEAT_INTERVAL="100" #ETCD_ELECTION_TIMEOUT="1000" #ETCD_QUOTA_BACKEND_BYTES="0" #ETCD_MAX_REQUEST_BYTES="1572864" #ETCD_GRPC_KEEPALIVE_MIN_TIME="5s" #ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s" #ETCD_GRPC_KEEPALIVE_TIMEOUT="20s" # #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.7:2380" ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.7:2379" #ETCD_DISCOVERY="" #ETCD_DISCOVERY_FALLBACK="proxy" #ETCD_DISCOVERY_PROXY="" #ETCD_DISCOVERY_SRV="" ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.6:2380,etcd2=https://192.168.1.7:2380,etcd3=https://192.168.1.8:2380" ETCD_INITIAL_CLUSTER_TOKEN="BigBoss" #ETCD_INITIAL_CLUSTER_STATE="new" #ETCD_STRICT_RECONFIG_CHECK="true" #ETCD_ENABLE_V2="true" # #[Proxy] #ETCD_PROXY="off" #ETCD_PROXY_FAILURE_WAIT="5000" #ETCD_PROXY_REFRESH_INTERVAL="30000" #ETCD_PROXY_DIAL_TIMEOUT="1000" #ETCD_PROXY_WRITE_TIMEOUT="5000" #ETCD_PROXY_READ_TIMEOUT="0" # #[Security] ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem" ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" #ETCD_CLIENT_CERT_AUTH="false" ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem" #ETCD_AUTO_TLS="false" ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem" ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" #ETCD_PEER_CLIENT_CERT_AUTH="false" ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem" #ETCD_PEER_AUTO_TLS="false" # #[Logging] #ETCD_DEBUG="false" #ETCD_LOG_PACKAGE_LEVELS="" #ETCD_LOG_OUTPUT="default" # #[Unsafe] #ETCD_FORCE_NEW_CLUSTER="false" # #[Version] #ETCD_VERSION="false" #ETCD_AUTO_COMPACTION_RETENTION="0" # #[Profiling] #ETCD_ENABLE_PPROF="false" #ETCD_METRICS="basic" # #[Auth] #ETCD_AUTH_TOKEN="simple" EOF chown -R etcd.etcd /etc/etcd systemctl enable etcd systemctl start etcd
- etcd3主機啟動etcd
yum install -y etcd cat << EOF > /etc/etcd/etcd.conf #[Member] #ETCD_CORS="" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" #ETCD_WAL_DIR="" ETCD_LISTEN_PEER_URLS="https://192.168.1.8:2380" ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.8:2379" #ETCD_MAX_SNAPSHOTS="5" #ETCD_MAX_WALS="5" ETCD_NAME="etcd3" #ETCD_SNAPSHOT_COUNT="100000" #ETCD_HEARTBEAT_INTERVAL="100" #ETCD_ELECTION_TIMEOUT="1000" #ETCD_QUOTA_BACKEND_BYTES="0" #ETCD_MAX_REQUEST_BYTES="1572864" #ETCD_GRPC_KEEPALIVE_MIN_TIME="5s" #ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s" #ETCD_GRPC_KEEPALIVE_TIMEOUT="20s" # #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.8:2380" ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.8:2379" #ETCD_DISCOVERY="" #ETCD_DISCOVERY_FALLBACK="proxy" #ETCD_DISCOVERY_PROXY="" #ETCD_DISCOVERY_SRV="" ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.6:2380,etcd2=https://192.168.1.7:2380,etcd3=https://192.168.1.8:2380" ETCD_INITIAL_CLUSTER_TOKEN="BigBoss" #ETCD_INITIAL_CLUSTER_STATE="new" #ETCD_STRICT_RECONFIG_CHECK="true" #ETCD_ENABLE_V2="true" # #[Proxy] #ETCD_PROXY="off" #ETCD_PROXY_FAILURE_WAIT="5000" #ETCD_PROXY_REFRESH_INTERVAL="30000" #ETCD_PROXY_DIAL_TIMEOUT="1000" #ETCD_PROXY_WRITE_TIMEOUT="5000" #ETCD_PROXY_READ_TIMEOUT="0" # #[Security] ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem" ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" #ETCD_CLIENT_CERT_AUTH="false" ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem" #ETCD_AUTO_TLS="false" ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem" ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" #ETCD_PEER_CLIENT_CERT_AUTH="false" ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem" #ETCD_PEER_AUTO_TLS="false" # #[Logging] #ETCD_DEBUG="false" #ETCD_LOG_PACKAGE_LEVELS="" #ETCD_LOG_OUTPUT="default" # #[Unsafe] #ETCD_FORCE_NEW_CLUSTER="false" # #[Version] #ETCD_VERSION="false" #ETCD_AUTO_COMPACTION_RETENTION="0" # #[Profiling] #ETCD_ENABLE_PPROF="false" #ETCD_METRICS="basic" # #[Auth] #ETCD_AUTH_TOKEN="simple" EOF chown -R etcd.etcd /etc/etcd systemctl enable etcd systemctl start etcd
- 檢查etcd叢集
etcdctl --endpoints "https://192.168.1.6:2379,https://192.168.1.7:2379,https://192.168.1.8:2379"--ca-file=/etc/etcd/ssl/etcd-ca.pem\ --cert-file=/etc/etcd/ssl/etcd.pem--key-file=/etc/etcd/ssl/etcd-key.pemcluster-health [root@node3 ~]# etcdctl --endpoints "https://192.168.1.6:2379,https://192.168.1.7:2379,https://192.168.1.8:2379"--ca-file=/etc/etcd/ssl/etcd-ca.pem\ > --cert-file=/etc/etcd/ssl/etcd.pem--key-file=/etc/etcd/ssl/etcd-key.pemcluster-health member 3639deb1869a1bda is healthy: got healthy result from https://127.0.0.1:2379 member b75e13f1faa57bd8 is healthy: got healthy result from https://127.0.0.1:2379 member e31fec5bb4c882f2 is healthy: got healthy result from https://127.0.0.1:2379
配置keepalived
- 在lb1機器上配置
yum install -y keepalived
cat << EOF > /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost#傳送郵箱
}
notification_email_from keepalived@localhost#郵箱地址
smtp_server 127.0.0.1#郵件伺服器地址
smtp_connect_timeout 30
router_id node1#主機名，每個節點不同即可
vrrp_mcast_group4 224.0.100.100#組播地址
}
vrrp_instance VI_1 {
state MASTER#在另一個節點上為BACKUP
interface eth0#IP地址漂移到的網絡卡
virtual_router_id 6 #多個節點必須相同
priority 100#優先順序，備用節點的值必須低於主節點的值
advert_int 1#通告間隔1秒
authentication {
auth_type PASS#預共享金鑰認證
auth_pass 571f97b2#金鑰
}
virtual_ipaddress {
192.168.1.100/24#VIP地址
}
}
EOF
systemctl enable keepalived
systemctl start keepalived
- 在lb2主機配置
yum install -y keepalived
cat << EOF > /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost#傳送郵箱
}
notification_email_from keepalived@localhost#郵箱地址
smtp_server 127.0.0.1#郵件伺服器地址
smtp_connect_timeout 30
router_id node2#主機名，每個節點不同即可
vrrp_mcast_group4 224.0.100.100#組播地址
}
vrrp_instance VI_1 {
state BACKUP#在另一個節點上為MASTER
interface eth0#IP地址漂移到的網絡卡
virtual_router_id 6 #多個節點必須相同
priority 80#優先順序，備用節點的值必須低於主節點的值
advert_int 1#通告間隔1秒
authentication {
auth_type PASS#預共享金鑰認證
auth_pass 571f97b2#金鑰
}
virtual_ipaddress {
192.168.1.100/24#漂移過來的IP地址
}
}
EOF
systemctl enable keepalived
systemctl start keepalived
配置Haproxy
- 在lb1主機上
yum install-y haproxy cat << EOF > /etc/haproxy/haproxy.cfg global log127.0.0.1 local2 chroot/var/lib/haproxy pidfile/var/run/haproxy.pid maxconn4000 userhaproxy grouphaproxy daemon defaults modetcp logglobal retries3 timeout connect10s timeout client1m timeout server1m frontend kubernetes bind *:6443 mode tcp default_backend kubernetes-master backend kubernetes-master balance roundrobin server master192.168.1.1:6443 check maxconn 2000 server master2 192.168.1.2:6443 check maxconn 2000 server master3 192.168.1.3:6443 check maxconn 2000 EOF systemctl enable haproxy systemctl start haproxy
- 在lb2主機上
yum install-y haproxy cat << EOF > /etc/haproxy/haproxy.cfg global log127.0.0.1 local2 chroot/var/lib/haproxy pidfile/var/run/haproxy.pid maxconn4000 userhaproxy grouphaproxy daemon defaults modetcp logglobal retries3 timeout connect10s timeout client1m timeout server1m frontend kubernetes bind *:6443 mode tcp default_backend kubernetes-master backend kubernetes-master balance roundrobin server master192.168.1.1:6443 check maxconn 2000 server master2 192.168.1.2:6443 check maxconn 2000 server master3 192.168.1.3:6443 check maxconn 2000 EOF systemctl enable haproxy systemctl start haproxy
初始化master
- 初始化master1
#kubeadm init配置檔案參考：https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init/#config-file cd $HOME cat << EOF > /root/kubeadm-init.yaml apiVersion: kubeadm.k8s.io/v1alpha2 kind: MasterConfiguration kubernetesVersion: v1.11.3# kubernetes的版本 api: advertiseAddress: 192.168.1.1 bindPort: 6443 controlPlaneEndpoint: 192.168.1.100:6443#VIP地址 apiServerCertSANs:#此處填所有的masterip和lbip和其它你可能需要通過它訪問apiserver的地址和域名或者主機名等 - master - master2 - master3 - 192.168.1.1 - 192.168.1.2 - 192.168.1.3 - 192.168.1.4 - 192.168.1.5 - 192.168.1.100 - 127.0.0.1 etcd:#ETCD的地址 external: endpoints: - "https://192.168.1.6:2379" - "https://192.168.1.7:2379" - "https://192.168.1.8:2379" caFile: /etc/kubernetes/pki/etcd/etcd-ca.pem certFile: /etc/kubernetes/pki/etcd/etcd.pem keyFile: /etc/kubernetes/pki/etcd/etcd-key.pem networking: podSubnet: 10.244.0.0/16# pod網路的網段 kubeProxy: config: mode: ipvs#啟用IPVS模式 featureGates: CoreDNS: true imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers# image的倉庫源 EOF systemctl enable kubelet kubeadm config images pull --config kubeadm-init.yaml docker tagregistry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1k8s.gcr.io/pause:3.1 kubeadm init --config /root/kubeadm-init.yaml mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config cat << EOF > /etc/profile.d/kubernetes.sh source <(kubectl completion bash) EOF source /etc/profile.d/kubernetes.sh scp -r /etc/kubernetes/pki 192.168.1.2:/etc/kubernetes/ scp -r /etc/kubernetes/pki 192.168.1.3:/etc/kubernetes/
- 初始化master2
cd /etc/kubernetes/pki/ rm -fr apiserver.crt apiserver.key cd $HOME cat << EOF > /root/kubeadm-init.yaml apiVersion: kubeadm.k8s.io/v1alpha2 kind: MasterConfiguration kubernetesVersion: v1.11.3# kubernetes的版本 api: advertiseAddress: 192.168.1.2 bindPort: 6443 controlPlaneEndpoint: 192.168.1.100:6443 apiServerCertSANs:#此處填所有的masterip和lbip和其它你可能需要通過它訪問apiserver的地址和域名或者主機名等 - master - master2 - master3 - 192.168.1.1 - 192.168.1.2 - 192.168.1.3 - 192.168.1.4 - 192.168.1.5 - 192.168.1.100 - 127.0.0.1 etcd: external: endpoints: - "https://192.168.1.6:2379" - "https://192.168.1.7:2379" - "https://192.168.1.8:2379" caFile: /etc/kubernetes/pki/etcd/etcd-ca.pem certFile: /etc/kubernetes/pki/etcd/etcd.pem keyFile: /etc/kubernetes/pki/etcd/etcd-key.pem networking: podSubnet: 10.244.0.0/16# pod網路的網段 kubeProxy: config: mode: ipvs featureGates: CoreDNS: true imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers# image的倉庫源 EOF systemctl enable kubelet kubeadm config images pull --config kubeadm-init.yaml docker tagregistry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1k8s.gcr.io/pause:3.1 kubeadm init --config /root/kubeadm-init.yaml mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config cat << EOF > /etc/profile.d/kubernetes.sh source <(kubectl completion bash) EOF source /etc/profile.d/kubernetes.sh
- 初始化master3
cd /etc/kubernetes/pki/ rm -fr apiserver.crt apiserver.key cd $HOME cat << EOF > /root/kubeadm-init.yaml apiVersion: kubeadm.k8s.io/v1alpha2 kind: MasterConfiguration kubernetesVersion: v1.11.3# kubernetes的版本 api: advertiseAddress: 192.168.1.3 bindPort: 6443 controlPlaneEndpoint: 192.168.1.100:6443 apiServerCertSANs:#此處填所有的masterip和lbip和其它你可能需要通過它訪問apiserver的地址和域名或者主機名等 - master - master2 - master3 - 192.168.1.1 - 192.168.1.2 - 192.168.1.3 - 192.168.1.4 - 192.168.1.5 - 192.168.1.100 - 127.0.0.1 etcd: external: endpoints: - "https://192.168.1.6:2379" - "https://192.168.1.7:2379" - "https://192.168.1.8:2379" caFile: /etc/kubernetes/pki/etcd/etcd-ca.pem certFile: /etc/kubernetes/pki/etcd/etcd.pem keyFile: /etc/kubernetes/pki/etcd/etcd-key.pem networking: podSubnet: 10.244.0.0/16# pod網路的網段 kubeProxy: config: mode: ipvs featureGates: CoreDNS: true imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers# image的倉庫源 EOF systemctl enable kubelet kubeadm config images pull --config kubeadm-init.yaml docker tagregistry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1k8s.gcr.io/pause:3.1 kubeadm init --config /root/kubeadm-init.yaml mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config cat << EOF > /etc/profile.d/kubernetes.sh source <(kubectl completion bash) EOF source /etc/profile.d/kubernetes.sh
將所有node節點加入叢集
- 獲取加入叢集的token
#在master主機執行獲取join命令 kubeadm token create --print-join-command [root@master ~]# kubeadm token create --print-join-command kubeadm join 192.168.1.100:6443 --token zpru0r.jkvrdyy2caexr8kk --discovery-token-ca-cert-hash sha256:a45c091dbd8a801152aacd877bcaaaaf152697bfa4536272c905a83612b3bf22
- 在所有node執行
systemctl enable kubelet.service kubeadm join 192.168.1.100:6443 --token zpru0r.jkvrdyy2caexr8kk --discovery-token-ca-cert-hash sha256:a45c091dbd8a801152aacd877bcaaaaf152697bfa4536272c905a83612b3bf22 [root@node6 ~]# kubeadm join 192.168.1.100:6443 --token zpru0r.jkvrdyy2caexr8kk --discovery-token-ca-cert-hash sha256:a45c091dbd8a801152aacd877bcaaaaf152697bfa4536272c905a83612b3bf22 [preflight] running pre-flight checks I0913 15:33:17.4290691907 kernel_validator.go:81] Validating kernel version I0913 15:33:17.4293351907 kernel_validator.go:96] Validating kernel config [WARNING SystemVerification]: docker version is greater than the most recently validated version. Docker version: 18.06.1-ce. Max validated version: 17.03 [discovery] Trying to connect to API Server "192.168.1.100:6443" [discovery] Created cluster-info discovery client, requesting info from "https://192.168.1.100:6443" [discovery] Requesting info from "https://192.168.1.100:6443" again to validate TLS against the pinned public key [discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server "192.168.1.100:6443" [discovery] Successfully established connection with API Server "192.168.1.100:6443" [kubelet] Downloading configuration for the kubelet from the "kubelet-config-1.11" ConfigMap in the kube-system namespace [kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" [kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env" [preflight] Activating the kubelet service [tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap... [patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "node6" as an annotation This node has joined the cluster: * Certificate signing request was sent to master and a response was received. * The Kubelet was informed of the new secure connection details. Run 'kubectl get nodes' on the master to see this node join the cluster. [root@node1 ~]#
- 檢視節點
#在master上執行 kubectl get node [root@master ~]# kubectl get node NAMESTATUSROLESAGEVERSION masterNotReadymaster2mv1.11.3 master2NotReadymaster1mv1.11.3 master3NotReadymaster1mv1.11.3 node1NotReady<none>18sv1.11.3 node2NotReady<none>12sv1.11.3
配置網路
- 使用flannel網路
cd /root/ mkdir flannel cd flannel wget https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml kubectl apply -f kube-flannel.yml #因為是拉去google上的image，可能會拉不下來，自己想辦法吧~~
- 檢視node狀態
kubectl get pod -n kube-system kubectl get node [root@master ~]# kubectl get pod -n kube-system NAMEREADYSTATUSRESTARTSAGE coredns-777d78ff6f-c5b9h1/1Running027m coredns-777d78ff6f-fv4fw1/1Running027m kube-apiserver-master1/1Running026m kube-apiserver-master21/1Running026m kube-apiserver-master31/1Running025m kube-controller-manager-master1/1Running026m kube-controller-manager-master21/1Running026m kube-controller-manager-master31/1Running025m kube-flannel-ds-4hd6r1/1Running09m kube-flannel-ds-g9tvn1/1Running09m kube-flannel-ds-gnrlc1/1Running09m kube-flannel-ds-kkswt1/1Running09m kube-flannel-ds-n7sqv1/1Running29m kube-proxy-7fpbb1/1Running025m kube-proxy-89g7s1/1Running026m kube-proxy-b8glx1/1Running027m kube-proxy-c6qj71/1Running026m kube-proxy-xn4k71/1Running025m kube-scheduler-master1/1Running026m kube-scheduler-master21/1Running026m kube-scheduler-master31/1Running025m #當上面的kube-flannel-ds-xxxx的容器都處於Running狀態時，node的狀態應該是Ready [root@master ~]# kubectl get node NAMESTATUSROLESAGEVERSION masterReadymaster32mv1.11.3 master2Readymaster31mv1.11.3 master3Readymaster31mv1.11.3 node1Ready<none>30mv1.11.3 node2Ready<none>30mv1.11.3
測試
- 建立一個nginx，測試應用和dns是否正常
cd /root && mkdir nginx && cd nginx cat << EOF > nginx.yaml --- apiVersion: v1 kind: Service metadata: name: nginx spec: selector: app: nginx type: NodePort ports: - port: 80 nodePort: 31000 name: nginx-port targetPort: 80 protocol: TCP --- apiVersion: apps/v1 kind: Deployment metadata: name: nginx spec: replicas: 2 selector: matchLabels: app: nginx template: metadata: name: nginx labels: app: nginx spec: containers: - name: nginx image: nginx ports: - containerPort: 80 EOF kubectl apply -f nginx.yaml
- 建立一個POD來測試DNS解析
kubectl run curl --image=radial/busyboxplus:curl -i --tty
nslookup kubernetes
nslookup nginx
curl nginx
exit
kubectl delete deployment curl
[root@master nginx]# kubectl run curl --image=radial/busyboxplus:curl -i --tty
If you don't see a command prompt, try pressing enter.
nslookup kubernetes
Server:10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name:kubernetes
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local
[ root@curl-87b54756-w99h6:/ ]$ nslookup nginx
Server:10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name:nginx
Address 1: 10.103.202.146 nginx.default.svc.cluster.local
[ root@curl-87b54756-w99h6:/ ]$ curl nginx
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
[ root@curl-87b54756-w99h6:/ ]$
- 測試master高可用
#將master:192.168.1.1關掉 init 0 #切換至master2 #執行get node kubectl get node #master已經宕機了！！！！ [root@master2 ~]# kubectl get node NAMESTATUSROLESAGEVERSION masterNotReadymaster1hv1.11.3 master2Readymaster59mv1.11.3 master3Readymaster59mv1.11.3 node1Ready<none>58mv1.11.3 node2Ready<none>58mv1.11.3 #重新建立一個pod，看看是否能建立成功 kubectl run curl --image=radial/busyboxplus:curl -i --tty exit kubectl delete deployment curl [root@master2 ~]# kubectl run curl --image=radial/busyboxplus:curl -i --tty If you don't see a command prompt, try pressing enter. [ root@curl-87b54756-xfgrn:/ ]$ [ root@curl-87b54756-xfgrn:/ ]$ [ root@curl-87b54756-xfgrn:/ ]$ [ root@curl-87b54756-xfgrn:/ ]$ [ root@curl-87b54756-xfgrn:/ ]$ date Thu Sep 13 09:41:31 UTC 2018
- 測試Haproxy高可用
#抓個包看看現在VIP在哪臺機器上，然後去關掉這臺機器 tcpdump -nn host 224.0.100.100 [root@master2 ~]# tcpdump -nn host 224.0.100.100 tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 17:45:59.768033 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:46:00.769503 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:46:01.771062 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 ^C #現在VIP是在192.168.1.4這臺主機上，關掉這臺機器 init 0 #關掉之後發現192.168.1.5立馬接管了。現在VIP是在1.5上 [root@node3 ~]# tcpdump -nn host 224.0.100.100 tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 17:48:25.031679 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:26.033805 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:27.035313 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:28.036628 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:29.039011 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:30.041249 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:31.043065 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:32.045007 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:33.046781 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:34.048776 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:35.051280 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20 17:48:35.929482 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 0, authtype simple, intvl 1s, length 20 17:48:36.618749 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20 17:48:37.699849 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20 17:48:38.700669 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20 17:48:39.702840 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20 17:48:40.704254 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20 17:48:41.706221 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20 17:48:42.706478 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20 ^C 19 packets captured 326 packets received by filter 172 packets dropped by kernel #切換到master2，再建立個POD試試 kubectl run curl --image=radial/busyboxplus:curl -i --tty date exit kubectl delete deployment curl [root@master2 ~]# kubectl run curl --image=radial/busyboxplus:curl -i --tty If you don't see a command prompt, try pressing enter. [ root@curl-87b54756-xfgrn:/ ]$ date Thu Sep 13 09:50:58 UTC 2018 [ root@curl-87b54756-xfgrn:/ ]$ [ root@curl-87b54756-xfgrn:/ ]$ exit Session ended, resume using 'kubectl attach curl-87b54756-xfgrn -c curl -i -t' command when the pod is running [root@master2 ~]# kubectl delete deployment curl deployment.extensions "curl" deleted