跟着炎炎盐实践k8s---Kubernetes 1.16.10 二进制高可用集群部署之通用配置

       文档将介绍如何使用二进制部署Kubernetes v1.16高可用集群，而不是使用自动化部署(kubeadm)集群。在部署过程中，将详细列出各个组件启动参数，以及相关配置说明。部署完成后，将理解k8s各个组件的交互原理，并且可以快速解决实际问题。

 

* 环境及组件版本

    Centos7.2

    Kubernetes v1.16.10

    Docker 18.09 .01(需要和k8s版本匹配)

    Etcd 3.3.13

    Flanneld 0.12.0

 

• 一、 环境准备

1. 服务器准备

     master节点

10.13.33.29  master-01
10.13.33.40  master-02
10.13.33.38  master-03

     node节点

10.13.33.31  node-01

1. 修改主机名

     为对应服务器设置永久主机名

     

cat >> /etc/hosts <<EOF
10.13.33.29  master-01
10.13.33.40  master-02
10.13.33.38  master-03
10.13.33.31  node-01
EOF

hostnamectl set-hostname master-01  #所有机器按照要求修改
bash        #刷新主机名

1. 更新PATH变量

           k8s所有文件和配置都会存储在/opt/k8s/ 中,运行以下命令，配置环境变量

echo 'PATH=/opt/k8s/bin:$PATH' >>/etc/profile    ##将k8s命令添加到环境变量中方便调用
source  /etc/profil

•      二、服务器优化

          所有节点配置

          如果不能访问互联网请使用以下代理配置

echo "proxy=http://ip：port" >>/etc/yum.conf
export http_proxy="http://ip：port"

        1、 创建相关目录

mkdir -p  /opt/k8s/{bin,work} /etc/{kubernetes,etcd}/cert
cd /opt/k8s/work
wget  wget https://github.com/kubernetes/kubernetes/archive/v1.16.10.tar.gz ##下载k8s

        2、 服务器时间写入硬件

timedatectl set-timezone Asia/Shanghai
timedatectl set-local-rtc 0

1.     3、配置时钟服务器地址

echo "
# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst

# Ignore stratum in source selection.
stratumweight 0

# Record the rate at which the system clock gains/losses time.
driftfile /var/lib/chrony/drift

# Enable kernel RTC synchronization.
rtcsync

# In first three updates step the system clock instead of slew
# if the adjustment is larger than 10 seconds.
makestep 10 3

# Allow NTP client access from local network.
#allow 192.168/16

# Listen for commands only on localhost.
bindcmdaddress 127.0.0.1
bindcmdaddress ::1

# Serve time even if not synchronized to any NTP server.
#local stratum 10

keyfile /etc/chrony.keys

# Specify the key used as password for chronyc.
commandkey 1

# Generate command key if missing.
generatecommandkey

# Disable logging of client accesses.
noclientlog

# Send a message to syslog if a clock adjustment is larger than 0.5 seconds.
logchange 0.5

logdir /var/log/chrony
#log measurements statistics tracking
server 时钟ip1 iburst prefer
server 时钟ip2 iburst
server 127.127.1.0
allow 0/0
local stratum 11
" > /etc/chrony.conf

systemctl restart chronyd                    ##重启时钟应用配置
systemctl status chronyd                     ##查看启动状态
systemctl enable chronyd                     ##开机自动启动
firewall-cmd --add-service=ntp --permanent   ##放开对应的防火墙
firewall-cmd --add-service=ntp               ##放开对应的防火墙
systemctl restart rsyslog 
systemctl restart crond
timedatectl status
chronyc sources -v                           ##验证时钟地址

             4、关闭其他非必要服务

systemctl stop postfix && systemctl disable postfix

              5、 持久化保存日志的目录

mkdir /var/log/journal  #持久化保存日志的目录
mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf << EOF
[Journal]
Storage=persistent
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
SystemMaxUse=10G
SystemMaxFileSize=200M
MaxRetentionSec=2week
ForwardToSyslog=no
EOF
systemctl restart systemd-journald

             6、关闭防火墙

systemctl stop firewalld
systemctl disable firewalld

             7、优化iptables

iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat
iptables -P FORWARD ACCEPT

             8、关闭swap及SELINUX

swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
#如果开启了swap分区，kubelet会启动失败(可以通过设置参数——-fail-swap-on设置为false)

             9、优化内核等

echo "
net.bridge.bridge-nf-call-ip6tables=1
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1
net.ipv4.conf.all.forwarding=1
net.ipv4.neigh.default.gc_thresh1=4096
net.ipv4.neigh.default.gc_thresh2=6144
net.ipv4.neigh.default.gc_thresh3=8192
net.ipv4.neigh.default.gc_interval=60
net.ipv4.neigh.default.gc_stale_time=120
# 参考 https://github.com/prometheus/node_exporter#disabled-by-default
kernel.perf_event_paranoid=-1
#sysctls for k8s node config
net.ipv4.tcp_slow_start_after_idle=0
net.core.rmem_max=16777216
fs.inotify.max_user_watches=524288
kernel.softlockup_all_cpu_backtrace=1
kernel.softlockup_panic=0
kernel.watchdog_thresh=30
fs.file-max=2097152
fs.inotify.max_user_instances=8192
fs.inotify.max_queued_events=16384
vm.max_map_count=262144
fs.may_detach_mounts=1
net.core.netdev_max_backlog=16384
net.ipv4.tcp_wmem=4096 12582912 16777216
net.core.wmem_max=16777216
net.core.somaxconn=32768
net.ipv4.ip_forward=1
net.ipv4.tcp_max_syn_backlog=8096
net.ipv4.tcp_rmem=4096 12582912 16777216
net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.default.disable_ipv6=1
net.ipv6.conf.lo.disable_ipv6=1
kernel.yama.ptrace_scope=0
vm.swappiness=0
# 可以控制core文件的文件名中是否添加pid作为扩展。
kernel.core_uses_pid=1
# Do not accept source routing
net.ipv4.conf.default.accept_source_route=0
net.ipv4.conf.all.accept_source_route=0
# Promote secondary addresses when the primary address is removed
net.ipv4.conf.default.promote_secondaries=1
net.ipv4.conf.all.promote_secondaries=1
# Enable hard and soft link protection
fs.protected_hardlinks=1
fs.protected_symlinks=1
# 源路由验证
# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce=2
net.ipv4.conf.all.arp_announce=2
# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets=5000
net.ipv4.tcp_syncookies=1
net.ipv4.tcp_fin_timeout=30
net.ipv4.tcp_synack_retries=2
kernel.sysrq=1
" >> /etc/sysctl.conf

sysctl -p

###nofile
cat >> /etc/security/limits.conf <<EOF
* soft nofile 65535
* hard nofile 65536
EOF

            10、更新内核版本

echo  Kernel_Version
uname -srm
export Kernel_Version=4.18.9-1
wget  http://mirror.rc.usf.edu/compute_lock/elrepo/kernel/el7/x86_64/RPMS/kernel-ml{,-devel}-${Kernel_Version}.el7.elrepo.x86_64.rpm
yum localinstall -y kernel-ml*
grub2-set-default 'CentOS Linux (4.18.9-1.el7.elrepo.x86_64) 7 (Core)'
shutdown -r -t 0                      ##如果没有自行重启，进行手动重启
uname -srm

             11、使用systemd-modules-load加载内核模块

touch /etc/sysconfig/modules/ipvs.modules

cat > /etc/sysconfig/modules/ipvs.modules << EOF 
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

•       三、部署安装k8s组件

            1、制作证书

##安装cfssl工作集
mkdir -p /opt/k8s/cert && cd /opt/k8s
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
mv cfssl_linux-amd64 /opt/k8s/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
mv cfssljson_linux-amd64 /opt/k8s/bin/cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
mv cfssl-certinfo_linux-amd64 /opt/k8s/bin/cfssl-certinfo
chmod +x /opt/k8s/bin/*
export PATH=/opt/k8s/bin:$PATH

##创建更证书及配置文件
cd  /opt/k8s
mv ~/cfssl* ./
mv cfssl_linux-amd64 /opt/k8s/bin/cfssl
mv cfssljson_linux-amd64 /opt/k8s/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /opt/k8s/bin/cfssl-certinfo
chmod +x /opt/k8s/bin/*
export PATH=/opt/k8s/bin:$PATH

##master1
cd /opt/k8s/work
cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF

##创建证书签名请求文件
cat > ca-csr.json <<EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ],
  "ca": {
    "expiry": "876000h"
 }
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

##生成CA证书和私钥
cd /opt/k8s/work
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*
##将证书分发到各个节点

##创建flannel证书
cat > flanneld-csr.json <<EOF
{
  "CN": "flanneld",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ]
}
EOF

cfssl gencert -ca=/opt/k8s/work/ca.pem \
  -ca-key=/opt/k8s/work/ca-key.pem \
  -config=/opt/k8s/work/ca-config.json \
  -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
ls flanneld*pem
mkdir -p /etc/flanneld/cert
cp flanneld*.pem /etc/flanneld/cert              ##分发到各个节点

1.         2、部署flannel

###安装flannel

cd /opt/k8s/work/
mkdir flannel
cp -f ~/k8s/ca*.pem /etc/kubernetes/cert/
wget https://github.com/coreos/flannel/releases/download/v0.12.0/flannel-v0.12.0-linux-amd64.tar.gz
tar -xzvf flannel-v0.12.0-linux-amd64.tar.gz -C flannel
cp flannel/{flanneld,mk-docker-opts.sh} /opt/k8s/bin/
chmod +x /opt/k8s/bin/*

##创建flanneld.service
touch /etc/systemd/system/flanneld.service
cat > /etc/systemd/system/flanneld.service << EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
ExecStart=/opt/k8s/bin/flanneld \\
  -etcd-cafile=/etc/kubernetes/cert/ca.pem \\
  -etcd-certfile=/etc/flanneld/cert/flanneld.pem \\
  -etcd-keyfile=/etc/flanneld/cert/flanneld-key.pem \\
  -etcd-endpoints=https://10.13.33.29:2379,https://10.13.33.38:2379,https://10.13.33.40:2379 \\
  -etcd-prefix=/kubernetes/network \\
  -iface=eth0 \\
  -ip-masq
ExecStartPost=/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=always
RestartSec=5
StartLimitInterval=0
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF

##启动flannel
systemctl daemon-reload && systemctl enable flanneld && systemctl restart flanneld || systemctl status flanneld|grep Active
systemctl status flanneld             ##查看flanneld运行状态，依赖docker可能会启动失败，后面安装docker后会启动成功
ip addr show|grep flannel.1

            3、安装docker

##更新yum源
mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/backup/
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum update -y
yum install -y yum-utils device-mapper-persistent-data lvm2 nfs-utils                              ##安装依赖包
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo     ##使用阿里云的docker镜像站
yum makecache fast

##安装指定版本的docker
yum -y install docker-ce-18.09.0-3.el7 docker-ce-cli-18.09.0-3.el7 containerd.io

##优化docker.service
cat > /usr/lib/systemd/system/docker.service <<EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
BindsTo=containerd.service
After=network-online.target firewalld.service flanneld.service                              ##使用flannel子网
Wants=network-online.target
Requires=flanneld.service                                                                   ##依赖flanneld
#Requires=docker.socket

[Service]
OOMScoreAdjust=-1000                                                                        ##防止出现oom
Type=notify
EnvironmentFile=-/run/flannel/docker
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
#ExecStart=/usr/bin/dockerd -H fd://
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
ExecStartPost=/usr/sbin/iptables -P FORWARD ACCEPT                                          ##开启 iptables 转发链
TimeoutSec=0
RestartSec=2
Restart=always

# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3

# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this option.
TasksMax=infinity

# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes

# kill only the docker process, not all processes in the cgroup
KillMode=process

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl restart docker  && systemctl enable docker  && systemctl status docker    ##启动docker
systemctl start rpcbind && systemctl start nfs
创建docker启动配置文件
touch /etc/docker/daemon.json
cat > /etc/docker/daemon.json <<EOF
{
    "oom-score-adjust": -1000,
    "log-driver": "json-file",         ##日志配置，每个容器只保存3个100m的日志文件
    "log-opts": {
    "max-size": "100m",
    "max-file": "3"
    },
    "max-concurrent-downloads": 10,
    "max-concurrent-uploads": 10,
    "insecure-registries":["ip:port"],
    "registry-mirrors": ["http://ip:port"],
    "storage-driver": "overlay2",
    "storage-opts": [
    "overlay2.override_kernel_check=true"
    ]
}
EOF

1. 
   

来源：oschina

链接：https://my.oschina.net/u/4392508/blog/4332069