下载confd 二进制文件
# 创建目录方便存放文件
mkdir confd
# 进入新创建的目录
cd confd
# 下载 confd
wget https://github.com/kelseyhightower/confd/releases/download/v0.16.0/confd-0.16.0-linux-amd64
# 重命名
mv confd-0.16.0-linux-amd64 confd
# 给confd 可执行权限
chmod +x confd生成confd 配置
# 创建confd 配置目录
mkdir -p ./conf.d
# 创建模版存放目录
mkdir -p ./templates
# 生成confd 配置文件
cat << EOF | tee ./conf.d/nginx.toml
[template]
src = "nginx.tmpl"
dest = "/etc/nginx/nginx.conf"
keys = [
"CP_HOSTS",
]
EOF
# 生成模版文件
cat << EOF | tee ./templates/nginx.tmpl
error_log stderr notice;
worker_processes auto;
events {
multi_accept on;
use epoll;
worker_connections 4096;
}
stream {
upstream kube_apiserver {
{{ \$servers := split (getenv "CP_HOSTS") "," }}{{range \$servers}}
server {{.}}:6443;
{{end}}
}
server {
listen 6443;
proxy_pass kube_apiserver;
proxy_timeout 30;
proxy_connect_timeout 2s;
}
}
EOF
# 生成启动文件
cat << EOF | tee ./nginx-proxy
#!/bin/sh
# Run confd
confd -onetime -backend env
# Start nginx
nginx -g 'daemon off;'
EOF
# 给启动文件执行权限
chmod +x ./nginx-proxyDockerfile
vim Dockerfile
# 基础镜像
FROM alpine
# 作者信息
MAINTAINER nginx 1.17.9 Docker Maintainers "87984115@qq.com"
# 修改源
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
# 安装ca 证书
RUN apk update && \
apk add --no-cache ca-certificates
# 设置环境变量
ENV NGINX_VERSION 1.17.9
ENV OPENSSL_VERSION 1.1.1e
# 编译安装NGINX
WORKDIR /tmp
RUN NGINX_CONFIG="\
--prefix=/etc/nginx \
--sbin-path=/usr/sbin/nginx \
--conf-path=/etc/nginx/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/run/nginx.pid \
--lock-path=/var/run/nginx.lock \
--http-client-body-temp-path=/var/cache/nginx/client_temp \
--http-proxy-temp-path=/var/cache/nginx/proxy_temp \
--http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \
--http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \
--http-scgi-temp-path=/var/cache/nginx/scgi_temp \
--with-pcre \
--user=nginx \
--group=nginx \
--with-compat \
--with-file-aio \
--with-threads \
--with-http_addition_module \
--with-http_auth_request_module \
--with-http_dav_module \
--with-http_flv_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_mp4_module \
--with-http_random_index_module \
--with-http_realip_module \
--with-http_secure_link_module \
--with-http_slice_module \
--with-http_ssl_module \
--with-http_stub_status_module \
--with-http_sub_module \
--with-http_v2_module \
--with-ipv6 \
--with-openssl=../openssl-$OPENSSL_VERSION \
--with-openssl-opt=enable-tls1_3 \
--with-mail \
--with-mail_ssl_module \
--with-stream \
--with-stream_realip_module \
--with-stream_ssl_module \
--with-stream_ssl_preread_module \
--with-ld-opt=-Wl,--as-needed \
" \
&& addgroup -S nginx \
&& adduser -D -S -h /www -s /sbin/nologin -G nginx nginx \
&& apk add --no-cache --virtual .build-deps \
gcc \
libc-dev \
make \
pcre-dev \
zlib-dev \
linux-headers \
curl \
gnupg \
libxslt-dev \
gd-dev \
geoip-dev \
libstdc++ wget \
libjpeg \
libpng \
libpng-dev \
freetype \
freetype-dev \
libxml2 \
libxml2-dev \
curl-dev \
libmcrypt \
libmcrypt-dev \
autoconf \
libjpeg-turbo-dev \
libmemcached \
libmemcached-dev \
gettext \
gettext-dev \
libzip \
git \
libzip-dev \
&& curl -fSL https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz -o /tmp/openssl-$OPENSSL_VERSION.tar.gz \
&& curl -fSL https://nginx.org/download/nginx-$NGINX_VERSION.tar.gz -o /tmp/nginx-$NGINX_VERSION.tar.gz \
&& cd /tmp \
&& tar -xzf openssl-$OPENSSL_VERSION.tar.gz \
&& tar -xzf nginx-$NGINX_VERSION.tar.gz \
&& cd /tmp/nginx-$NGINX_VERSION \
&& ./configure $NGINX_CONFIG \
&& make -j$(getconf _NPROCESSORS_ONLN) \
&& make install
# 构建confd nginx 镜像
FROM alpine
# 作者信息
MAINTAINER nginx 1.17.9 Docker Maintainers "87984115@qq.com"
# 修改源
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
# 安装ca 证书
RUN apk update && \
apk add --no-cache ca-certificates
# 设置环境变量
ENV NGINX_VERSION 1.17.9
ENV OPENSSL_VERSION 1.1.1e
RUN mkdir -p /var/lib/nginx/cache \
&& apk add --no-cache \
curl \
wget \
pcre \
&& addgroup -S nginx \
&& adduser -D -S -h /var/lib/nginx -s /sbin/nologin -G nginx nginx \
&& chown -R nginx:nginx /var/lib/nginx \
&& mkdir -p /var/log/nginx \
&& rm -rf /var/cache/apk/* \
&& mkdir -p /etc/confd \
&& mkdir -p /var/cache/nginx/client_temp
#COPY 编译结果
COPY --from=0 /usr/sbin/nginx /usr/sbin/nginx
COPY --from=0 /etc/nginx /etc/nginx
ADD confd /usr/sbin/confd
ADD conf.d /etc/confd/conf.d
ADD templates /etc/confd/templates
ADD nginx-proxy /usr/bin/nginx-proxy
STOPSIGNAL SIGTERM
ENTRYPOINT ["/usr/bin/nginx-proxy"]
生成镜像
[root@nginx-1 confd]# tree
.
|-- Dockerfile
|-- conf.d
| `-- nginx.toml
|-- confd
|-- nginx-proxy
`-- templates
`-- nginx.tmpl
2 directories, 5 files
# 生成镜像
docker build -t ha-tools:v1.17.9 . # 镜像名字自己修改 我这里以ng 版本为tag
# 给进行打新tag
docker tag ha-tools:v1.17.9 juestnow/ha-tools:v1.17.9
# 上传镜像
docker push juestnow/ha-tools:v1.17.9测试生成的镜像
# 单个IP
docker run -tid --network=host --name=ha-proxy -e "CP_HOSTS=192.168.2.175" juestnow/ha-tools:v1.17.9s CP_HOSTS=192.168.2.175
# 多个IP
docker run -tid --network=host --name=ha-proxy -e "CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177" juestnow/ha-tools:v1.17.9 CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177
# 进去容器查看是否正常
docker ps
docker exec -ti 27733e5f9a97 /bin/sh
/ # ps -ef
PID USER TIME COMMAND
1 root 0:00 {nginx-proxy} /bin/sh /usr/bin/nginx-proxy CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177
12 root 0:00 nginx: master process nginx -g daemon off;
13 nginx 0:00 nginx: worker process
14 nginx 0:00 nginx: worker process
15 nginx 0:00 nginx: worker process
16 nginx 0:00 nginx: worker process
17 root 0:00 /bin/sh
22 root 0:00 ps -ef
# 查看端口监听
/ # netstat -tnlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:6443 0.0.0.0:* LISTEN 12/nginx: master pr
# 验证访问
/ # curl -k https://127.0.0.1:6443
{
"kind": "Status",
"apiVersion": "v1",
"metadata": {
},
"status": "Failure",
"message": "Unauthorized",
"reason": "Unauthorized",
"code": 401
}/ #
代理正常有数据返回k8s 使用 ha-tools
# kube-apiserver 节点不部署 ha-tools 只是node 节点部署
# 二进制部署kube-apiserver 证书签名时加上127.0.0.1 这个IP 以后整个集群访问都走127.0.0.1 这个IP+端口 同时kube-apiserver 改成0.0.0.0如果不修改master 安装kubelet 的时候记得修改IP
# kubeadm 安装时 请加入apiserver-cert-extra-sans=127.0.0.1 这样才能127.0.0.1 访问不然会一致报错
# 每个node 节点运行
docker run -tid --network=host --name=ha-proxy -e "CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177" juestnow/ha-tools:v1.17.9 CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177
# 还可以放到kubelet manifests 目录
[root@nginx-1 manifests]# cat ha-tools.yaml
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
component: ha-tools
tier: control-plane
name: ha-tools
namespace: kube-system
spec:
containers:
- args:
- "CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177"
image: juestnow/ha-tools:v1.17.9
imagePullPolicy: IfNotPresent
name: ha-tools
env:
- name: CP_HOSTS
value: "192.168.2.175,192.168.2.176,192.168.2.177"
hostNetwork: true
priorityClassName: system-cluster-critical
status: {}
# 二进制方式部署推荐使用以上的方式
[root@localhost ~]# kubectl get pod -A | grep ha-tools
kube-system ha-tools-nginx-1 1/1 Running 0 14h
# kubeadm 方式部署高可用修改kube-proxy 让它连接127.0.0.1
kubectl -n kube-system edit configmaps kube-proxy
# 二进制部署直接在 kubeconfig 添加就可以
来源：51CTO
作者：juestnow
链接：https://blog.51cto.com/juestnow/2479933