集群版本:
redis-4.0.14
集群节点:
| 节点角色 | IP |
|---|---|
| redis-master | 10.100.8.21 |
| redis-slave1 | 10.100.8.22 |
| redis-slave2 | 10.100.8.23 |
| redis-sentinel01 | 10.100.8.24 |
| redis-sentinel02 | 10.100.8.25 |
| redis-sentinel03 | 10.100.8.26 |
主从部署
全部节点执行：
useradd -M -s /sbin/nologin redis;
mkdir /srv/{app,data,logs}/redis;
cd /srv/tmp/
wget http://download.redis.io/releases/redis-4.0.14.tar.gz
tar -zxf redis-4.0.14.tar.gz
cd redis-4.0.14
make MALLOC=libc
make PREFIX=/srv/app/redis install
mkdir /srv/app/redis/conf;
chown -Rf redis:redis /srv/{app,data,logs}/redis;
修改redis-master配置文件/srv/app/redis/conf/redis.conf,这里我们开启了AOF持久化，并启用了密码:
bind 0.0.0.0
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize no
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile /srv/logs/redis/redis.log
databases 16
requirepass "d03fatznjxp6wkk6qvfg5tqjc5u5buod7cc8se7yzdyai9x0"
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /srv/data/redis
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
修改master-slave1从节点的配置/srv/app/redis/conf/redis.conf,配置slaveof:
bind 0.0.0.0
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize no
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile /srv/logs/redis/redis.log
databases 16
requirepass "d03fatznjxp6wkk6qvfg5tqjc5u5buod7cc8se7yzdyai9x0"
slaveof 10.100.8.21 6379
masterauth "d03fatznjxp6wkk6qvfg5tqjc5u5buod7cc8se7yzdyai9x0"
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /srv/data/redis
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
修改master-slave2从节点的配置/srv/app/redis/conf/redis.conf,配置slaveof:
bind 0.0.0.0
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize no
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile /srv/logs/redis/redis.log
databases 16
requirepass "d03fatznjxp6wkk6qvfg5tqjc5u5buod7cc8se7yzdyai9x0"
slaveof 10.100.8.21 6379
masterauth "d03fatznjxp6wkk6qvfg5tqjc5u5buod7cc8se7yzdyai9x0"
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /srv/data/redis
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
添加redis开机自启/etc/systemd/system/redis.service，其中/srv/app/redis/bin/redis-shutdown脚本需要自行添加(redis所有节点和sentinel节点都需要)
[Unit]
Description=Redis persistent key-value database
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
ExecStart=/srv/app/redis/bin/redis-server /srv/app/redis/conf/redis.conf --supervised systemd
ExecStop=/srv/app/redis/bin/redis-shutdown
Type=notify
User=redis
Group=redis
RuntimeDirectory=redis
RuntimeDirectoryMode=0755
[Install]
WantedBy=multi-user.target
添加/srv/app/redis/bin/redis-shutdown脚本到所有集群,并给予执行权限chmod 755 /srv/app/redis/bin/redis-shutdown
#!/bin/bash
#
# Wrapper to close properly redis and sentinel
test x"$REDIS_DEBUG" != x && set -x
REDIS_CLI=/srv/app/redis/bin/redis-cli
# Retrieve service name
SERVICE_NAME="$1"
if [ -z "$SERVICE_NAME" ]; then
SERVICE_NAME=redis
fi
# Get the proper config file based on service name
CONFIG_FILE="/srv/app/redis/conf/$SERVICE_NAME.conf"
# Use awk to retrieve host, port from config file
HOST=`awk '/^[[:blank:]]*bind/ { print $2 }' $CONFIG_FILE | tail -n1`
PORT=`awk '/^[[:blank:]]*port/ { print $2 }' $CONFIG_FILE | tail -n1`
PASS=`awk '/^[[:blank:]]*requirepass/ { print $2 }' $CONFIG_FILE | tail -n1`
SOCK=`awk '/^[[:blank:]]*unixsocket\s/ { print $2 }' $CONFIG_FILE | tail -n1`
# Just in case, use default host, port
HOST=${HOST:-127.0.0.1}
if [ "$SERVICE_NAME" = redis ]; then
PORT=${PORT:-6379}
else
PORT=${PORT:-26739}
fi
# Setup additional parameters
# e.g password-protected redis instances
[ -z "$PASS" ] || ADDITIONAL_PARAMS="-a $PASS"
# shutdown the service properly
if [ -e "$SOCK" ] ; then
$REDIS_CLI -s $SOCK $ADDITIONAL_PARAMS shutdown
else
$REDIS_CLI -h $HOST -p $PORT $ADDITIONAL_PARAMS shutdown
fi
启动redis节点：
chown -Rf redis:redis /srv/{app,data,logs}/redis;
systemctl daemon-reload && systemctl enable redis && systemctl restart redis
查看redis主从信息``：
[root@localhost ~]# /srv/app/redis/bin/redis-cli -h 127.0.0.1 -p 6379 -a d03fatznjxp6wkk6qvfg5tqjc5u5buod7cc8se7yzdyai9x0
Warning: Using a password with '-a' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:2
slave0:ip=10.100.8.22,port=6379,state=online,offset=548291,lag=0
slave1:ip=10.100.8.23,port=6379,state=online,offset=548291,lag=0
master_replid:0a56572596b7c46792946836cda553a6dda31898
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:548291
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:548291
sentinel集群部署
注意:以下所有部署步骤只在sentinel角色机器执行！
之前的部署已经编译安装了redis，我们只需要在每台sentinel节点添加配置/srv/app/redis/conf/redis-sentinel.conf即可：
bind 0.0.0.0
port 26379
dir /tmp
protected-mode yes
daemonize yes
supervised systemd
sentinel monitor mymaster 10.100.8.21 6379 2
sentinel auth-pass mymaster "d03fatznjxp6wkk6qvfg5tqjc5u5buod7cc8se7yzdyai9x0"
sentinel down-after-milliseconds mymaster 30000
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster 180000
sentinel deny-scripts-reconfig yes
logfile /srv/logs/redis/redis-sentinel.log
设置开机自启/etc/systemd/system/redis-sentinel.service：
[Unit]
Description=Redis Sentinel
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
ExecStart=/srv/app/redis/bin/redis-sentinel /srv/app/redis/conf/redis-sentinel.conf --supervised systemd
ExecStop=/srv/app/redis/bin/redis-shutdown redis-sentinel
Type=notify
User=redis
Group=redis
RuntimeDirectory=redis
RuntimeDirectoryMode=0755
[Install]
WantedBy=multi-user.target
将上面的配置文件各自复制到三个sentinel节点，注意：复制配置文件一定要在启动redis-sentinel服务之前，因为启动服务后配置文件中会生成id号，服务个节点的id相同会造成无法选举新的master节点
启动sentinel
chown -Rf redis:redis /srv/{app,data,logs}/redis;
systemctl daemon-reload && systemctl enable redis-sentinel && systemctl restart redis-sentinel
我们再次查看配置:
[root@vm-10-100-8-24 ~]# cat /srv/app/redis/conf/redis-sentinel.conf
bind 0.0.0.0
port 26379
dir "/tmp"
daemonize yes
protected-mode yes
supervised systemd
sentinel myid f040baa9d10e86988c9658dc475735a851215b04
sentinel deny-scripts-reconfig yes
sentinel monitor mymaster 10.100.8.21 6379 2
sentinel auth-pass mymaster d03fatznjxp6wkk6qvfg5tqjc5u5buod7cc8se7yzdyai9x0
sentinel config-epoch mymaster 0
sentinel leader-epoch mymaster 0
logfile "/srv/logs/redis/redis-sentinel.log"
# Generated by CONFIG REWRITE
maxclients 4064
sentinel known-slave mymaster 10.100.8.23 6379
sentinel known-slave mymaster 10.100.8.22 6379
sentinel known-sentinel mymaster 10.100.8.26 26379 4db3eb7d479d03a3e964f09541e6b2384ccbdbb2
sentinel known-sentinel mymaster 10.100.8.25 26379 cbcf5b81ec4e77ec1fa2f4f3a15f9e3afe041a24
sentinel current-epoch 0
配置redis+sentinel的日志自动切割(每个节点都需要添加)：
[root@localhost ~]# vim /etc/logrotate.d/redis
/srv/logs/redis/*.log {
weekly
rotate 10
copytruncate
delaycompress
compress
notifempty
missingok
}
验证集群
至此, redis主从+sentinel集群已经部署完毕,接下来我们验证sentinel集群的信息:
[root@vm-10-100-8-24 ~]# /srv/app/redis/bin/redis-cli -p #哨兵模式查看
127.0.0.1:26379> sentinel master mymaster #输出被监控的主节点的状态信息
1) "name"
2) "mymaster"
3) "ip"
4) "10.100.8.21"
5) "port"
6) "6379"
7) "runid"
8) "f35e9a7eea37303e86079cfc8847d37f77c8c66f"
9) "flags"
10) "master"
11) "link-pending-commands"
12) "0"
13) "link-refcount"
14) "1"
15) "last-ping-sent"
16) "0"
17) "last-ok-ping-reply"
18) "841"
19) "last-ping-reply"
20) "841"
21) "down-after-milliseconds"
22) "30000"
23) "info-refresh"
24) "4592"
25) "role-reported"
26) "master"
27) "role-reported-time"
28) "3859508"
29) "config-epoch"
30) "0"
31) "num-slaves"
32) "2"
33) "num-other-sentinels"
34) "2"
35) "quorum"
36) "2"
37) "failover-timeout"
38) "180000"
39) "parallel-syncs"
40) "1"
127.0.0.1:26379> sentinel slaves mymaster #查看mymaster的从信息，可以看到有2个从节点
1) 1) "name"
2) "10.100.8.23:6379"
3) "ip"
4) "10.100.8.23"
5) "port"
6) "6379"
7) "runid"
8) "1db40103b8a18e823f30854bdc8b064b7379c7b4"
9) "flags"
10) "slave"
11) "link-pending-commands"
12) "0"
13) "link-refcount"
14) "1"
15) "last-ping-sent"
16) "0"
17) "last-ok-ping-reply"
18) "376"
19) "last-ping-reply"
20) "376"
21) "down-after-milliseconds"
22) "30000"
23) "info-refresh"
24) "3418"
25) "role-reported"
26) "slave"
27) "role-reported-time"
28) "3868335"
29) "master-link-down-time"
30) "0"
31) "master-link-status"
32) "ok"
33) "master-host"
34) "10.100.8.21"
35) "master-port"
36) "6379"
37) "slave-priority"
38) "90"
39) "slave-repl-offset"
40) "729144"
2) 1) "name"
2) "10.100.8.22:6379"
3) "ip"
4) "10.100.8.22"
5) "port"
6) "6379"
7) "runid"
8) "c73bcc5709123194589324feba5e854bd9b921dc"
9) "flags"
10) "slave"
11) "link-pending-commands"
12) "0"
13) "link-refcount"
14) "1"
15) "last-ping-sent"
16) "0"
17) "last-ok-ping-reply"
18) "376"
19) "last-ping-reply"
20) "376"
21) "down-after-milliseconds"
22) "30000"
23) "info-refresh"
24) "3418"
25) "role-reported"
26) "slave"
27) "role-reported-time"
28) "3868342"
29) "master-link-down-time"
30) "0"
31) "master-link-status"
32) "ok"
33) "master-host"
34) "10.100.8.21"
35) "master-port"
36) "6379"
37) "slave-priority"
38) "100"
39) "slave-repl-offset"
40) "729144"
127.0.0.1:26379> sentinel sentinels mymaster #查看其它sentinel信息
1) 1) "name"
2) "4db3eb7d479d03a3e964f09541e6b2384ccbdbb2"
3) "ip"
4) "10.100.8.26"
5) "port"
6) "26379"
7) "runid"
8) "4db3eb7d479d03a3e964f09541e6b2384ccbdbb2"
9) "flags"
10) "s_down,sentinel,disconnected"
11) "link-pending-commands"
12) "2"
13) "link-refcount"
14) "1"
15) "last-ping-sent"
16) "3152356"
17) "last-ok-ping-reply"
18) "3152356"
19) "last-ping-reply"
20) "3152356"
21) "s-down-time"
22) "3122313"
23) "down-after-milliseconds"
24) "30000"
25) "last-hello-message"
26) "1656"
27) "voted-leader"
28) "?"
29) "voted-leader-epoch"
30) "0"
2) 1) "name"
2) "cbcf5b81ec4e77ec1fa2f4f3a15f9e3afe041a24"
3) "ip"
4) "10.100.8.25"
5) "port"
6) "26379"
7) "runid"
8) "cbcf5b81ec4e77ec1fa2f4f3a15f9e3afe041a24"
9) "flags"
10) "s_down,sentinel,disconnected"
11) "link-pending-commands"
12) "2"
13) "link-refcount"
14) "1"
15) "last-ping-sent"
16) "3428000"
17) "last-ok-ping-reply"
18) "3428000"
19) "last-ping-reply"
20) "3428000"
21) "s-down-time"
22) "3397980"
23) "down-after-milliseconds"
24) "30000"
25) "last-hello-message"
26) "1259"
27) "voted-leader"
28) "?"
29) "voted-leader-epoch"
30) "0"
- 哨兵模式下的主从测试
[root@vm-10-100-8-21 ~]# service redis start #停掉master
Redirecting to /bin/systemctl start redis.service
[root@vm-10-100-8-22 ~]# tail -f /srv/logs/redis/redis.log #查看其中一台slave的日志
4701:S 20 Jul 23:18:53.716 * MASTER <-> SLAVE sync started
4701:S 20 Jul 23:18:53.717 # Error condition on socket for SYNC: Connection refused
4701:S 20 Jul 23:18:53.853 * SLAVE OF 10.100.8.23:6379 enabled (user request from 'id=33 addr=10.100.8.25:51437 fd=9 name=sentinel-cbcf5b81-cmd age=79 idle=0 flags=x db=0 sub=0 psub=0 multi=3 qbuf=137 qbuf-free=32631 obl=36 oll=0 omem=0 events=r cmd=exec')
4701:S 20 Jul 23:18:53.854 # CONFIG REWRITE executed with success.
4701:S 20 Jul 23:18:54.719 * Connecting to MASTER 10.100.8.23:6379
4701:S 20 Jul 23:18:54.719 * MASTER <-> SLAVE sync started
4701:S 20 Jul 23:18:54.721 * Non blocking connect for SYNC fired the event.
4701:S 20 Jul 23:18:54.722 * Master replied to PING, replication can continue...
4701:S 20 Jul 23:18:54.723 * Trying a partial resynchronization (request bd76cf3afae3bfa4312299fdba0a30cffa57caf1:13534).
4701:S 20 Jul 23:18:54.724 * Successful partial resynchronization with master.
4701:S 20 Jul 23:18:54.724 # Master replication ID changed to 105cf31a94bd9fd0dfb89b0ae230c930d42ee230
4701:S 20 Jul 23:18:54.724 * MASTER <-> SLAVE sync: Master accepted a Partial Resynchronization.
[root@vm-10-100-8-24 ~]# tail -f /srv/logs/redis/redis-sentinel.log #查看哨兵日志
31874:X 20 Jul 23:17:06.392 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
31874:X 20 Jul 23:17:06.393 # Redis version=4.0.14, bits=64, commit=00000000, modified=0, pid=31874, just started
31874:X 20 Jul 23:17:06.393 # Configuration loaded
31874:X 20 Jul 23:17:06.393 * supervised by systemd, will signal readiness
31874:X 20 Jul 23:17:06.395 * Increased maximum number of open files to 4096 (it was originally set to 1024).
31874:X 20 Jul 23:17:06.397 * Running mode=sentinel, port=26379.
31874:X 20 Jul 23:17:06.397 # Sentinel ID is f040baa9d10e86988c9658dc475735a851215b04
31874:X 20 Jul 23:17:06.397 # +monitor master mymaster 10.100.8.21 6379 quorum 1
31874:X 20 Jul 23:17:36.418 # +sdown sentinel 4db3eb7d479d03a3e964f09541e6b2384ccbdbb2 10.100.8.26 26379 @ mymaster 10.100.8.21 6379
31874:X 20 Jul 23:17:49.037 # -sdown sentinel 4db3eb7d479d03a3e964f09541e6b2384ccbdbb2 10.100.8.26 26379 @ mymaster 10.100.8.21 6379
31874:X 20 Jul 23:18:52.654 # +new-epoch 5
31874:X 20 Jul 23:18:52.658 # +vote-for-leader cbcf5b81ec4e77ec1fa2f4f3a15f9e3afe041a24 5
31874:X 20 Jul 23:18:52.670 # +sdown master mymaster 10.100.8.21 6379
31874:X 20 Jul 23:18:52.670 # +odown master mymaster 10.100.8.21 6379 #quorum 1/1
31874:X 20 Jul 23:18:52.670 # Next failover delay: I will not start a failover before Mon Jul 20 23:24:52 2020
31874:X 20 Jul 23:18:53.853 # +config-update-from sentinel cbcf5b81ec4e77ec1fa2f4f3a15f9e3afe041a24 10.100.8.25 26379 @ mymaster 10.100.8.21 6379
31874:X 20 Jul 23:18:53.853 # +switch-master mymaster 10.100.8.21 6379 10.100.8.23 6379
31874:X 20 Jul 23:18:53.854 * +slave slave 10.100.8.22:6379 10.100.8.22 6379 @ mymaster 10.100.8.23 6379
31874:X 20 Jul 23:18:53.854 * +slave slave 10.100.8.21:6379 10.100.8.21 6379 @ mymaster 10.100.8.23 6379
来源：oschina
链接：https://my.oschina.net/u/4306166/blog/4411272