一、OSD管理
1.1 移出故障osd
查看当前节点的osd的id
[root@ceph2 ceph]# df -hT
Filesystem Type Size Used Avail Use% Mounted on /dev/vda1 xfs 40G 2.7G 38G 7% / devtmpfs devtmpfs 1.9G 0 1.9G 0% /dev tmpfs tmpfs 1.9G 0 1.9G 0% /dev/shm tmpfs tmpfs 1.9G 201M 1.7G 11% /run tmpfs tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup /dev/vdb1 xfs 15G 213M 15G 2% /var/lib/ceph/osd/ceph-0 /dev/vdc1 xfs 15G 228M 15G 2% /var/lib/ceph/osd/ceph-3 /dev/vdd1 xfs 15G 152M 15G 1% /var/lib/ceph/osd/ceph-6 tmpfs tmpfs 380M 0 380M 0% /run/user/0
停止故障osd
[root@ceph2 ceph]# systemctl stop ceph-osd@0
产看状态
[root@ceph2 ceph]# ceph -s
cluster:
id: 35a91e48-8244-4e96-a7ee-980ab989d20d
health: HEALTH_WARN
1 osds down
Degraded data redundancy: 67/663 objects degraded (10.106%), 170 pgs unclean, 170 pgs degraded
services:
mon: 3 daemons, quorum ceph2,ceph3,ceph4
mgr: ceph4(active), standbys: ceph3, ceph2
mds: cephfs-1/1/1 up {0=ceph2=up:active}, 1 up:standby
osd: 9 osds: 8 up, 9 in
rbd-mirror: 1 daemon active
data:
pools: 13 pools, 504 pgs
objects: 221 objects, 241 MB
usage: 1770 MB used, 133 GB / 134 GB avail
pgs: 67/663 objects degraded (10.106%)
334 active+clean
170 active+undersized+degraded
[root@ceph2 ceph]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF -9 0.13499 root dc1 -10 0.04500 rack rack1 -3 0.04500 host ceph2 0 hdd 0.01500 osd.0 down 1.00000 1.00000 3 hdd 0.01500 osd.3 up 1.00000 1.00000 6 hdd 0.01500 osd.6 up 1.00000 1.00000 -11 0.04500 rack rack2 -7 0.04500 host ceph3 2 hdd 0.01500 osd.2 up 1.00000 1.00000 5 hdd 0.01500 osd.5 up 1.00000 1.00000 8 hdd 0.01500 osd.8 up 1.00000 1.00000 -12 0.04500 rack rack3 -5 0.04500 host ceph4 1 hdd 0.01500 osd.1 up 1.00000 1.00000 4 hdd 0.01500 osd.4 up 1.00000 0.50000 7 hdd 0.01500 osd.7 up 1.00000 1.00000 -1 0.13499 root default -3 0.04500 host ceph2 0 hdd 0.01500 osd.0 down 1.00000 1.00000 3 hdd 0.01500 osd.3 up 1.00000 1.00000 6 hdd 0.01500 osd.6 up 1.00000 1.00000 -7 0.04500 host ceph3 2 hdd 0.01500 osd.2 up 1.00000 1.00000 5 hdd 0.01500 osd.5 up 1.00000 1.00000 8 hdd 0.01500 osd.8 up 1.00000 1.00000 -5 0.04500 host ceph4 1 hdd 0.01500 osd.1 up 1.00000 1.00000 4 hdd 0.01500 osd.4 up 1.00000 0.50000 7 hdd 0.01500 osd.7 up 1.00000 1.00000
[root@ceph2 ceph]# ceph osd out osd.0
marked out osd.0.
[root@ceph2 ceph]# ceph -s
cluster:
id: 35a91e48-8244-4e96-a7ee-980ab989d20d
health: HEALTH_WARN
Degraded data redundancy: 126/663 objects degraded (19.005%), 24 pgs unclean, 24 pgs degraded
services:
mon: 3 daemons, quorum ceph2,ceph3,ceph4
mgr: ceph4(active), standbys: ceph3, ceph2
mds: cephfs-1/1/1 up {0=ceph2=up:active}, 1 up:standby
osd: 9 osds: 8 up, 8 in
rbd-mirror: 1 daemon active
data:
pools: 13 pools, 504 pgs
objects: 221 objects, 241 MB
usage: 1587 MB used, 118 GB / 119 GB avail
pgs: 126/663 objects degraded (19.005%)
480 active+clean
23 active+recovery_wait+degraded
1 active+recovering+degraded
io:
client: 4196 B/s rd, 0 B/s wr, 3 op/s rd, 0 op/s wr
recovery: 2873 kB/s, 0 keys/s, 2 objects/s
[root@ceph2 ceph]# ceph osd crush rm osd.0
removed item id 0 name 'osd.0' from crush map
[root@ceph2 ceph]# ceph auth list|grep osd.0
installed auth entries: osd.0
[root@ceph2 ceph]# ceph auth rm osd.0
updated
[root@ceph2 ceph]# ceph -s
cluster:
id: 35a91e48-8244-4e96-a7ee-980ab989d20d
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph2,ceph3,ceph4
mgr: ceph4(active), standbys: ceph3, ceph2
mds: cephfs-1/1/1 up {0=ceph2=up:active}, 1 up:standby
osd: 9 osds: 8 up, 8 in
rbd-mirror: 1 daemon active
data:
pools: 13 pools, 504 pgs
objects: 221 objects, 241 MB
usage: 1656 MB used, 118 GB / 119 GB avail
pgs: 504 active+clean
io:
client: 0 B/s wr, 0 op/s rd, 0 op/s wr
[root@ceph2 ceph]# ceph osd rm osd.0
removed osd.0
[root@ceph2 ceph]# ceph -s
cluster:
id: 35a91e48-8244-4e96-a7ee-980ab989d20d
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph2,ceph3,ceph4
mgr: ceph4(active), standbys: ceph3, ceph2
mds: cephfs-1/1/1 up {0=ceph2=up:active}, 1 up:standby
osd: 8 osds: 8 up, 8 in
rbd-mirror: 1 daemon active
data:
pools: 13 pools, 504 pgs
objects: 221 objects, 241 MB
usage: 1656 MB used, 118 GB / 119 GB avail
pgs: 504 active+clean
io:
client: 5321 B/s rd, 0 B/s wr, 5 op/s rd, 0 op/s wr
方法二
ceph osd out osd.3 systemctl stop ceph-osd@3 ceph osd purge osd.3 #综合这一步，就可以完成操作 删除配置文件中针对该osd的配置
1.2添加回osd
上述实验，是模拟osd坏掉，然后进行移除操作，但是osd这个硬盘的挂载，分区并没有清除，也就是说，在本次添加的时候，直接从创建osd秘钥开始，但是在生产环境中，更换了osd后，需要从创建盘格式化开始，过程请参照https://www.cnblogs.com/zyxnhr/p/10553717.html
具体过程如下
[root@ceph2 ceph-0]# ceph osd create
0
[root@ceph2 ceph-0]# ceph-authtool --create-keyring /etc/ceph/ceph.osd.0.keyring --gen-key -n osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *'
creating /etc/ceph/ceph.osd.0.keyring
[root@ceph2 ceph-0]# ceph auth import -i /etc/ceph/ceph.osd.0.keyring
imported keyring
[root@ceph2 ceph-0]# ceph auth get-or-create osd.0 -o /var/lib/ceph/osd/ceph-0/keyring
[root@ceph2 ceph-0]# ceph-osd -i 0 --mkfs --cluster ceph
2019-03-29 07:57:58.928076 7f564d51fd00 -1 created object store /var/lib/ceph/osd/ceph-0 for osd.0 fsid 35a91e48-8244-4e96-a7ee-980ab989d20d
[root@ceph2 ceph-0]# cd /var/lib/ceph/osd/ceph-0
[root@ceph2 ceph-0]# rm -f journal
[root@ceph2 ceph-0]# partuuid_0=`blkid /dev/vdb1|awk -F "[\"\"]" '{print $8}'`
[root@ceph2 ceph-0]# echo $partuuid_0
745dce53-1c63-4c50-b434-d441038dafe4
[root@ceph2 ceph-0]# ln -s /dev/disk/by-partuuid/$partuuid_0 ./journal
[root@ceph2 ceph-0]# ll
total 64
-rw-r--r-- 1 root root 393 Mar 16 12:46 activate.monmap
-rw-r--r-- 1 ceph ceph 3 Mar 16 12:46 active
-rw-r--r-- 1 ceph ceph 37 Mar 16 12:46 ceph_fsid
drwxr-xr-x 344 ceph ceph 12288 Mar 28 10:40 current
-rw-r--r-- 1 ceph ceph 37 Mar 16 12:46 fsid
lrwxrwxrwx 1 root root 58 Mar 29 07:59 journal -> /dev/disk/by-partuuid/745dce53-1c63-4c50-b434-d441038dafe4
-rw-r--r-- 1 ceph ceph 37 Mar 16 12:46 journal_uuid
-rw------- 1 ceph ceph 56 Mar 29 07:57 keyring
-rw-r--r-- 1 ceph ceph 21 Mar 16 12:46 magic
-rw-r--r-- 1 ceph ceph 6 Mar 16 12:46 ready
-rw-r--r-- 1 ceph ceph 4 Mar 16 12:46 store_version
-rw-r--r-- 1 ceph ceph 53 Mar 16 12:46 superblock
-rw-r--r-- 1 ceph ceph 0 Mar 16 12:47 systemd
-rw-r--r-- 1 ceph ceph 10 Mar 16 12:46 type
-rw-r--r-- 1 ceph ceph 2 Mar 16 12:46 whoami
[root@ceph2 ceph-0]# chown ceph.ceph -R /var/lib/ceph
[root@ceph2 ceph-0]# ceph-osd --mkjournal -i 0 --cluster ceph
2019-03-29 08:00:02.007442 7f416ec90d00 -1 journal read_header error decoding journal header
2019-03-29 08:00:02.018206 7f416ec90d00 -1 created new journal /var/lib/ceph/osd/ceph-0/journal for object store /var/lib/ceph/osd/ceph-0
[root@ceph2 ceph-0]# chown ceph.ceph /dev/disk/by-partuuid/$partuuid_0
[root@ceph2 ceph-0]# ceph osd crush add-bucket ceph2 host --cluster ceph
bucket 'ceph2' already exists #不用创建bucket，在移除的时候，并没有移除主机的bucket
[root@ceph2 ceph-0]# ceph osd crush move ceph2 root=default --cluster ceph #也不需要把ceph2添加到default的这个crushrule中
no need to move item id -3 name 'ceph2' to location {root=default} in crush map
[root@ceph2 ceph-0]# ceph osd crush add osd.0 0.01500 root=default host=ceph2
add item id 0 name 'osd.0' weight 0.015 at location {host=ceph2,root=default} to crush map
[root@ceph2 ceph-0]# systemctl start ceph-osd@0
[root@ceph2 ceph-0]# systemctl enable ceph-osd@0
[root@ceph2 ceph-0]# ps -ef|grep osd
ceph 1147069 1 0 Mar28 ? 00:02:51 /usr/bin/ceph-osd -f --cluster ceph --id 6 --setuser ceph --setgroup ceph
ceph 1147169 1 0 Mar28 ? 00:03:19 /usr/bin/ceph-osd -f --cluster ceph --id 3 --setuser ceph --setgroup ceph
ceph 1220601 1 6 08:04 ? 00:00:01 /usr/bin/ceph-osd -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
root 1220713 1156971 0 08:04 pts/0 00:00:00 grep --color=auto osd
[root@ceph2 ceph-0]# ceph -s
cluster:
id: 35a91e48-8244-4e96-a7ee-980ab989d20d
health: HEALTH_WARN
1/2481 objects misplaced (0.040%)
Degraded data redundancy: 423/2481 objects degraded (17.050%), 8 pgs unclean, 16 pgs degraded #有数据正在重平衡
services:
mon: 3 daemons, quorum ceph2,ceph3,ceph4
mgr: ceph4(active), standbys: ceph3, ceph2
mds: cephfs-1/1/1 up {0=ceph2=up:active}, 1 up:standby
osd: 9 osds: 9 up, 9 in; 1 remapped pgs
rbd-mirror: 1 daemon active
data:
pools: 13 pools, 504 pgs
objects: 827 objects, 2206 MB
usage: 6747 MB used, 128 GB / 134 GB avail
pgs: 423/2481 objects degraded (17.050%)
1/2481 objects misplaced (0.040%)
487 active+clean
14 active+recovery_wait+degraded
2 active+recovering+degraded
1 active+remapped+backfill_wait
io:
client: 4093 B/s rd, 0 B/s wr, 4 op/s rd, 0 op/s wr
recovery: 20080 kB/s, 0 keys/s, 6 objects/s
[root@ceph2 ceph-0]# ceph -s
cluster:
id: 35a91e48-8244-4e96-a7ee-980ab989d20d
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph2,ceph3,ceph4
mgr: ceph4(active), standbys: ceph3, ceph2
mds: cephfs-1/1/1 up {0=ceph2=up:active}, 1 up:standby
osd: 9 osds: 9 up, 9 in #恢复正常，有9个OSD，切集群处于健康状态
rbd-mirror: 1 daemon active
data:
pools: 13 pools, 504 pgs
objects: 827 objects, 2206 MB
usage: 6614 MB used, 128 GB / 134 GB avail
pgs: 504 active+clean
io:
client: 4093 B/s rd, 0 B/s wr, 4 op/s rd, 0 op/s wr
1.2 移除故障节点
方法一：
1. 先移除节点上所有osd
2. ceph osd crush remove serverc
方法二：
1. 先迁移节点上所有osd
2. 修改crushmap，删除所有与该节点相关的配置
1.3 恢复和回填OSD
在OSD添加或移除时，Ceph会重平衡PG。数据回填和恢复操作可能会产生大量的后端流量，影响集群性能。
为避免性能降低，可对回填/恢复操作进行配置：
osd_recovery_op_priority # 值为1-63，默认为10，相对于客户端操作，恢复操作的优先级，默认客户端操作的优先级为63，参数为osd_client_op_priority
osd_recovery_max_active # 每个osd一次处理的活跃恢复请求数量，默认为15，增大此值可加速恢复，但会增加集群负载
osd_recovery_threads # 用于数据恢复时的线程数，默认为1
osd_max_backfills # 单个osd的最大回填操作数，默认为10
osd_backfill_scan_min # 回填操作时最小扫描对象数量，默认为64
osd_backfill_scan_max # 回填操作的最大扫描对象数量，默认为512
osd_backfill_full_ratio # osd的占满率达到多少时，拒绝接受回填请求，默认为0.85
osd_backfill_retry_interval # 回填重试的时间间隔
二、 monitor管理
2.1 摘除monitor
停止monitor进程
[root@ceph2 ceph]# systemctl stop ceph-mon@ceph2
删除monitor
[root@ceph2 ceph]# ceph mon remove ceph2
removing mon.ceph2 at 172.25.250.11:6789/0, there will be 2 monitors
[root@ceph2 ceph]# ceph -s
services:
mon: 2 daemons, quorum ceph3,ceph4
mgr: ceph4(active), standbys: ceph2, ceph3
mds: cephfs-1/1/1 up {0=ceph2=up:active}, 1 up:standby
osd: 9 osds: 9 up, 9 in
rbd-mirror: 1 daemon active
删除monitor文件
[root@ceph2 ceph]# cd /var/lib/ceph/mon/ [root@ceph2 mon]# ls ceph-ceph2 [root@ceph2 mon]# rm -rf ceph-ceph2/
2.2 添加monitor节点
[root@ceph2 mon]# cd
创建文件
[root@ceph2 ~]# mkdir /var/lib/ceph/mon/ceph-ceph2
[root@ceph2 ~]# chown ceph.ceph -R !$
chown ceph.ceph -R /var/lib/ceph/mon/ceph-ceph2
[root@ceph2 ~]# ceph auth get mon.
exported keyring for mon. [mon.] key = AQDqfYxcAAAAABAAIc47ZLcYh013gzu3WWruew== caps mon = "allow *"
[root@ceph2 ~]# ceph auth get mon. -o /tmp/ceph.mon.keyring
exported keyring for mon.
[root@ceph2 ~]# cat /tmp/ceph.mon.keyring
[mon.] key = AQDqfYxcAAAAABAAIc47ZLcYh013gzu3WWruew== caps mon = "allow *"
[root@ceph2 ~]# ceph mon getmap -o /tmp/monmap.bin
got monmap epoch 2
[root@ceph2 ~]# monmaptool --print /tmp/monmap.bin
monmaptool: monmap file /tmp/monmap.bin epoch 2 fsid 35a91e48-8244-4e96-a7ee-980ab989d20d last_changed 2019-03-28 08:57:25.819243 created 2019-03-16 12:39:14.839999 0: 172.25.250.12:6789/0 mon.ceph3 1: 172.25.250.13:6789/0 mon.ceph4
[root@ceph2 ~]# sudo -u ceph ceph-mon -i ceph2 --mkfs --monmap /tmp/monmap.bin --keyring /tmp/ceph.mon.keyring
[root@ceph2 ~]# ll /var/lib/ceph/mon/ceph-ceph2/
total 8 -rw------- 1 ceph ceph 77 Mar 28 09:03 keyring -rw-r--r-- 1 ceph ceph 8 Mar 28 09:03 kv_backend drwxr-xr-x 2 ceph ceph 112 Mar 28 09:03 store.db
[root@ceph2 ~]# ps -ef |grep ceph-mon
root 1135665 1088603 0 09:12 pts/0 00:00:00 grep --color=auto ceph-mon
[root@ceph2 ~]# sudo -u ceph ceph-mon -i ceph2 --public-addr 172.25.250.11:6789
[root@ceph2 ~]# !ps
ps -ef |grep ceph-mon ceph 1135726 1 1 09:13 ? 00:00:00 ceph-mon -i ceph2 --public-addr 172.25.250.11:6789 root 1135771 1088603 0 09:13 pts/0 00:00:00 grep --color=auto ceph-mon
[root@ceph2 ~]# ceph -s
services:
mon: 3 daemons, quorum ceph2,ceph3,ceph4
mgr: ceph4(active), standbys: ceph2, ceph3
mds: cephfs-1/1/1 up {0=ceph2=up:active}, 1 up:standby
osd: 9 osds: 9 up, 9 in
rbd-mirror: 1 daemon active
2.3 monitor故障排查
[root@ceph2 ~]# ceph daemon mon.ceph2 quorum_status
{ #查看monipot票数
"election_epoch": 128,
"quorum": [
0,
1,
2
],
"quorum_names": [
"ceph2",
"ceph3",
"ceph4"
],
"quorum_leader_name": "ceph2",
"monmap": {
"epoch": 3,
"fsid": "35a91e48-8244-4e96-a7ee-980ab989d20d",
"modified": "2019-03-28 09:13:19.932456",
"created": "2019-03-16 12:39:14.839999",
"features": {
"persistent": [
"kraken",
"luminous"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "ceph2",
"addr": "172.25.250.11:6789/0",
"public_addr": "172.25.250.11:6789/0"
},
{
"rank": 1,
"name": "ceph3",
"addr": "172.25.250.12:6789/0",
"public_addr": "172.25.250.12:6789/0"
},
{
"rank": 2,
"name": "ceph4",
"addr": "172.25.250.13:6789/0",
"public_addr": "172.25.250.13:6789/0"
}
]
}
}
2.4 利用admin sockets管理守护进程
通过admin sockets，管理员可以直接与守护进程交互。如查看和修改守护进程的配置参数。
守护进程的socket文件一般是/var/run/ceph/$cluster-$type.$id.asok
基于admin sockets的操作：
ceph daemon $type.$id command
或者ceph --admin-daemon /var/run/ceph/$cluster-$type.$id.asok command
常用command如下：
help
config get parameter
config set parameter
config show
perf dump
查看
[root@ceph2 ceph]# ceph daemon osd.6 config show|grep osd_default
"osd_default_data_pool_replay_window": "45",
"osd_default_notify_timeout": "30",
[root@ceph2 ceph]# ceph daemon osd.6 config get xio_mp_max_64
{
"xio_mp_max_64": "65536"
}
修改
[root@ceph2 ceph]# ceph tell osd.* injectargs --xio_mp_max_64 65536
osd.1: xio_mp_max_64 = '65536' (not observed, change may require restart) osd.2: xio_mp_max_64 = '65536' (not observed, change may require restart) osd.3: xio_mp_max_64 = '65536' (not observed, change may require restart) osd.4: xio_mp_max_64 = '65536' (not observed, change may require restart) osd.5: xio_mp_max_64 = '65536' (not observed, change may require restart) osd.6: xio_mp_max_64 = '65536' (not observed, change may require restart) osd.7: xio_mp_max_64 = '65536' (not observed, change may require restart) osd.8: xio_mp_max_64 = '65536' (not observed, change may require restart)
博主声明：本文的内容来源主要来自誉天教育晏威老师，由本人实验完成操作验证，需要的博友请联系誉天教育（http://www.yutianedu.com/），获得官方同意或者晏老师（https://www.cnblogs.com/breezey/）本人同意即可转载，谢谢！
来源：https://www.cnblogs.com/zyxnhr/p/10617542.html