ceph 添加新硬盘删除旧的OSD-白红宇

接上一篇:

为每个OSD NODE添加一块磁盘分区后再进行实验.

[root@ceph_node1 osd]# df

Filesystem 1K-blocks Used Available Use% Mounted on

/dev/mapper/centos-root 7022592 6789812 232780 97% /

devtmpfs 933432 0 933432 0% /dev

tmpfs 942208 0 942208 0% /dev/shm

tmpfs 942208 32268 909940 4% /run

tmpfs 942208 0 942208 0% /sys/fs/cgroup

/dev/vda1 508588 139920 368668 28% /boot

/dev/vdb1 8377344 32928 8344416 1% /osd

移除OSD,这里只做演示,实际生产环境中由于数据存放的原因此操作要慎重.

[root@ceph_monitor ~]# ceph osd out osd.3

marked out osd.3.

[root@ceph_monitor ~]# ceph osd out osd.4

marked out osd.4.

[root@ceph_monitor ~]# ceph osd down osd.4

marked down osd.4.

[root@ceph_monitor ~]# ceph osd down osd.3

marked down osd.3.

[root@ceph_monitor ~]# ceph osd rm osd.3

osd.3 does not exist.

[root@ceph_monitor ~]# ceph osd rm osd.4

removed osd.4

[root@ceph_node2 osd]# ceph -w

cluster f35a65ad-1a6a-4e8d-8f7e-cb5f113c0a02

health HEALTH_ERR

64 pgs stale

64 pgs stuck stale

2 full osd(s)

monmap e1: 1 mons at {ceph_monitor=10.0.2.33:6789/0}

election epoch 1, quorum 0 ceph_monitor

osdmap e22: 5 osds: 2 up, 2 in

flags full

pgmap v1644: 64 pgs, 1 pools, 0 bytes data, 0 objects

13260 MB used, 455 MB / 13716 MB avail

64 stale+active+clean

2015-09-14 13:21:39.089705 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:21:49.988763 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:22:00.864299 mon.0 [INF] from='client.? 10.0.2.33:0/1014270' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["osd.3"]}]: dispatch

2015-09-14 13:22:00.980026 mon.0 [INF] from='client.? 10.0.2.33:0/1014270' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["osd.3"]}]': finished

2015-09-14 13:22:01.043549 mon.0 [INF] osdmap e23: 5 osds: 2 up, 1 in full

2015-09-14 13:22:01.064547 mon.0 [INF] pgmap v1645: 64 pgs: 64 stale+active+clean; 0 bytes data, 6629 MB used, 228 MB / 6858 MB avail

2015-09-14 13:22:03.944095 mon.0 [INF] from='client.? 10.0.2.33:0/1014303' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:22:04.058603 mon.0 [INF] from='client.? 10.0.2.33:0/1014303' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["osd.4"]}]': finished

2015-09-14 13:22:04.136343 mon.0 [INF] osdmap e24: 5 osds: 2 up, 0 in full

2015-09-14 13:22:04.209347 mon.0 [INF] pgmap v1646: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:22:04.496508 mon.0 [INF] osdmap e25: 5 osds: 2 up, 0 in

2015-09-14 13:22:04.514888 mon.0 [INF] pgmap v1647: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:22:11.113375 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:22:19.412532 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:22:41.918295 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:22:51.015761 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:23:12.023552 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:23:15.296925 mon.0 [INF] pgmap v1648: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:23:19.301533 mon.0 [INF] pgmap v1649: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:23:22.419805 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:23:46.828958 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:23:54.523184 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:24:17.832867 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:24:27.627448 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:24:51.037841 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:24:57.930727 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:25:15.338544 mon.0 [INF] pgmap v1650: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:19.336484 mon.0 [INF] pgmap v1651: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:23.142214 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:25:27.034190 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:25:40.654246 mon.0 [INF] from='client.? 10.0.2.33:0/1014387' entity='client.admin' cmd=[{"prefix": "osd down", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:25:40.778345 mon.0 [INF] from='client.? 10.0.2.33:0/1014387' entity='client.admin' cmd='[{"prefix": "osd down", "ids": ["osd.4"]}]': finished

2015-09-14 13:25:40.801149 mon.0 [INF] osdmap e26: 5 osds: 1 up, 0 in

2015-09-14 13:25:40.820513 mon.0 [INF] pgmap v1652: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:42.980782 mon.0 [INF] osd.4 10.0.2.32:6800/4919 boot

2015-09-14 13:25:42.998345 mon.0 [INF] osdmap e27: 5 osds: 2 up, 0 in

2015-09-14 13:25:43.016972 mon.0 [INF] pgmap v1653: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:45.315515 mon.0 [INF] from='client.? 10.0.2.33:0/1014420' entity='client.admin' cmd=[{"prefix": "osd down", "ids": ["osd.3"]}]: dispatch

2015-09-14 13:25:44.430084 osd.4 [WRN] map e26 wrongly marked me down

2015-09-14 13:25:45.428227 mon.0 [INF] from='client.? 10.0.2.33:0/1014420' entity='client.admin' cmd='[{"prefix": "osd down", "ids": ["osd.3"]}]': finished

2015-09-14 13:25:45.445942 mon.0 [INF] osdmap e28: 5 osds: 1 up, 0 in

2015-09-14 13:25:45.464796 mon.0 [INF] pgmap v1654: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:51.637408 mon.0 [INF] from='client.? 10.0.2.33:0/1014453' entity='client.admin' cmd=[{"prefix": "osd rm", "ids": ["osd.3"]}]: dispatch

2015-09-14 13:25:51.741785 mon.0 [INF] from='client.? 10.0.2.33:0/1014453' entity='client.admin' cmd='[{"prefix": "osd rm", "ids": ["osd.3"]}]': finished

2015-09-14 13:25:51.852460 mon.0 [INF] osdmap e29: 4 osds: 1 up, 0 in

2015-09-14 13:25:51.875071 mon.0 [INF] pgmap v1655: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:25:54.707323 mon.0 [INF] from='client.? 10.0.2.33:0/1014486' entity='client.admin' cmd=[{"prefix": "osd rm", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:25:53.957396 osd.4 [WRN] OSD near full (96%)

2015-09-14 13:25:57.937731 osd.3 [WRN] OSD near full (96%)

2015-09-14 13:26:01.313844 mon.0 [INF] from='client.? 10.0.2.33:0/1014519' entity='client.admin' cmd=[{"prefix": "osd down", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:26:01.434547 mon.0 [INF] from='client.? 10.0.2.33:0/1014519' entity='client.admin' cmd='[{"prefix": "osd down", "ids": ["osd.4"]}]': finished

2015-09-14 13:26:01.452332 mon.0 [INF] osdmap e30: 4 osds: 0 up, 0 in

2015-09-14 13:26:01.477136 mon.0 [INF] pgmap v1656: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail

2015-09-14 13:26:04.414501 mon.0 [INF] from='client.? 10.0.2.33:0/1014552' entity='client.admin' cmd=[{"prefix": "osd rm", "ids": ["osd.3"]}]: dispatch

2015-09-14 13:26:07.547864 mon.0 [INF] from='client.? 10.0.2.33:0/1014585' entity='client.admin' cmd=[{"prefix": "osd rm", "ids": ["osd.4"]}]: dispatch

2015-09-14 13:26:07.675705 mon.0 [INF] from='client.? 10.0.2.33:0/1014585' entity='client.admin' cmd='[{"prefix": "osd rm", "ids": ["osd.4"]}]': finished

2015-09-14 13:26:07.693444 mon.0 [INF] osdmap e31: 3 osds: 0 up, 0 in

2015-09-14 13:26:07.718181 mon.0 [INF] pgmap v1657: 64 pgs: 64 stale+active+clean; 0 bytes data, 0 kB used, 0 kB / 0 kB avail