ceph 009 管理定义crushmap 故障域

管理和自定义crushmap

定义pg到osd的映射关系
通过crush算法使三副本映射到理想的主机或者机架
更改故障域提高可靠性

pg到osd映射由crush实现

ceph 009 管理定义crushmap  故障域插图

下载时需要将对象从osd搜索到,组成文件,那么对象多了就会效率变低,那么从pg组里面搜索。提高效率
对象放在pg要通过hash算法 95个pg / 100 取余 对象放在第95个pg里
pg属于存储池

默认池有32个pg
pg映射到osd,这个就属于crush算法

三种存储(rbd cephfs rgw)底层都是一样的

文件切分,是在客户端完成的
客户端被告知映射视图
然后开始存osd

ceph 009 管理定义crushmap  故障域插图1

crush的作用,就是根据pg id 得一个osd列表
ceph 009 管理定义crushmap  故障域插图2

crush map 的解译 编译 更新

导出一个二进制crush文件

[root@clienta ~]# cephadm shell
[ceph: root@clienta /]# ceph osd getcrushmap -o crushmap.bin
20
[ceph: root@clienta /]# ls
bin   crushmap.bin  etc   lib	 lost+found  mnt  proc	run   srv  tmp	var
boot  dev	    home  lib64  media	     opt  root	sbin  sys  usr
[ceph: root@clienta /]#   

将二进制转换为文本文件

[ceph: root@clienta /]# crushtool -d crushmap.bin -o crushmap.txt 
[ceph: root@clienta /]# cat crushmap.txt 
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54

# devices
device 0 osd.0 class hdd
device 1 osd.1 class hdd
device 2 osd.2 class hdd
device 3 osd.3 class hdd
device 4 osd.4 class hdd
device 5 osd.5 class hdd
device 6 osd.6 class hdd
device 7 osd.7 class hdd
device 8 osd.8 class hdd

# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 zone
type 10 region
type 11 root

# buckets
host serverc {
    id -3		# do not change unnecessarily
    id -4 class hdd		# do not change unnecessarily
    # weight 0.029
    alg straw2
    hash 0	# rjenkins1
    item osd.0 weight 0.010
    item osd.1 weight 0.010
    item osd.2 weight 0.010
}
host serverd {
    id -5		# do not change unnecessarily
    id -6 class hdd		# do not change unnecessarily
    # weight 0.029
    alg straw2
    hash 0	# rjenkins1
    item osd.3 weight 0.010
    item osd.5 weight 0.010
    item osd.7 weight 0.010
}
host servere {
    id -7		# do not change unnecessarily
    id -8 class hdd		# do not change unnecessarily
    # weight 0.029
    alg straw2
    hash 0	# rjenkins1
    item osd.4 weight 0.010
    item osd.6 weight 0.010
    item osd.8 weight 0.010
}
root default {
    id -1		# do not change unnecessarily
    id -2 class hdd		# do not change unnecessarily
    # weight 0.088
    alg straw2
    hash 0	# rjenkins1
    item serverc weight 0.029
    item serverd weight 0.029
    item servere weight 0.029
}

# rules
rule replicated_rule {
    id 0
    type replicated
    min_size 1
    max_size 10
    step take default
    step chooseleaf firstn 0 type host
    step emit
}

# end crush map

devices 会识别你的硬盘是ssd还是hdd
有时候会识别错误,但是可以人为干预

types 为故障域
三副本情况下
osd,找所有osd找三个 (三个osd在一个主机上,这样就容易丢数据)
host,所有主机找三个
rack,机架级别的
room、datacenter 房间,数据中心(也是故障域)
osd只能识别到host级别
其他级别就得自定义

ceph 009 管理定义crushmap  故障域插图3
放在同一个根下面就会有关系
三副本,要是你用这个图的数据中心级别,那么两个数据中心是不够的

默认故障域关系

[ceph: root@clienta /]# ceph osd tree
ID  CLASS  WEIGHT   TYPE NAME         STATUS  REWEIGHT  PRI-AFF
-1         0.08817  root default                    #root            
-3         0.02939      host serverc                           
0    hdd  0.00980          osd.0         up   1.00000  1.00000
1    hdd  0.00980          osd.1         up   1.00000  1.00000
2    hdd  0.00980          osd.2         up   1.00000  1.00000
-5         0.02939      host serverd                           
3    hdd  0.00980          osd.3         up   1.00000  1.00000
5    hdd  0.00980          osd.5         up   1.00000  1.00000
7    hdd  0.00980          osd.7         up   1.00000  1.00000
-7         0.02939      host servere                           
4    hdd  0.00980          osd.4         up   1.00000  1.00000
6    hdd  0.00980          osd.6         up   1.00000  1.00000
8    hdd  0.00980          osd.8         up   1.00000  1.00000
[ceph: root@clienta /]# 



# buckets
host serverc {
    id -3		# do not change unnecessarily
    id -4 class hdd		# do not change unnecessarily
    # weight 0.029
    alg straw2
    hash 0	# rjenkins1
    item osd.0 weight 0.010
    item osd.1 weight 0.010
    item osd.2 weight 0.010
}

我们能改变的只是怎么去分布就好了。他这个自动已经通过算法识别好了,没必要改
权重1Tb为1 我一个osd 10G,主机权重为osd的和

root default {
    id -1		# do not change unnecessarily
    id -2 class hdd		# do not change unnecessarily
    # weight 0.088
    alg straw2
    hash 0	# rjenkins1
    item serverc weight 0.029
    item serverd weight 0.029
    item servere weight 0.029
}

写一个rack级别

三个节点在不同机架

rack rack1 {
    id -9		# do not change unnecessarily
    id -10 class hdd		# do not change unnecessarily
    # weight 0.088
    alg straw2
    hash 0	# rjenkins1
    item serverc weight 0.029
}


rack rack2 {
    id -11		# do not change unnecessarily
    id -12 class hdd		# do not change unnecessarily
    # weight 0.088
    alg straw2
    hash 0	# rjenkins1
    item serverd weight 0.029
}

rack rack3 {
    id -13		# do not change unnecessarily
    id -14 class hdd		# do not change unnecessarily
    # weight 0.088
    alg straw2
    hash 0	# rjenkins1
    item servere weight 0.029
}

[ceph: root@clienta /]# cp crushmap.txt crushmap-new.txt
增加到new里面并且反编译




[ceph: root@clienta /]# crushtool -c crushmap-new.txt -o crushmap-new.bin
[ceph: root@clienta /]# ceph osd setcrushmap -i crushmap-new.bin 
21
更新之后20变21  + 1   数据变了就知道更新了



[ceph: root@clienta /]# ceph osd tree
ID   CLASS  WEIGHT   TYPE NAME         STATUS  REWEIGHT  PRI-AFF
-13         0.02899  rack rack3                                 
-3         0.02899      host serverc                           
0    hdd  0.00999          osd.0         up   1.00000  1.00000
1    hdd  0.00999          osd.1         up   1.00000  1.00000
2    hdd  0.00999          osd.2         up   1.00000  1.00000
-11         0.02899  rack rack2                                 
-3         0.02899      host serverd                           
0    hdd  0.00999          osd.3         up   1.00000  1.00000
1    hdd  0.00999          osd.5         up   1.00000  1.00000
2    hdd  0.00999          osd.7         up   1.00000  1.00000
-9         0.02899  rack rack1                                 
-3         0.02899      host servere                           
0    hdd  0.00999          osd.4         up   1.00000  1.00000
1    hdd  0.00999          osd.6         up   1.00000  1.00000
2    hdd  0.00999          osd.8         up   1.00000  1.00000
-1         0.08698  root default                               
-3         0.02899      host serverc                           
0    hdd  0.00999          osd.0         up   1.00000  1.00000
1    hdd  0.00999          osd.1         up   1.00000  1.00000
2    hdd  0.00999          osd.2         up   1.00000  1.00000
-5         0.02899      host serverd                           
3    hdd  0.00999          osd.3         up   1.00000  1.00000
5    hdd  0.00999          osd.5         up   1.00000  1.00000
7    hdd  0.00999          osd.7         up   1.00000  1.00000
-7         0.02899      host servere                           
4    hdd  0.00999          osd.4         up   1.00000  1.00000
6    hdd  0.00999          osd.6         up   1.00000  1.00000
8    hdd  0.00999          osd.8         up   1.00000  1.00000
[ceph: root@clienta /]# 

无根节点,就是三个rack不在一起。一定要有根节点。
创建存储池可以选择根节点。好多个根节点,就得指定

ceph 009 管理定义crushmap  故障域插图4

dc如果是根,那它就到头了。不过不是,那还可以向上延伸。多个根之间没有任何关系。

root dc1 {
        id -15           # do not change unnecessarily
        id -16 class hdd         # do not change unnecessarily
        # weight 0.088
        alg straw2
        hash 0  # rjenkins1
        item rack1 weight 0.029
}

root dc2 {
        id -17          # do not change unnecessarily
        id -18 class hdd         # do not change unnecessarily
        # weight 0.088
        alg straw2
        hash 0  # rjenkins1
        item rack2 weight 0.029
        item rack3 weight 0.029
}

root dc3 {
        id -19          # do not change unnecessarily
        id -20 class hdd         # do not change unnecessarily
        # weight 0.088
        alg straw2
        hash 0  # rjenkins1
        item rack1 weight 0.029
        item rack2 weight 0.029
	    item rack3 weight 0.029

}

增加三个根节点dc
一个机架一个主机,所以一个主机权重就是机架权重

[ceph: root@clienta /]# ceph osd tree
ID   CLASS  WEIGHT   TYPE NAME             STATUS  REWEIGHT  PRI-AFF
-19         0.08698  root dc3                                       
-9         0.02899      rack rack1                                 
-3         0.02899          host serverc                           
0    hdd  0.00999              osd.0         up   1.00000  1.00000
1    hdd  0.00999              osd.1         up   1.00000  1.00000
2    hdd  0.00999              osd.2         up   1.00000  1.00000
-11         0.02899      rack rack2                                 
-5         0.02899          host serverd                           
3    hdd  0.00999              osd.3         up   1.00000  1.00000
5    hdd  0.00999              osd.5         up   1.00000  1.00000
7    hdd  0.00999              osd.7         up   1.00000  1.00000
-13         0.02899      rack rack3                                 
-7         0.02899          host servere                           
4    hdd  0.00999              osd.4         up   1.00000  1.00000
6    hdd  0.00999              osd.6         up   1.00000  1.00000
8    hdd  0.00999              osd.8         up   1.00000  1.00000
-17         0.05798  root dc2                                       
-11         0.02899      rack rack2                                 
-5         0.02899          host serverd                           
3    hdd  0.00999              osd.3         up   1.00000  1.00000
5    hdd  0.00999              osd.5         up   1.00000  1.00000
7    hdd  0.00999              osd.7         up   1.00000  1.00000
-13         0.02899      rack rack3                                 
-7         0.02899          host servere                           
4    hdd  0.00999              osd.4         up   1.00000  1.00000
6    hdd  0.00999              osd.6         up   1.00000  1.00000
8    hdd  0.00999              osd.8         up   1.00000  1.00000
-15         0.02899  root dc1                                       
-9         0.02899      rack rack1                                 
-3         0.02899          host serverc                           
0    hdd  0.00999              osd.0         up   1.00000  1.00000
1    hdd  0.00999              osd.1         up   1.00000  1.00000
2    hdd  0.00999              osd.2         up   1.00000  1.00000
-1         0.08698  root default                                   
-3         0.02899      host serverc                               
0    hdd  0.00999          osd.0             up   1.00000  1.00000
1    hdd  0.00999          osd.1             up   1.00000  1.00000
2    hdd  0.00999          osd.2             up   1.00000  1.00000
-5         0.02899      host serverd                               
3    hdd  0.00999          osd.3             up   1.00000  1.00000
5    hdd  0.00999          osd.5             up   1.00000  1.00000
7    hdd  0.00999          osd.7             up   1.00000  1.00000
-7         0.02899      host servere                               
4    hdd  0.00999          osd.4             up   1.00000  1.00000
6    hdd  0.00999          osd.6             up   1.00000  1.00000
8    hdd  0.00999          osd.8             up   1.00000  1.00000

ceph 009 管理定义crushmap  故障域插图5
对应架构图

无法引用,没有设置规则

pool 6 'pool1' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 208 flags hashpspool stripe_width 0

创建一个pool1 他的规则仍然是crush_rule 0,没有任何改变

[ceph: root@clienta /]# ceph osd crush rule ls
replicated_rule

    # rules
rule replicated_rule {
    id 0
    type replicated   #类型
    min_size 1     #副本数在1-10之间,副本超过11这个规则用不了
    max_size 10
    step take default    # 这个规则引用了default根  并没用我的dc根
    step chooseleaf firstn 0 type host  # 叶子节点就是rack那个 他这为host(默认),故障域(机架的  主机的  osd的)
    step emit
}

自己做一个rule

# rules
rule replicated_rule1 {
        id 1
        type replicated
        min_size 1
        max_size 10
        step take dc3   #根节点
        step chooseleaf firstn 0 type rack   #故障域 
        step emit
}

应用它

[ceph: root@clienta /]# vi crushmap-new.txt 
[ceph: root@clienta /]# 
[ceph: root@clienta /]# crushtool -c crushmap-new.txt -o crushmap-new.bin
[ceph: root@clienta /]# ceph osd setcrushmap -i crushmap-new.bin
24
[ceph: root@clienta /]# ceph osd crush rule ls
replicated_rule
replicated_rule1


[ceph: root@clienta /]# ceph pg dump pgs_brief | grep ^8
dumped pgs_brief
8.4      active+clean  [6,5,1]           6  [6,5,1]               6
8.7      active+clean  [1,5,4]           1  [1,5,4]               1
8.6      active+clean  [6,2,7]           6  [6,2,7]               6
8.1      active+clean  [3,1,8]           3  [3,1,8]               3
8.0      active+clean  [1,7,4]           1  [1,7,4]               1
8.3      active+clean  [8,3,2]           8  [8,3,2]               8
8.2      active+clean  [5,8,1]           5  [5,8,1]               5
8.d      active+clean  [2,7,6]           2  [2,7,6]               2
8.c      active+clean  [0,5,4]           0  [0,5,4]               0
8.f      active+clean  [4,5,1]           4  [4,5,1]               4
8.a      active+clean  [5,4,1]           5  [5,4,1]               5
8.9      active+clean  [6,3,0]           6  [6,3,0]               6
8.b      active+clean  [0,8,5]           0  [0,8,5]               0
8.8      active+clean  [1,3,6]           1  [1,3,6]               1
8.e      active+clean  [5,2,4]           5  [5,2,4]               5
8.5      active+clean  [6,2,7]           6  [6,2,7]               6
8.1a     active+clean  [0,7,4]           0  [0,7,4]               0
8.1b     active+clean  [5,4,0]           5  [5,4,0]               5
8.18     active+clean  [4,2,7]           4  [4,2,7]               4
8.19     active+clean  [8,5,1]           8  [8,5,1]               8
8.1e     active+clean  [1,7,6]           1  [1,7,6]               1
8.1f     active+clean  [7,6,1]           7  [7,6,1]               7
8.1c     active+clean  [2,8,7]           2  [2,8,7]               2
8.1d     active+clean  [6,7,2]           6  [6,7,2]               6
8.12     active+clean  [8,7,0]           8  [8,7,0]               8
8.13     active+clean  [3,4,1]           3  [3,4,1]               3
8.10     active+clean  [0,4,3]           0  [0,4,3]               0
8.11     active+clean  [2,8,3]           2  [2,8,3]               2
8.16     active+clean  [5,4,0]           5  [5,4,0]               5
8.17     active+clean  [8,2,5]           8  [8,2,5]               8
8.14     active+clean  [4,2,7]           4  [4,2,7]               4
8.15     active+clean  [3,8,1]           3  [3,8,1]               3
[ceph: root@clienta /]# 

分布在了三个rack上

# rules
rule replicated_rule1 {
        id 1
        type replicated
        min_size 1
        max_size 10
        step take dc3
        step chooseleaf firstn 0 type osd
        step emit
}

更改规则为osd,可以发现0,1,5 0,1属于一个host,rack

[ceph: root@clienta /]# vi crushmap-new.txt 
[ceph: root@clienta /]# crushtool -c crushmap-new.txt -o crushmap-new.bin
[ceph: root@clienta /]# ceph osd setcrushmap -i crushmap-new.bin
26
[ceph: root@clienta /]# ceph pg dump pgs_brief | grep ^8
dumped pgs_brief
8.4      active+clean  [6,5,3]           6  [6,5,3]               6
8.7      active+clean  [1,5,3]           1  [1,5,3]               1
8.6      active+clean  [6,2,8]           6  [6,2,8]               6
8.1      active+clean  [3,7,1]           3  [3,7,1]               3
8.0      active+clean  [1,0,7]           1  [1,0,7]               1
8.3      active+clean  [8,4,3]           8  [8,4,3]               8
8.2      active+clean  [5,8,7]           5  [5,8,7]               5
8.d      active+clean  [2,7,6]           2  [2,7,6]               2
8.c      active+clean  [0,5,1]           0  [0,5,1]               0
8.f      active+clean  [4,5,1]           4  [4,5,1]               4
8.a      active+clean  [5,4,6]           5  [5,4,6]               5
8.9      active+clean  [6,3,5]           6  [6,3,5]               6
8.b      active+clean  [0,8,2]           0  [0,8,2]               0
8.8      active+clean  [1,3,6]           1  [1,3,6]               1
8.e      active+clean  [5,2,1]           5  [5,2,1]               5
8.5      active+clean  [6,2,7]           6  [6,2,7]               6
8.1a     active+clean  [0,7,4]           0  [0,7,4]               0
8.1b     active+clean  [5,4,0]           5  [5,4,0]               5
8.18     active+clean  [4,2,7]           4  [4,2,7]               4
8.19     active+clean  [8,4,5]           8  [8,4,5]               8
8.1e     active+clean  [1,7,6]           1  [1,7,6]               1
8.1f     active+clean  [7,5,6]           7  [7,5,6]               7
8.1c     active+clean  [2,8,7]           2  [2,8,7]               2
8.1d     active+clean  [6,7,2]           6  [6,7,2]               6
8.12     active+clean  [8,7,0]           8  [8,7,0]               8
8.13     active+clean  [3,4,1]           3  [3,4,1]               3
8.10     active+clean  [0,4,1]           0  [0,4,1]               0
8.11     active+clean  [2,8,6]           2  [2,8,6]               2
8.16     active+clean  [5,4,8]           5  [5,4,8]               5
8.17     active+clean  [8,6,2]           8  [8,6,2]               8
8.14     active+clean  [4,2,7]           4  [4,2,7]               4
8.15     active+clean  [3,8,1]           3  [3,8,1]               3
[ceph: root@clienta /]# 

冲突案例

# rules
rule replicated_rule2 {
        id 1
        type replicated
        min_size 1
        max_size 10
        step take dc2   #根节点 (class ssd)
        step chooseleaf firstn 0 type rack  #firstn0 你有三副本就选择三个rack
        step emit
}

这个是池是三副本默认值,但是dc2是只有两个机架(两个主机,我一个机架一个主机)。那么firstn0他硬要选择三副本,所以会引发冲突

将replicated_rule2加入配置文件

[ceph: root@clienta /]# crushtool -c crushmap-new.txt -o crushmap-new.bin
[ceph: root@clienta /]# ceph osd setcrushmap -i crushmap-new.bin
27
[ceph: root@clienta /]# ceph osd pool create pool4 replicated_rule2
pool 'pool4' created
[ceph: root@clienta /]# ceph pg dump pgs_brief | grep ^9
dumped pgs_brief
9.5      active+undersized    [5,6]           5    [5,6]               5
9.6      active+undersized    [4,5]           4    [4,5]               4
9.7      active+undersized    [6,3]           6    [6,3]               6
9.0      active+undersized    [5,4]           5    [5,4]               5
9.1      active+undersized    [3,4]           3    [3,4]               3
9.2      active+undersized    [8,5]           8    [8,5]               8
9.3      active+undersized    [7,4]           7    [7,4]               7
9.c      active+undersized    [3,4]           3    [3,4]               3
9.d      active+undersized    [3,4]           3    [3,4]               3
9.e      active+undersized    [7,4]           7    [7,4]               7
9.b      active+undersized    [5,4]           5    [5,4]               5
9.8      active+undersized    [8,3]           8    [8,3]               8
9.a      active+undersized    [3,4]           3    [3,4]               3
9.9      active+undersized    [5,8]           5    [5,8]               5
9.f      active+undersized    [4,5]           4    [4,5]               4
9.4      active+undersized    [8,3]           8    [8,3]               8
9.1b     active+undersized    [5,4]           5    [5,4]               5
9.1a     active+undersized    [8,7]           8    [8,7]               8
9.19     active+undersized    [6,3]           6    [6,3]               6
9.18     active+undersized    [5,4]           5    [5,4]               5
9.1f     active+undersized    [6,7]           6    [6,7]               6
9.1e     active+undersized    [7,8]           7    [7,8]               7
9.1d     active+undersized    [6,3]           6    [6,3]               6
9.1c     active+undersized    [5,4]           5    [5,4]               5
9.13     active+undersized    [8,5]           8    [8,5]               8
9.12     active+undersized    [5,8]           5    [5,8]               5
9.11     active+undersized    [8,3]           8    [8,3]               8
9.10     active+undersized    [5,4]           5    [5,4]               5
9.17     active+undersized    [8,7]           8    [8,7]               8
9.16     active+undersized    [5,4]           5    [5,4]               5
9.15     active+undersized    [7,4]           7    [7,4]               7
9.14     active+undersized    [4,3]           4    [4,3]               4
[ceph: root@clienta /]# 

undersized超出了,不是很健康

[ceph: root@clienta /]# ceph -s
cluster:
    id:     2ae6d05a-229a-11ec-925e-52540000fa0c
    health: HEALTH_WARN
            Degraded data redundancy: 32 pgs undersized

services:
    mon: 4 daemons, quorum serverc.lab.example.com,clienta,serverd,servere (age 3h)
    mgr: serverc.lab.example.com.aiqepd(active, since 3h), standbys: clienta.nncugs, servere.kjwyko, serverd.klrkci
    osd: 9 osds: 9 up (since 3h), 9 in (since 9M)
    rgw: 2 daemons active (2 hosts, 1 zones)

data:
    pools:   9 pools, 233 pgs
    objects: 221 objects, 4.9 KiB
    usage:   245 MiB used, 90 GiB / 90 GiB avail
    pgs:     201 active+clean
            32  active+undersized

io:
    client:   71 KiB/s rd, 0 B/s wr, 71 op/s rd, 47 op/s wr

ceph -s 警告了
pg属于降级状态,未达到规定副本数

ceph 009 管理定义crushmap  故障域插图6

修复状态

更改配置文件

# rules
rule replicated_rule2 {
        id 2
        type replicated
        min_size 1
        max_size 10
        step take dc2
        step chooseleaf firstn 2 type rack  #定义2副本
        step emit
        step take dc1  
        step chooseleaf firstn 1 type rack #定义剩下的1个副本
        step emit
} 

[ceph: root@clienta /]# vi crushmap-new.txt 
[ceph: root@clienta /]# crushtool -c crushmap-new.txt -o crushmap-new.bin
[ceph: root@clienta /]# ceph osd setcrushmap -i crushmap-new.bin
28



[ceph: root@clienta /]# ceph pg dump pgs_brief | grep ^9
dumped pgs_brief
9.5      active+clean  [5,6,0]           5  [5,6,0]               5
9.6        activating  [4,5,2]           4  [4,5,2]               4
9.7        activating  [6,3,2]           6  [6,3,2]               6
9.0        activating  [5,4,1]           5  [5,4,1]               5
9.1      active+clean  [3,4,0]           3  [3,4,0]               3
9.2        activating  [8,5,1]           8  [8,5,1]               8
9.3      active+clean  [7,4,0]           7  [7,4,0]               7
9.c        activating  [3,4,2]           3  [3,4,2]               3
9.d      active+clean  [3,4,0]           3  [3,4,0]               3
9.e        activating  [7,4,2]           7  [7,4,2]               7
9.b      active+clean  [5,4,0]           5  [5,4,0]               5
9.8        activating  [8,3,2]           8  [8,3,2]               8

firstn = 0 则是在根节点下寻找3个叶子(副本数)节点存放
firstn > 0 则在根节点寻找2个叶子节点存放副本(三副本),剩余副本则使用下面的规则
firstn -1

ceph 009 管理定义crushmap  故障域插图7

把上面做的还原

创建基于ssd的存储池

元数据 检索要求磁盘较快(ssd)

删掉原有磁盘类型
手动改类型

[ceph: root@clienta /]# ceph osd crush rm-device-class osd.1 
done removing class of osd(s): 1
[ceph: root@clienta /]# ceph osd crush rm-device-class osd.5
done removing class of osd(s): 5
[ceph: root@clienta /]# ceph osd crush rm-device-class osd.6
done removing class of osd(s): 6
[ceph: root@clienta /]# 

[ceph: root@clienta /]# ceph osd tree
ID  CLASS  WEIGHT   TYPE NAME         STATUS  REWEIGHT  PRI-AFF
-1         0.08698  root default                               
-3         0.02899      host serverc                           
1         0.00999          osd.1         up   1.00000  1.00000
0    hdd  0.00999          osd.0         up   1.00000  1.00000
2    hdd  0.00999          osd.2         up   1.00000  1.00000
-5         0.02899      host serverd                           
5         0.00999          osd.5         up   1.00000  1.00000
3    hdd  0.00999          osd.3         up   1.00000  1.00000
7    hdd  0.00999          osd.7         up   1.00000  1.00000
-7         0.02899      host servere                           
6         0.00999          osd.6         up   1.00000  1.00000
4    hdd  0.00999          osd.4         up   1.00000  1.00000
8    hdd  0.00999          osd.8         up   1.00000  1.00000
[ceph: root@clienta /]# 

更改为ssd

[ceph: root@clienta /]# for i in 1 5 6;do  ceph osd crush set-device-class ssd osd.$i; done

[ceph: root@clienta /]# ceph osd  tree
ID  CLASS  WEIGHT   TYPE NAME         STATUS  REWEIGHT  PRI-AFF
-1         0.08698  root default                               
-3         0.02899      host serverc                           
0    hdd  0.00999          osd.0         up   1.00000  1.00000
2    hdd  0.00999          osd.2         up   1.00000  1.00000
1    ssd  0.00999          osd.1         up   1.00000  1.00000
-5         0.02899      host serverd                           
3    hdd  0.00999          osd.3         up   1.00000  1.00000
7    hdd  0.00999          osd.7         up   1.00000  1.00000
5    ssd  0.00999          osd.5         up   1.00000  1.00000
-7         0.02899      host servere                           
4    hdd  0.00999          osd.4         up   1.00000  1.00000
8    hdd  0.00999          osd.8         up   1.00000  1.00000
6    ssd  0.00999          osd.6         up   1.00000  1.00000
[ceph: root@clienta /]# 


[ceph: root@clienta /]# ceph osd crush class ls
[
    "hdd",
    "ssd"
]
[ceph: root@clienta /]# 

命令行创建规则

[ceph: root@clienta /]# ceph osd crush rule create-replicated ssd_rule default host ssd
[ceph: root@clienta /]# ceph osd crush rule ls
replicated_rule
ssd_rule
[ceph: root@clienta /]# 

创建存储池

[ceph: root@clienta /]# ceph osd crush rule create-replicated ssd_rule default host ssd
[ceph: root@clienta /]# ceph osd crush rule ls
replicated_rule
ssd_rule
[ceph: root@clienta /]# ceph osd pool create pool1 ssd_rule
pool 'pool1' created

[ceph: root@clienta /]# ceph osd pool ls detail
pool 1 'device_health_metrics' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 249 flags hashpspool stripe_width 0 pg_num_min 1 application mgr_devicehealth
pool 2 '.rgw.root' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 48 flags hashpspool stripe_width 0 application rgw
pool 3 'default.rgw.log' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 50 flags hashpspool stripe_width 0 application rgw
pool 4 'default.rgw.control' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 52 flags hashpspool stripe_width 0 application rgw
pool 5 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode on last_change 184 lfor 0/184/182 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 8 application rgw
pool 10 'pool1' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 266 flags hashpspool stripe_width 0


[ceph: root@clienta /]# ceph pg dump pgs_brief | grep ^10
dumped pgs_brief
10.6     active+clean  [6,1,5]           6  [6,1,5]               6
10.5     active+clean  [1,5,6]           1  [1,5,6]               1
10.4     active+clean  [5,1,6]           5  [5,1,6]               5
10.3     active+clean  [5,6,1]           5  [5,6,1]               5
10.2     active+clean  [6,5,1]           6  [6,5,1]               6
10.1     active+clean  [1,5,6]           1  [1,5,6]               1
10.0     active+clean  [6,5,1]           6  [6,5,1]               6
10.f     active+clean  [6,1,5]           6  [6,1,5]               6
10.e     active+clean  [1,5,6]           1  [1,5,6]               1
10.d     active+clean  [6,5,1]           6  [6,5,1]               6
10.8     active+clean  [1,5,6]           1  [1,5,6]               1
10.b     active+clean  [6,5,1]           6  [6,5,1]               6
10.9     active+clean  [1,5,6]           1  [1,5,6]               1
10.a     active+clean  [5,1,6]           5  [5,1,6]               5
10.c     active+clean  [1,5,6]           1  [1,5,6]               1
10.7     active+clean  [5,6,1]           5  [5,6,1]               5
10.18    active+clean  [1,5,6]           1  [1,5,6]               1
10.19    active+clean  [1,6,5]           1  [1,6,5]               1
10.1a    active+clean  [6,1,5]           6  [6,1,5]               6
10.1b    active+clean  [6,5,1]           6  [6,5,1]               6
10.1c    active+clean  [5,1,6]           5  [5,1,6]               5
10.1d    active+clean  [6,1,5]           6  [6,1,5]               6
10.1e    active+clean  [1,5,6]           1  [1,5,6]               1
10.1f    active+clean  [6,1,5]           6  [6,1,5]               6
10.10    active+clean  [1,6,5]           1  [1,6,5]               1
10.11    active+clean  [5,6,1]           5  [5,6,1]               5
10.12    active+clean  [6,5,1]           6  [6,5,1]               6
10.13    active+clean  [5,1,6]           5  [5,1,6]               5
10.14    active+clean  [1,6,5]           1  [1,6,5]               1
10.15    active+clean  [6,1,5]           6  [6,1,5]               6
10.16    active+clean  [6,1,5]           6  [6,1,5]               6
10.17    active+clean  [1,6,5]           1  [1,6,5]               1
[ceph: root@clienta /]# 

全部在指定磁盘上

可以整一个性能较好的池(都是ssd盘)

命令行在配置文件里做了这些操作

[ceph: root@clienta /]# ceph osd getcrushmap -o crushmap.bin
44
[ceph: root@clienta /]# crushtool -d crushmap.bin -o crushmap.txt 
[ceph: root@clienta /]# vi crushmap.txt 


rule ssd_rule {
        id 1
        type replicated
        min_size 1
        max_size 10
        step take default class ssd
        step chooseleaf firstn 0 type host
        step emit
}
# devices
device 0 osd.0 class hdd
device 1 osd.1 class ssd
device 2 osd.2 class hdd
device 3 osd.3 class hdd
device 4 osd.4 class hdd
device 5 osd.5 class ssd
device 6 osd.6 class ssd
device 7 osd.7 class hdd
device 8 osd.8 class hdd

执行lab命令(会报错,改变devices标签的时候,选择手动改标签)
lab start map-crush

[ceph: root@clienta /]# ceph osd crush add-bucket cl260 root 
added bucket cl260 type root to crush map
[ceph: root@clienta /]# ceph osd crush add-bucket rack1 rack
added bucket rack1 type rack to crush map
[ceph: root@clienta /]# ceph osd crush add-bucket rack2 rack
added bucket rack2 type rack to crush map
[ceph: root@clienta /]# ceph osd crush add-bucket rack3 rack
added bucket rack3 type rack to crush map
[ceph: root@clienta /]# ceph osd tree
ID   CLASS  WEIGHT   TYPE NAME         STATUS  REWEIGHT  PRI-AFF
-16               0  rack rack3                                 
-15               0  rack rack2                                 
-14               0  rack rack1                                 
-13               0  root cl260                                 
-1         0.08698  root default                               
-3         0.02899      host serverc                           
0    hdd  0.00999          osd.0         up   1.00000  1.00000
2    hdd  0.00999          osd.2         up   1.00000  1.00000
1    ssd  0.00999          osd.1         up   1.00000  1.00000
-5         0.02899      host serverd                           
3    hdd  0.00999          osd.3         up   1.00000  1.00000
7    hdd  0.00999          osd.7         up   1.00000  1.00000
5    ssd  0.00999          osd.5         up   1.00000  1.00000
-7         0.02899      host servere                           
4    hdd  0.00999          osd.4         up   1.00000  1.00000
8    hdd  0.00999          osd.8         up   1.00000  1.00000
6    ssd  0.00999          osd.6         up   1.00000  1.00000
[ceph: root@clienta /]# 




[ceph: root@clienta /]# ceph osd crush add-bucket hostc host
added bucket hostc type host to crush map
[ceph: root@clienta /]# ceph osd crush add-bucket hostd host
added bucket hostd type host to crush map
[ceph: root@clienta /]# ceph osd crush add-bucket hoste host
added bucket hoste type host to crush map
[ceph: root@clienta /]# ceph osd crush move rack1 root=cl260
moved item id -14 name 'rack1' to location {root=cl260} in crush map
[ceph: root@clienta /]# ceph osd crush move rack2 root=cl260
moved item id -15 name 'rack2' to location {root=cl260} in crush map
[ceph: root@clienta /]# ceph osd crush move rack3 root=cl260
moved item id -16 name 'rack3' to location {root=cl260} in crush map
[ceph: root@clienta /]# ceph osd crush move hostc rack=rack1
moved item id -17 name 'hostc' to location {rack=rack1} in crush map
[ceph: root@clienta /]# ceph osd crush move hostd rack=rack2
moved item id -18 name 'hostd' to location {rack=rack2} in crush map
[ceph: root@clienta /]# ceph osd crush move hoste rack=rack3
moved item id -19 name 'hoste' to location {rack=rack3} in crush map
[ceph: root@clienta /]# ceph osd tree
ID   CLASS  WEIGHT   TYPE NAME           STATUS  REWEIGHT  PRI-AFF
-13               0  root cl260                                   
-14               0      rack rack1                               
-17               0          host hostc                           
-15               0      rack rack2                               
-18               0          host hostd                           
-16               0      rack rack3                               
-19               0          host hoste                           
-1         0.08698  root default                                 
-3         0.02899      host serverc                             
0    hdd  0.00999          osd.0           up   1.00000  1.00000
2    hdd  0.00999          osd.2           up   1.00000  1.00000
1    ssd  0.00999          osd.1           up   1.00000  1.00000
-5         0.02899      host serverd                             
3    hdd  0.00999          osd.3           up   1.00000  1.00000
7    hdd  0.00999          osd.7           up   1.00000  1.00000
5    ssd  0.00999          osd.5           up   1.00000  1.00000
-7         0.02899      host servere                             
4    hdd  0.00999          osd.4           up   1.00000  1.00000
8    hdd  0.00999          osd.8           up   1.00000  1.00000
6    ssd  0.00999          osd.6           up   1.00000  1.00000
[ceph: root@clienta /]# 


[ceph: root@clienta /]# ceph osd crush set osd.1 1.0 root=cl260 rack=rack1 host=hostc
set item id 1 name 'osd.1' weight 1 at location {host=hostc,rack=rack1,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.5 1.0 root=cl260 rack=rack1 host=hostc
set item id 5 name 'osd.5' weight 1 at location {host=hostc,rack=rack1,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.6 1.0 root=cl260 rack=rack1 host=hostc
set item id 6 name 'osd.6' weight 1 at location {host=hostc,rack=rack1,root=cl260} to crush map
[ceph: root@clienta /]# 
权重1.0,大小都一样所以1.0



[ceph: root@clienta /]# ceph osd crush set osd.1 1.0 root=cl260 rack=rack1 host=hostc
set item id 1 name 'osd.1' weight 1 at location {host=hostc,rack=rack1,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.5 1.0 root=cl260 rack=rack1 host=hostc
set item id 5 name 'osd.5' weight 1 at location {host=hostc,rack=rack1,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.6 1.0 root=cl260 rack=rack1 host=hostc
set item id 6 name 'osd.6' weight 1 at location {host=hostc,rack=rack1,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.0 1.0 root=cl260 rack=rack2 host=hostd
set item id 0 name 'osd.0' weight 1 at location {host=hostd,rack=rack2,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.3 1.0 root=cl260 rack=rack2 host=hostd
set item id 3 name 'osd.3' weight 1 at location {host=hostd,rack=rack2,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.4 1.0 root=cl260 rack=rack2 host=hostd
set item id 4 name 'osd.4' weight 1 at location {host=hostd,rack=rack2,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.2 1.0 root=cl260 rack=rack3 host=hoste
set item id 2 name 'osd.2' weight 1 at location {host=hoste,rack=rack3,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.7 1.0 root=cl260 rack=rack3 host=hoste
set item id 7 name 'osd.7' weight 1 at location {host=hoste,rack=rack3,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd crush set osd.8 1.0 root=cl260 rack=rack3 host=hoste
set item id 8 name 'osd.8' weight 1 at location {host=hoste,rack=rack3,root=cl260} to crush map
[ceph: root@clienta /]# ceph osd tree
ID   CLASS  WEIGHT   TYPE NAME           STATUS  REWEIGHT  PRI-AFF
-13         9.00000  root cl260                                   
-14         3.00000      rack rack1                               
-17         3.00000          host hostc                           
1    ssd  1.00000              osd.1       up   1.00000  1.00000
5    ssd  1.00000              osd.5       up   1.00000  1.00000
6    ssd  1.00000              osd.6       up   1.00000  1.00000
-15         3.00000      rack rack2                               
-18         3.00000          host hostd                           
0    hdd  1.00000              osd.0       up   1.00000  1.00000
3    hdd  1.00000              osd.3       up   1.00000  1.00000
4    hdd  1.00000              osd.4       up   1.00000  1.00000
-16         3.00000      rack rack3                               
-19         3.00000          host hoste                           
2    hdd  1.00000              osd.2       up   1.00000  1.00000
7    hdd  1.00000              osd.7       up   1.00000  1.00000
8    hdd  1.00000              osd.8       up   1.00000  1.00000
-1               0  root default                                 
-3               0      host serverc                             
-5               0      host serverd                             
-7               0      host servere                             
[ceph: root@clienta /]# 

改配置文件

[ceph: root@clienta /]# ceph osd getcrushmap -o cm-org.bin
66
[ceph: root@clienta /]# crushtool -d cm-org.bin -o cm-org.txt 
[ceph: root@clienta /]# cp cm-org.txt cm-new.txt
[ceph: root@clienta /]# vi cm-new.txt 
[ceph: root@clienta /]# 


rule ssd_first {
        id 2
        type replicated
        min_size 1
        max_size 10
        step take rack1 class ssd
        step chooseleaf firstn 1 type host
        step emit
        step take cl260 class hdd
        step chooseleaf firstn -1 type rack
        step emit

}

第一个副本为主osd,给他放到ssd上
剩下的副本在不同的rack(hdd)里选择一个osd

[ceph: root@clienta /]# crushtool -c cm-new.txt -o cm-new.bin 
[ceph: root@clienta /]# ceph osd setcrushmap -i cm-new.bin 
ceph 67
[ceph: root@clienta /]# ceph osd tree
ID   CLASS  WEIGHT   TYPE NAME           STATUS  REWEIGHT  PRI-AFF
-13         9.00000  root cl260                                   
-14         3.00000      rack rack1                               
-17         3.00000          host hostc                           
1    ssd  1.00000              osd.1       up   1.00000  1.00000
5    ssd  1.00000              osd.5       up   1.00000  1.00000
6    ssd  1.00000              osd.6       up   1.00000  1.00000
-15         3.00000      rack rack2                               
-18         3.00000          host hostd                           
0    hdd  1.00000              osd.0       up   1.00000  1.00000
3    hdd  1.00000              osd.3       up   1.00000  1.00000
4    hdd  1.00000              osd.4       up   1.00000  1.00000
-16         3.00000      rack rack3                               
-19         3.00000          host hoste                           
2    hdd  1.00000              osd.2       up   1.00000  1.00000
7    hdd  1.00000              osd.7       up   1.00000  1.00000
8    hdd  1.00000              osd.8       up   1.00000  1.00000
-1               0  root default                                 
-3               0      host serverc                             
-5               0      host serverd                             
-7               0      host servere                             
[ceph: root@clienta /]# ceph osd crush rule ls
replicated_rule
ssd_rule
ssd_first
[ceph: root@clienta /]# 

创建存储池查看效果

[ceph: root@clienta /]# ceph osd pool create ssdpool ssd_first
pool 'ssdpool' created
[ceph: root@clienta /]# ceph osd pool ls detail
pool 1 'device_health_metrics' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 249 flags hashpspool stripe_width 0 pg_num_min 1 application mgr_devicehealth
pool 2 '.rgw.root' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 48 flags hashpspool stripe_width 0 application rgw
pool 3 'default.rgw.log' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 50 flags hashpspool stripe_width 0 application rgw
pool 4 'default.rgw.control' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 52 flags hashpspool stripe_width 0 application rgw
pool 5 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode on last_change 184 lfor 0/184/182 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 8 application rgw
pool 10 'pool1' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 266 flags hashpspool stripe_width 0
pool 11 'ssdpool' replicated size 3 min_size 2 crush_rule 2 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 338 flags hashpspool stripe_width 0

[ceph: root@clienta /]# ceph pg dump pgs_brief | grep ^11
dumped pgs_brief
11.7              active+clean  [1,7,4]           1  [1,7,4]               1
11.4              active+clean  [5,0,8]           5  [5,0,8]               5
11.5              active+clean  [1,8,3]           1  [1,8,3]               1
11.2              active+clean  [5,8,3]           5  [5,8,3]               5
11.3              active+clean  [6,7,3]           6  [6,7,3]               6
11.0              active+clean  [1,2,0]           1  [1,2,0]               1
11.1              active+clean  [6,0,2]           6  [6,0,2]               6
11.e              active+clean  [5,4,7]           5  [5,4,7]               5
11.f              active+clean  [6,0,7]           6  [6,0,7]               6
11.c              active+clean  [1,4,8]           1  [1,4,8]               1
11.9              active+clean  [1,8,3]           1  [1,8,3]               1
11.a              active+clean  [5,7,3]           5  [5,7,3]               5
11.8              active+clean  [6,8,4]           6  [6,8,4]               6
11.b              active+clean  [6,2,4]           6  [6,2,4]               6
11.d              active+clean  [6,7,3]           6  [6,7,3]               6
11.6              active+clean  [1,8,3]           1  [1,8,3]               1
11.19             active+clean  [1,8,3]           1  [1,8,3]               1
11.18             active+clean  [6,2,4]           6  [6,2,4]               6
11.1b             active+clean  [6,2,4]           6  [6,2,4]               6
11.1a             active+clean  [5,3,7]           5  [5,3,7]               5
11.1d             active+clean  [1,7,4]           1  [1,7,4]               1
11.1c             active+clean  [6,2,4]           6  [6,2,4]               6
11.1f             active+clean  [6,4,8]           6  [6,4,8]               6
11.1e             active+clean  [5,3,7]           5  [5,3,7]               5
11.11             active+clean  [1,0,7]           1  [1,0,7]               1
11.10             active+clean  [5,2,3]           5  [5,2,3]               5
11.13             active+clean  [5,0,8]           5  [5,0,8]               5
11.12             active+clean  [5,2,0]           5  [5,2,0]               5
11.15             active+clean  [5,0,2]           5  [5,0,2]               5
11.14             active+clean  [6,3,2]           6  [6,3,2]               6
11.17             active+clean  [5,0,8]           5  [5,0,8]               5
11.16             active+clean  [1,0,7]           1  [1,0,7]               1
[ceph: root@clienta /]# 

提升读效率,对外提供服务会好
写的时候第一个快了。主osd写完,后复制到另外两hdd

纠删码池和复制池是通用定义域,但是规则会有区别,各用各的

关于纠删码池的规则
ceph osd erasure-code-profile set myprofile k=3 m=2 crush-root=DC2 crush-failure-domain=rack crush-device-class=ssd
ceph osd pool create myecpool 50 50 erasure myprofile
ceph osd crush rule ls

[ceph: root@clienta /]# ceph osd erasure-code-profile set myprofile2  crush-root=cl260 crush-failure-domain=osd
[ceph: root@clienta /]# ceph osd pool create  myecpool2  erasure myprofile2
pool 'myecpool2' created
[ceph: root@clienta /]# ceph pg dump pgs_brief | grep ^13  
dumped pgs_brief
13.1          creating+peering           [8,4,5,7]           8           [8,4,5,7]               8
13.2          creating+peering           [8,0,5,4]           8           [8,0,5,4]               8
13.3          creating+peering           [2,6,0,4]           2           [2,6,0,4]               2

手动改映射

[ceph: root@clienta /]# ceph pg map 11.7
osdmap e338 pg 11.7 (11.7) -> up [1,7,4] acting [1,7,4]
[ceph: root@clienta /]# ceph osd pg-upmap-items 11.7 7 8 
set 11.7 pg_upmap_items mapping to [7->8]
[ceph: root@clienta /]# ceph pg map 11.7
osdmap e340 pg 11.7 (11.7) -> up [1,8,4] acting [1,8,4]
[ceph: root@clienta /]# 

命令概括

1. 假设每台主机的最后一个osd为ssd
for i in 0 3 6;do ceph osd crush rm-device-class osd.$i;done
for i in 0 3 6;do ceph osd crush set-device-class ssd osd.$i;done
ceph osd crush class ls 
ceph osd crush rule create-replicated ssd_rule default host ssd
ceph osd crush rule ls 
1. 创建基于ssd_rule规则的存储池
ceph osd pool create cache 64 64 ssd_rule
1. 将一个现有的池迁移至ssd的osd上
ceph osd pool set cephfs_metadata crush_rule ssd_rule
1. 写入数据,测试数据分布
rados -p cache put test test.txt
ceph osd map cache test

3.命令行管理crushmap
1.移除osd.1 osd.5 osd.6的设备类型
ceph  osd crush rm-device-class osd.1
ceph  osd crush rm-device-class osd.5
ceph  osd crush rm-device-class osd.6

2.设置osd.1 osd.5 osd.6的设备类型
ceph osd crush set-device-class ssd osd.1
ceph osd crush set-device-class ssd osd.5
ceph osd crush set-device-class ssd osd.6

3.添加root节点
ceph osd crush add-bucket cl260 root

4.添加rack节点
ceph osd crush add-bucket rack1 rack
ceph osd crush add-bucket rack2 rack
ceph osd crush add-bucket rack3 rack

5.添加主机节点
ceph osd crush add-bucket hostc host
ceph osd crush add-bucket hostd host
ceph osd crush add-bucket hoste host

6.将rack移动到root节点下
ceph osd crush move rack1 root=cl260
ceph osd crush move rack2 root=cl260
ceph osd crush move rack3 root=cl260

7.将host移动到rack下
ceph osd crush move hostc rack=rack1
ceph osd crush move hostd rack=rack2
ceph osd crush move hoste rack=rack3

8.将osd移动到host下
ceph osd crush set osd.1 1.0 root=cl260 rack=rack1 host=hostc
ceph osd crush set osd.5 1.0 root=cl260 rack=rack1 host=hostc
ceph osd crush set osd.6 1.0 root=cl260 rack=rack1 host=hostc
ceph osd crush set osd.0 1.0 root=cl260 rack=rack1 host=hostd
ceph osd crush set osd.3 1.0 root=cl260 rack=rack1 host=hostd
ceph osd crush set osd.4 1.0 root=cl260 rack=rack1 host=hostd
ceph osd crush set osd.2 1.0 root=cl260 rack=rack1 host=hoste
ceph osd crush set osd.7 1.0 root=cl260 rack=rack1 host=hoste
ceph osd crush set osd.8 1.0 root=cl260 rack=rack1 host=hoste


9添加规则
ceph osd getcrushmap -o cm-org.bin
crushtool -d cm-org.bin -o cm-org.txt
cp cm-org.txt cm-new.txt 
vi cm-new.txt
rule ssd_first {
    id 2
    type replicated
    min_size 1
    max_size 10
    step take rack1
    step chooseleaf firstn 1 type host  # 第1个副本在rack1上
    step emit
    step take cl260 class hdd
    step chooseleaf firstn -1 type rack # 剩余副本在cl260根下的hdd上
    step emit
}

crushtool -c cm-new.txt -o cm-new.bin
ceph osd setcrushmap -i cm-new.bin
ceph osd tree
ceph osd crush ls 
ceph osd crush rule ls 
ceph osd pool create ssdpool ssd_first

ceph pg dump pgs_brief | grep ^10

文章来源于互联网:ceph 009 管理定义crushmap 故障域

THE END
分享
二维码