Ceph Cluster 02 - OSD/RBD

Ceph的使用笔记。

USAGE

创建存储池

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
# 创建一个PG为64 ,PGP为64的存储池。
[ceph: root@ceph01 /]# ceph osd pool create test-64 64 64
pool 'test-64' created

# 创建一个自动识别的大小的存储池。
[ceph: root@ceph01 /]# ceph osd pool create test
pool 'test' created

# 查看已经存在的存储池。
[ceph: root@ceph01 /]# ceph osd pool ls
device_health_metrics
test
test-64

# 查看存储池的PG 和 PGP 的信息和关系。
[ceph: root@ceph01 /]# ceph pg ls-by-pool test

# 查看OSD的状态。
[ceph: root@ceph01 /]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0.58557 root default
-5 0.14639 host ceph01
2 hdd 0.04880 osd.2 up 1.00000 1.00000
7 hdd 0.04880 osd.7 up 1.00000 1.00000
11 hdd 0.04880 osd.11 up 1.00000 1.00000
-7 0.14639 host ceph02
3 hdd 0.04880 osd.3 up 1.00000 1.00000
6 hdd 0.04880 osd.6 up 1.00000 1.00000
10 hdd 0.04880 osd.10 up 1.00000 1.00000
-9 0.14639 host ceph03
1 hdd 0.04880 osd.1 up 1.00000 1.00000
5 hdd 0.04880 osd.5 up 1.00000 1.00000
9 hdd 0.04880 osd.9 up 1.00000 1.00000
-3 0.14639 host ceph04
0 hdd 0.04880 osd.0 up 1.00000 1.00000
4 hdd 0.04880 osd.4 up 1.00000 1.00000
8 hdd 0.04880 osd.8 up 1.00000 1.00000

# 测试上传一个文件并且查看文件的状态和信息。
[ceph: root@ceph01 rpm]# pwd
/var/lib/rpm
[ceph: root@ceph01 rpm]# ls -lh ./Packages
-rw-r--r-- 1 root root 20M Jul 8 17:56 ./Packages

[ceph: root@ceph01 rpm]# rados put msg1 ./Packages --pool=test
[ceph: root@ceph01 rpm]# rados put msg1 ./Packages --pool=test-64

[ceph: root@ceph01 rpm]# rados ls --pool=test
msg1

[ceph: root@ceph01 rpm]# ceph osd map test msg1
osdmap e68 pool 'test' (2) object 'msg1' -> pg 2.c833d430 (2.10) -> up ([9,3,0], p9) acting ([9,3,0], p9)

[ceph: root@ceph01 rpm]# ceph osd map test-64 msg1
osdmap e68 pool 'test-64' (3) object 'msg1' -> pg 3.c833d430 (3.30) -> up ([2,0,3], p2) acting ([2,0,3], p2)

# 删除上传的文件
[ceph: root@ceph01 rpm]# rados rm msg1 --pool=test
[ceph: root@ceph01 rpm]# rados rm msg1 --pool=test-64
[ceph: root@ceph01 rpm]# rados ls --pool=test
[ceph: root@ceph01 rpm]# rados ls --pool=test-64

# 删除刚刚添加的存储池。(临时的解决方案是使用如下的命令, 永久生效的配置写入 /etc/ceph/ceph.conf)
[ceph: root@ceph01 ~]# ceph tell mon.* injectargs '--mon-allow-pool-delete=true'
mon.ceph01: {}
mon.ceph01: mon_allow_pool_delete = 'true'
mon.ceph02: {}
mon.ceph02: mon_allow_pool_delete = 'true'
mon.ceph03: {}
mon.ceph03: mon_allow_pool_delete = 'true'

# 执行删除命令就不会报错了。
[ceph: root@ceph01 ~]# ceph osd pool rm test test --yes-i-really-really-mean-it
pool 'test' removed
[ceph: root@ceph01 ~]# ceph osd pool rm test-64 test-64 --yes-i-really-really-mean-it
pool 'test-64' removed

# 创建一个KVM的块设备池。
[ceph: root@ceph01 ~]# ceph osd pool create kvm 256 256
pool 'kvm' created

# 设置Application enable flag
[ceph: root@ceph01 ~]# ceph osd pool application enable kvm rbd
enabled application 'rbd' on pool 'kvm'
# 初始化池。
[ceph: root@ceph01 ~]# rbd pool init -p kvm
# 创建一个可挂载的镜像。
[ceph: root@ceph01 ~]# rbd create disk01 --size 5G --pool kvm
# 查看镜像的相关信息
[ceph: root@ceph01 ~]# rbd ls --pool kvm
disk01
truesize 5 GiB in 1280 objects
trueorder 22 (4 MiB objects)
truesnapshot_count: 0
trueid: 1703d21a8f2ba
trueblock_name_prefix: rbd_data.1703d21a8f2ba
trueformat: 2
truefeatures: layering, exclusive-lock, object-map, fast-diff, deep-flatten
trueop_features:
trueflags:
truecreate_timestamp: Mon Aug 23 09:13:58 2021
trueaccess_timestamp: Mon Aug 23 09:13:58 2021
truemodify_timestamp: Mon Aug 23 09:13:58 2021

# 安装Ceph的客户端以及rbd命令。
[root@HaydenArchDesktop hayden]# sudo pacman -S ceph
[root@HaydenArchDesktop hayden]# rbd -p kvm map disk01
/dev/rbd0
[root@HaydenArchDesktop hayden]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTS
rbd0 254:0 0 5G 0 disk

# 格式化文件系统。
[root@HaydenArchDesktop test]# mkfs.xfs /dev/rbd0
# 挂载。
[root@HaydenArchDesktop test]# mount /dev/rbd0 /opt/test/
# 移除挂载
[root@HaydenArchDesktop test]# umount /opt/test
# 取消这个映射关系。
[root@HaydenArchDesktop test]# rbd unmap /dev/rbd0
文章目录
  1. 1. USAGE
    1. 1.1. 创建存储池