ceph 006 rbd高级特性 rbd快照镜像克隆 rbd缓存 rbd增量备份 rbd镜像单向同步

版本

[root@clienta ~]# ceph -v

ceph version 16.2.0-117.el8cp (0e34bb74700060ebfaa22d99b7d2cdc037b28a57) pacific (stable)

[root@clienta ~]#

rbd高级特性

创建一个块

[root@clienta ~]# ceph osd pool create rbd

pool 'rbd' created

[root@clienta ~]# rbd pool init rbd

[root@clienta ~]# ceph osd pool ls detail | grep rbd

pool 6 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 226 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd

[root@clienta ~]#

[root@clienta ~]#

[root@clienta ~]#

[root@clienta ~]# rbd create --size 1G -p rbd image1

[root@clienta ~]# rbd -p rbd ls

image1

[root@clienta ~]# rbd info image1

rbd image 'image1':

    size 1 GiB in 256 objects

    order 22 (4 MiB objects)

    snapshot_count: 0

    id: fab9c898fcc1

    block_name_prefix: rbd_data.fab9c898fcc1

    format: 2

    features: layering, exclusive-lock, object-map, fast-diff, deep-flatten

    op_features:

    flags:

    create_timestamp: Sun Aug 14 05:10:43 2022

    access_timestamp: Sun Aug 14 05:10:43 2022

    modify_timestamp: Sun Aug 14 05:10:43 2022

[root@clienta ~]#

默认特性

[root@clienta ~]# ceph config ls | grep feature

enable_experimental_unrecoverable_data_corrupting_features

mon_debug_no_initial_persistent_features

rbd_default_features

[root@clienta ~]# ceph config get osd rbd_default_features

layering,exclusive-lock,object-map,fast-diff,deep-flatten

[root@clienta ~]#

layering 支持克隆

striping 支持条带化  （raid）

exclusive-lock  排他锁。是一个分布式锁，主要用于防止多个客户端同时写入image导致数据不一致问题。（保证性能一致，性能可能下降，多客户端访问）

object-map  精简配置，不立马分配空间，写数据了才分配  （require exclusive-lock）

fast-diff   io加速

deep-flatten  扁平化RBD镜像的所有快照

journaling   支持日志 镜像功能依赖这个

data-pool    ec数据池支持（ec纠删码池）

[root@clienta ~]# rbd feature enable rbd/image1 journaling

[root@clienta ~]# rbd info image1

rbd image 'image1':

    size 1 GiB in 256 objects

    order 22 (4 MiB objects)

    snapshot_count: 0

    id: fab9c898fcc1

    block_name_prefix: rbd_data.fab9c898fcc1

    format: 2

    features: layering, exclusive-lock, object-map, fast-diff, deep-flatten, journaling

    op_features:

开启新特性   （我的版本不支持data-pool功能）

[root@clienta ~]# rbd feature disable rbd/image1 journaling

有的报错会是不支持[0x40]  可以通过id来判断

rbd_defaults_features=24   (8 object-map + 16 fast-diff)

striping(类似raid条带)

stripe-unit=1M

stripe-count=4 数据块

并发写4M

[root@clienta ~]# rbd create --size 1G --stripe-unit=1M --stripe-count=4 image2

[root@clienta ~]# rbd info image2

rbd image 'image2':

    size 1 GiB in 256 objects

    order 22 (4 MiB objects)

    snapshot_count: 0

    id: fad5ece2e6b6

    block_name_prefix: rbd_data.fad5ece2e6b6

    format: 2

    features: layering, striping, exclusive-lock, object-map, fast-diff, deep-flatten

    op_features:

    flags:

    create_timestamp: Sun Aug 14 05:38:12 2022

    access_timestamp: Sun Aug 14 05:38:12 2022

    modify_timestamp: Sun Aug 14 05:38:12 2022

    stripe unit: 1 MiB

    stripe count: 4

[root@clienta ~]#

默认一次写一个块，一个块写满4M

[root@clienta ~]# rbd map rbd/image2

/dev/rbd0

1.rbd快照

[root@clienta ~]# rbd showmapped

id  pool  namespace  image   snap  device

0   rbd              image2  -     /dev/rbd0

[root@clienta ~]# rbd unmap  rbd/image2

[root@clienta ~]# rbd rm image1

Removing image: 100% complete...done.

[root@clienta ~]# rbd rm image2

Removing image: 100% complete...done.

[root@clienta ~]# 

[root@clienta ~]# rbd create rbd/image1 --size 1G

[root@clienta ~]# rbd map image1

/dev/rbd0

[root@clienta ~]# mkfs.xfs /dev/rbd0

meta-data=/dev/rbd0              isize=512    agcount=8, agsize=32768 blks

        =                       sectsz=512   attr=2, projid32bit=1

        =                       crc=1        finobt=1, sparse=1, rmapbt=0

        =                       reflink=1

data     =                       bsize=4096   blocks=262144, imaxpct=25

        =                       sunit=16     swidth=16 blks

naming   =version 2              bsize=4096   ascii-ci=0, ftype=1

log      =internal log           bsize=4096   blocks=2560, version=2

        =                       sectsz=512   sunit=16 blks, lazy-count=1

realtime =none                   extsz=4096   blocks=0, rtextents=0

Discarding blocks...Done.

[root@clienta ~]# mkdir /mnt/rbddev

[root@clienta ~]# mount /dev/rbd0 /mnt/rbddev/

[root@clienta ~]# 

[root@clienta ~]# cp /etc/passwd   /etc/profile  /mnt/rbddev/

[root@clienta ~]# cd /mnt/rbddev/

[root@clienta rbddev]# dd if=/dev/zero of=file1 bs=1M count=10

10+0 records in

10+0 records out

10485760 bytes (10 MB, 10 MiB) copied, 0.0169812 s, 617 MB/s

[root@clienta rbddev]# ls

file1  passwd  profile

[root@clienta rbddev]# sync

[root@clienta rbddev]# rbd snap  create rbd/image1@snap1

Creating snap: 100% complete...done.

[root@clienta rbddev]# rbd info image1

rbd image 'image1':

    size 1 GiB in 256 objects

    order 22 (4 MiB objects)

    snapshot_count: 1

    id: fae5f58d3b62

    block_name_prefix: rbd_data.fae5f58d3b62

    format: 2

    features: layering, exclusive-lock, object-map, fast-diff, deep-flatten

    op_features:

    flags:

    create_timestamp: Sun Aug 14 05:42:02 2022

    access_timestamp: Sun Aug 14 05:42:02 2022

    modify_timestamp: Sun Aug 14 05:42:02 2022

[root@clienta rbddev]# rbd snap ls image1

SNAPID  NAME   SIZE   PROTECTED  TIMESTAMP

 4  snap1  1 GiB             Sun Aug 14 05:48:07 2022

不可以挂载快照 （只读的不让挂）

快照和原本的块，信息一样

测试快照效果

[root@clienta rbddev]# ls

file1  passwd  profile

[root@clienta rbddev]# rm -rf passwd

[root@clienta rbddev]# ls

file1  profile

[root@clienta ~]# umount /mnt/rbddev

[root@clienta ~]# rbd unmap /dev/rbd0

[root@clienta ~]# rbd snap rollback rbd/image1@snap1

Rolling back to snapshot: 100% complete...done.

[root@clienta ~]#

他需要取消映射与挂载，才可以恢复

[root@clienta ~]# rbd showmapped

id  pool  namespace  image   snap  device

0   rbd              image1  -     /dev/rbd0

[root@clienta ~]# rbd unmap /dev/rbd0

rbd: sysfs write failed

回滚成功

[root@clienta ~]# mount /dev/rbd0 /mnt/rbddev/

[root@clienta ~]# cd /mnt/rbddev/

[root@clienta rbddev]# ls

file1  passwd  profile

存储池是对象块如果丢了，对单个对象进行回滚，这个是直接回滚到快照时的镜像

快照原理

拍快照时，建议暂停访问文件系统（停业务）

快照时源镜像的只读副本，写时复制功能COW 节省空间

快照一开始不占空间，当你将原文件修改时，原文件拷贝到快照，并被修改（写前拷贝）

有的快照，他用一次就没了

但是这里。你用了他还会存在

SNAPID  NAME   SIZE   PROTECTED  TIMESTAMP

    4  snap1  1 GiB             Sun Aug 14 05:48:07 2022

[root@clienta rbddev]#

快照越多性能越慢

偷懒举一个例子

[root@clienta rbddev]# rbd snap rm image1@snap1

Removing snap: 100% complete...done.

[root@clienta rbddev]# rbd snap ls image1

[root@clienta rbddev]#

rbd快照是原始镜像的只读副本

rbd克隆功能（依赖快照）

快照数据不会变，基于快照，再去克隆

2.镜像克隆

这是可以读写的

[root@clienta rbddev]# rbd clone rbd/image1@snap1 rbd/clone1

2022-08-14T06:30:28.220-0400 7febbdb2f700 -1 librbd::image::CloneRequest: 0x55f9906c6b00 validate_parent: parent snapshot must be protected

rbd: clone error: (22) Invalid argument

[root@clienta rbddev]# rbd snap  protect rbd/image1@snap1

[root@clienta rbddev]# rbd clone rbd/image1@snap1 rbd/clone1

[root@clienta rbddev]# 

[root@clienta rbddev]# rbd ls

clone1

image1

[root@clienta rbddev]# 

[root@clienta ~]# blkid

/dev/vda1: PARTUUID="fac7f1fb-3e8d-4137-a512-961de09a5549"

/dev/vda2: SEC_TYPE="msdos" UUID="7B77-95E7" BLOCK_SIZE="512" TYPE="vfat" PARTUUID="68b2905b-df3e-4fb3-80fa-49d1e773aa33"

/dev/vda3: LABEL="root" UUID="d47ead13-ec24-428e-9175-46aefa764b26" BLOCK_SIZE="512" TYPE="xfs" PARTUUID="6264d520-3fb9-423f-8ab8-7a0a8e3d3562"

/dev/rbd1: UUID="4d62ee62-03dc-4acc-b925-eeea77238c1b" BLOCK_SIZE="512" TYPE="xfs"

/dev/rbd0: UUID="4d62ee62-03dc-4acc-b925-eeea77238c1b" BLOCK_SIZE="512" TYPE="xfs"

[root@clienta ~]#

克隆和镜像一模一样

克隆可以继续创建文件，读写副本  （rbd快照用作基础）（无操作时，依然是cow映射，还是看到的是原始镜像）

克隆可以独立于原始镜像

快照为cow   克隆支持cor 写前复制，读前复制

启用cor，不然你每次读的都是源镜像，源镜像压力增大，不如读前复制到克隆里

查看子镜像。

并且使子镜像（克隆）独立

[root@clienta ~]# rbd children image1@snap1

rbd/clone1

[root@clienta ~]# rbd flatten rbd/clone1

Image flatten: 100% complete...done.

[root@clienta ~]# rbd children image1@snap1

[root@clienta ~]#

3.rbd缓存

服务端直接写到硬盘，他的数据是没有经过缓存的数据聚合

但是用客户端的缓存先写客户端内存，然后刷到集群那一端缓存功能默认开启

第一个和第六个都得开，才是开始回写

可以查

[root@clienta ~]# ceph config ls | grep rbd_cache

rbd_cache

rbd_cache_policy

rbd_cache_writethrough_until_flush

rbd_cache_size

rbd_cache_max_dirty

rbd_cache_target_dirty

rbd_cache_max_dirty_age

rbd_cache_max_dirty_object

rbd_cache_block_writes_upfront

4.镜像的导入导出

像镜像的备份

第一个ceph集群

[root@clienta ~]# rbd ls

clone1

image1

[root@clienta ~]# rbd snap ls image1

SNAPID  NAME   SIZE   PROTECTED  TIMESTAMP

    6  snap1  1 GiB  yes        Sun Aug 14 06:29:50 2022

[root@clienta ~]#

[root@clienta ~]# rbd export rbd/image1 image1-v1

Exporting image: 100% complete...done.

[root@clienta ~]# ls

-  image1-v1

[root@clienta ~]# rsync image1-v1  root@serverf:~

Warning: Permanently added 'serverf,172.25.250.15' (ECDSA) to the list of known hosts.

[root@clienta ~]#

第二个ceph集群

[root@serverf ~]# ceph osd pool create f-rbd

pool 'f-rbd' created

[root@serverf ~]# rbd pool init f-rbd

[root@serverf ~]#

[root@serverf ~]# rbd import image1-v1 f-rbd/image1

Importing image: 100% complete...done.

[root@serverf ~]# rbd info f-rbd/image1

    rbd image 'image1':

    size 1 GiB in 256 objects

    order 22 (4 MiB objects)

    snapshot_count: 0

    id: ac64adf717b3

    block_name_prefix: rbd_data.ac64adf717b3

    format: 2

    features: layering, exclusive-lock, object-map, fast-diff, deep-flatten

    op_features:

    flags:

    create_timestamp: Sun Aug 14 07:42:46 2022

    access_timestamp: Sun Aug 14 07:42:46 2022

    modify_timestamp: Sun Aug 14 07:42:46 2022

[root@serverf ~]# rbd map f-rbd/image1

/dev/rbd0

[root@serverf ~]# mount /dev/rbd0 /mnt/

[root@serverf ~]# cd /mnt/

[root@serverf mnt]# ls

file1  passwd  profile

[root@serverf mnt]#

基于时间点备份成功，那我继续在clienta修改镜像呢

[root@clienta ~]# rbd map image1

/dev/rbd0

[root@clienta ~]# mount /dev/rbd0 /mnt/rbddev/

[root@clienta ~]#

[root@clienta rbddev]# cp /etc/group .

[root@clienta rbddev]# ls

file1  group  passwd  profile

可能是错误示范，得从快照导出，标记出时间点

再来

[root@clienta ~]# rbd snap  ls image1

SNAPID  NAME   SIZE   PROTECTED  TIMESTAMP

    6  snap1  1 GiB  yes        Sun Aug 14 06:29:50 2022

[root@clienta ~]# rbd export rbd/image1@snap1 image-snap1

Exporting image: 100% complete...done.

[root@clienta ~]#

[root@clienta ~]# rsync image-snap1 root@serverf:~

[root@clienta ~]# 

[root@serverf ~]# rbd import image-snap1  f-rbd/image1

Importing image: 100% complete...done.

[root@serverf ~]# rbd map f-rbd/image1

/dev/rbd0

[root@serverf ~]# mount /dev/rbd0 /mnt

cd[root@serverf ~]# cd /mnt

[root@serverf mnt]# ls

file1  passwd  profile

[root@clienta ~]# rbd export-diff    --from-snap snap1  rbd/image1 image1-v1-v2

[root@clienta ~]# rsync  image1-v1-v2  root@serverf:~

客户端(在之前的全量备份的基础上，再打一个快照)

[root@serverf ~]# rbd snap create f-rbd/image1@snap1

Creating snap: 100% complete...done.

[root@serverf ~]# rbd import-diff image1-v1-v2 f-rbd/image1

Importing image diff: 100% complete...done.

[root@serverf ~]# mount /dev/rbd0 /mnt

[root@serverf ~]# cd /mnt

[root@serverf mnt]# ls

file1  group  passwd  profile

理一遍思路（增量备份）

[root@clienta ~]# rbd create test  --size 1G --pool rbd

[root@clienta ~]# rbd ls

clone1

image1

test

[root@clienta ~]# rbd map test

/dev/rbd0

[root@clienta ~]# mkfs.xfs /dev/rbd0

meta-data=/dev/rbd0              isize=512    agcount=8, agsize=32768 blks

        =                       sectsz=512   attr=2, projid32bit=1

        =                       crc=1        finobt=1, sparse=1, rmapbt=0

        =                       reflink=1

data     =                       bsize=4096   blocks=262144, imaxpct=25

        =                       sunit=16     swidth=16 blks

naming   =version 2              bsize=4096   ascii-ci=0, ftype=1

log      =internal log           bsize=4096   blocks=2560, version=2

        =                       sectsz=512   sunit=16 blks, lazy-count=1

realtime =none                   extsz=4096   blocks=0, rtextents=0

Discarding blocks...Done.

[root@clienta ~]# mount /dev/rbd0 /mnt/rbddev/

[root@clienta ~]# cd /mnt/rbddev/

[root@clienta rbddev]# ls

[root@clienta rbddev]# touch file{1..10}

[root@clienta rbddev]# ls

file1  file10  file2  file3  file4  file5  file6  file7  file8  file9

[root@clienta rbddev]#

[root@clienta ~]# umount /mnt/rbddev

[root@clienta ~]# rbd unmap rbd/test

[root@clienta ~]# rbd showmapped

[root@clienta ~]# rbd export rbd/test test-v1

Exporting image: 100% complete...done.

[root@clienta ~]#

[root@clienta ~]# ls

-  test-v1

[root@clienta ~]# rsync test-v1 root@serverf:~

[root@serverf ~]# ls

-  ceph  test-v1

[root@serverf ~]# rbd import test-v1 f-rbd/test1

Importing image: 100% complete...done.

[root@serverf ~]# rbd map f-rbd/test1

/dev/rbd0

[root@serverf ~]# mount /dev/rbd0 /mnt/

[root@serverf ~]# cd /mnt/

[root@serverf mnt]# ls

file1  file10  file2  file3  file4  file5  file6  file7  file8  file9

[root@serverf mnt]#

卸载

[root@serverf mnt]# cd

[root@serverf ~]#

[root@serverf ~]# umount /mnt

[root@serverf ~]# rbd unmap f-rbd/test1

[root@serverf ~]#

第一次全量备份完成

两边同时打上快照

[root@clienta ~]# rbd snap  create test@testsnap1

Creating snap: 100% complete...done.

[root@serverf ~]# rbd snap create f-rbd/test1@testsnap1     #上下两个快照名字得一样

Creating snap: 100% complete...done.

倘若名字不一样，在最后一步会导致报错

[root@serverf ~]# rbd import-diff snap1-snap2 f-rbd/test1

start snapshot 'testsnap1' does not exist in the image, aborting

Importing image diff: 0% complete...failed.

在主节点增量

[root@clienta ~]# rbd map rbd/test

/dev/rbd0

[root@clienta ~]# mount /dev/rbd0 /mnt/rbddev/

[root@clienta ~]# cd /mnt/rbddev/

[root@clienta rbddev]# ls

file1  file10  file2  file3  file4  file5  file6  file7  file8  file9

[root@clienta rbddev]# touch hello

[root@clienta rbddev]# touch mqy

[root@clienta rbddev]# cp /etc/passwd .

[root@clienta rbddev]# ls

file1  file10  file2  file3  file4  file5  file6  file7  file8  file9  hello  mqy  passwd

[root@clienta rbddev]#

[root@clienta ~]# umount /mnt/rbddev

[root@clienta ~]# rbd unmap rbd/test

[root@clienta ~]#

创建第二次快照，并导出差异数据，传输

[root@clienta ~]# rbd snap create rbd/test@testsnap2

Creating snap: 100% complete...done.

[root@clienta ~]# rbd export-diff --from-snap testsnap1 rbd/test@testsnap2  snap1-snap2

Exporting image: 100% complete...done.

[root@clienta ~]# rsync snap1-snap2 root@serverf:~

[root@clienta ~]#

备份节点，同步增量的数据

[root@serverf ~]# rbd import-diff snap1-snap2 f-rbd/test1

Importing image diff: 100% complete...done.

[root@serverf ~]# rbd map f-rbd/test1

/dev/rbd0

[root@serverf ~]# mount /dev/rbd0 /mnt/

[root@serverf ~]# cd /mnt/

[root@serverf mnt]# ls

file1  file10  file2  file3  file4  file5  file6  file7  file8  file9  hello  mqy  passwd

serverf在这个版本，非得给被增量的节点打一个快照（意义不明，以前版本并不用）

防止集群出现故障，可以拷贝，备份一下

5.rbd镜像功能（单向同步）

容灾，主集群直接挂掉了，不可访问。无缝切换到备用集群

人为的问题不能解决。主被误写，备集群也误写

人为的操作，只能通过备份解决。搞错了，可以再用备份导进来

两边强一致，导致他们在同步时，延迟就会很高，备集群，写成功，才会告诉主集群，然后主机群才写。

RBD-mirror

此功能需要开启journal 性能会降低一点，先写日志。备集群读，拉取日志到备用集群。然后修改image。备集群主动拉取日志。这是通过日志，而不是原来缓慢的三副本写完同步。

镜像级别同步，单独设置1对1镜像同步

池级别同步池和池的一对一单双向区别是看谁安装rbd-mirror

双向的话两边都有rbd-mirror

实践

[root@clienta ~]# ceph osd pool create rbd

pool 'rbd' created

[root@clienta ~]# rbd pool init rbd

[root@serverf ~]# ceph osd pool create rbd

pool 'rbd' created

[root@serverf ~]# rbd pool init rbd

备集群需要rbd-mirror去要主集群的数据

[root@serverf ~]# ceph orch apply rbd-mirror --placement=serverf.lab.example.com

Scheduled rbd-mirror update...

[root@serverf ~]# ceph -s

cluster:

    id:     0bf7c358-25e1-11ec-ae02-52540000fa0f

    health: HEALTH_OK

services:

    mon:        1 daemons, quorum serverf.lab.example.com (age 9m)

    mgr:        serverf.lab.example.com.vuoooq(active, since 7m)

    osd:        5 osds: 5 up (since 8m), 5 in (since 10M)

    rbd-mirror: 1 daemon active (1 hosts)

    rgw:        1 daemon active (1 hosts, 1 zones)

data:

    pools:   6 pools, 137 pgs

    objects: 222 objects, 4.9 KiB

    usage:   96 MiB used, 50 GiB / 50 GiB avail

    pgs:     137 active+clean

多出rbd-mirror

主集群创建镜像

[root@clienta ~]# rbd create image1 --size 1024 --pool rbd --image-feature exclusive-lock,journaling

[root@clienta ~]# rbd info image1

rbd image 'image1':

    size 1 GiB in 256 objects

    order 22 (4 MiB objects)

    snapshot_count: 0

    id: d3bf73566d45

    block_name_prefix: rbd_data.d3bf73566d45

    format: 2

    features: exclusive-lock, journaling

    op_features:

    flags:

    create_timestamp: Sun Aug 14 11:12:54 2022

    access_timestamp: Sun Aug 14 11:12:54 2022

    modify_timestamp: Sun Aug 14 11:12:54 2022

    journal: d3bf73566d45

    mirroring state: disabled

[root@clienta ~]#

开启池模式

[root@clienta ~]# rbd mirror pool enable rbd pool

[root@clienta ~]# rbd info image1

rbd image 'image1':

    size 1 GiB in 256 objects

    order 22 (4 MiB objects)

    snapshot_count: 0

    id: d3bf73566d45

    block_name_prefix: rbd_data.d3bf73566d45

    format: 2

    features: exclusive-lock, journaling

    op_features:

    flags:

    create_timestamp: Sun Aug 14 11:12:54 2022

    access_timestamp: Sun Aug 14 11:12:54 2022

    modify_timestamp: Sun Aug 14 11:12:54 2022

    journal: d3bf73566d45

    mirroring state: enabled

    mirroring mode: journal

    mirroring global id: 431aba12-fbe6-488f-b57e-320ab526d47a

    mirroring primary: true

[root@clienta ~]#

建立联系

将主集群取名prod并将信息导出

[root@clienta ~]# rbd mirror pool peer bootstrap create --site-name prod rbd > /root/prod

[root@clienta ~]# cat prod

eyJmc2lkIjoiMmFlNmQwNWEtMjI5YS0xMWVjLTkyNWUtNTI1NDAwMDBmYTBjIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBSEV2bGlISnVwQ0JBQVZBZUk3Wnc3d215eEI3TytCZTR5V2c9PSIsIm1vbl9ob3N0IjoiW3YyOjE3Mi4yNS4yNTAuMTI6MzMwMC8wLHYxOjE3Mi4yNS4yNTAuMTI6Njc4OS8wXSJ9

[root@clienta ~]# rbd mirror pool info rbd

Mode: pool

Site Name: prod

Peer Sites: none

[root@clienta ~]#

[root@clienta ~]# rsync prod root@serverf:~

Warning: Permanently added 'serverf,172.25.250.15' (ECDSA) to the list of known hosts.

[root@clienta ~]# 

[root@serverf ~]# rbd mirror pool peer bootstrap import --site-name bup --direction rx-only rbd prod

    2022-08-14T11:21:20.321-0400 7f777f0392c0 -1 auth: unable to find a keyring on /etc/ceph/..keyring,/etc/ceph/.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory

    2022-08-14T11:21:20.323-0400 7f777f0392c0 -1 auth: unable to find a keyring on /etc/ceph/..keyring,/etc/ceph/.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory

    2022-08-14T11:21:20.323-0400 7f777f0392c0 -1 auth: unable to find a keyring on /etc/ceph/..keyring,/etc/ceph/.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory

[root@serverf ~]# rbd mirror pool info rbd

Mode: pool

Site Name: bup

Peer Sites: 

UUID: cd5db8bb-4f03-449b-ae29-c04ac38df226

Name: prod

Direction: rx-only

Client: client.rbd-mirror-peer

[root@clienta ~]# rbd create image3 --size 1024 --pool rbd --image-feature exclusive-lock,journaling

[root@serverf ~]# rbd ls

image1

image3

[root@serverf ~]# rbd map image3

rbd: sysfs write failed

RBD image feature set mismatch. You can disable features unsupported by the kernel with "rbd feature disable image3 journaling".

In some cases useful info is found in syslog - try "dmesg | tail".

rbd: map failed: (6) No such device or address

[root@serverf ~]# dmesg | tail

[ 2442.874097] Key type dns_resolver registered

[ 2442.963090] Key type ceph registered

[ 2442.974459] libceph: loaded (mon/osd proto 15/24)

[ 2442.996598] rbd: loaded (major 251)

[ 2443.015744] libceph: mon0 (1)172.25.250.15:6789 session established

[ 2443.018969] libceph: mon0 (1)172.25.250.15:6789 socket closed (con state OPEN)

[ 2443.019780] libceph: mon0 (1)172.25.250.15:6789 session lost, hunting for new mon

[ 2443.024317] libceph: mon0 (1)172.25.250.15:6789 session established

[ 2443.026864] libceph: client34233 fsid 0bf7c358-25e1-11ec-ae02-52540000fa0f

[ 2443.053943] rbd: image image3: image uses unsupported features: 0x40

[root@serverf ~]#

报错0x40，因为journaling特性的开启，反而不能用了

可以先创建池子，后给journaling特性

本来有个集群A，不再用了

开启池模式，单向同步，一次性导入到集群B？

所以这个功能好像不是那么常用

总结（单向同步）

主集群操作

ceph osd pool  create rbd

rbd pool  init rbd

rbd create image1 --size 1024 --pool rbd --image-feature exclusive-lock,journaling

rbd mirror pool enable rbd pool

rbd mirror pool peer bootstrap create --site-name prod rbd > /root/prod

rsycn prod root@serverf:~

备集群操作

ceph osd pool  create rbd

rbd pool init rbd

ceph orch apply rbd-mirror --placement=serverf.lab.example.com

rbd mirror pool peer bootstrap import --site-name bup --direction rx-only rbd /root/prod

rbd ls

官方文档真的很好用

https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/5/html/block_device_guide/mirroring-ceph-block-devices

scp可能出现安全问题，所以用rsync?

ceph作为一个开源的万金油分布式存储，势必不如其他专一方向的分布式存储，功能可能不如他们。但是原理是通用

巴特西

ceph 006 rbd高级特性 rbd快照镜像克隆 rbd缓存 rbd增量备份 rbd镜像单向同步

rbd高级特性

1.rbd快照

2.镜像克隆

3.rbd缓存

4.镜像的导入导出

理一遍思路（增量备份）

5.rbd镜像功能（单向同步）

实践

总结（单向同步）

最新文章

热门文章

巴特西

ceph 006 rbd高级特性 rbd快照 镜像克隆 rbd缓存 rbd增量备份 rbd镜像单向同步

rbd高级特性

1.rbd快照

2.镜像克隆

3.rbd缓存

4.镜像的导入导出

理一遍思路（增量备份）

5.rbd镜像功能（单向同步）

实践

总结（单向同步）

最新文章

热门文章

ceph 006 rbd高级特性 rbd快照镜像克隆 rbd缓存 rbd增量备份 rbd镜像单向同步