K8s上用Ceph-rbd存储

ceph osd pool create pool_1 90

rbd create pool_1/lun1 --size 10G

使用ceph-deploy --overwrite-conf admin部署的keyring权限太大，可以自己创建一个keyring client.rdb给块设备客户端node用


# ceph auth get-or-create client.rbd mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=pool_1'  > ceph.client.rbd.keyring

k8s节点需要安装ceph
yum install ceph-common
echo 'rbd' > /etc/modules-load.d/rbd.conf
modprobe rbd
lsmod | grep rbd
rbd                    83640  0 
libceph               306625  1 rbd

配置文件秘钥传到k8s上
[root@ceph ceph]# scp ceph.client.rbd.keyring 192.168.6.102:/etc/ceph/
root@192.168.6.102's password: 
ceph.client.rdb.keyring                                                                                                       100%   63     8.5KB/s   00:00    
[root@ceph ceph]# scp ceph.conf 192.168.6.102:/etc/ceph/                
root@192.168.6.102's password: 
ceph.conf                                                                                                                       100%  310    25.1KB/s   00:00    
[root@ceph ceph]#

[root@node1 ceph]# ceph -s --name client.rdb
  cluster:
    id:     cbc04385-1cdf-4512-a3f5-a5b3e8686a05
    health: HEALTH_WARN
            application not enabled on 1 pool(s)
 
  services:
    mon: 1 daemons, quorum ceph
    mgr: ceph(active)
    osd: 1 osds: 1 up, 1 in
 
  data:
    pools:   1 pools, 90 pgs
    objects: 5 objects, 709B
    usage:   1.00GiB used, 19.0GiB / 20.0GiB avail
    pgs:     90 active+clean
    
警告解决办法：
ceph health detail
ceph osd pool application enable pool_1 rbd

# rbd map pool_1/lun1 --name client.rbd
rbd: sysfs write failed
RBD image feature set mismatch. Try disabling features unsupported by the kernel with "rbd feature disable".
In some cases useful info is found in syslog - try "dmesg | tail".
rbd: map failed: (6) No such device or address

解决办法：
在ceph节点上
rbd feature disable pool_1/lun1 exclusive-lock, object-map, fast-diff, deep-flatten

rbd map pool_1/lun1 --name client.rbd
mkfs.ext4 /dev/rbd0

对ceph.client.admin.keyring 的内容进行base64编码
[root@node1 ceph]# ceph auth get-key client.rbd | base64
QVFCTktERmRzeXpKQUJBQVVvVGVvWVYyamxhRi8zNU1hZ2R2dFE9PQ==
根据上面的输出，创建secret ceph-client-rbd
[root@node1 ceph]# cat ceph-secret.yml 
apiVersion: v1
kind: Secret
metadata:
  name: ceph-client-rbd
type: "kubernetes.io/rbd"  
data:
  key: QVFCTktERmRzeXpKQUJBQVVvVGVvWVYyamxhRi8zNU1hZ2R2dFE9PQ==
  
  
kubectl apply -f ceph-secret.yml
创建pv，注意： 这里是user：rbd 而不是user: client.rbd

[root@node1 ceph]# cat pv.yml 
kind: PersistentVolume
apiVersion: v1
metadata:
  name: ceph-pool1-lun1
spec:
  storageClassName: manual
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce          
  rbd:
    fsType: ext4
    image: lun1
    monitors:
      - '192.168.6.101:6789'
    pool: pool_1
    readOnly: false
    secretRef:
      name: ceph-client-rbd
      namespace: default
    user: rbd
    
[root@node1 ceph]# kubectl apply -f pv.yml 
persistentvolume/ceph-pool1-lun1 created
[root@node1 ceph]# kubectl get pv
NAME              CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM   STORAGECLASS   REASON   AGE
ceph-pool1-lun1   10Gi       RWO            Retain           Available           manual                  4s
创建pvc

[root@node1 ceph]# cat pvc.yml 
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: pvc1
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
      
 
[root@node1 ceph]# kubectl apply -f pvc.yml 
persistentvolumeclaim/pvc1 created
[root@node1 ceph]# kubectl get pvc
NAME   STATUS   VOLUME            CAPACITY   ACCESS MODES   STORAGECLASS   AGE
pvc1   Bound    ceph-pool1-lun1   10Gi       RWO            manual         7s