rook使用教程，快速编排ceph

git clone https://github.com/rook/rook

cd cluster/examples/kubernetes/ceph

kubectl create -f operator.yaml

[root@dev-86-201 ~]# kubectl get pod -n rook-ceph-system

NAME                                  READY   STATUS    RESTARTS   AGE

rook-ceph-agent-5z6p7                 1/1     Running   0          88m

rook-ceph-agent-6rj7l                 1/1     Running   0          88m

rook-ceph-agent-8qfpj                 1/1     Running   0          88m

rook-ceph-agent-xbhzh                 1/1     Running   0          88m

rook-ceph-operator-67f4b8f67d-tsnf2   1/1     Running   0          88m

rook-discover-5wghx                   1/1     Running   0          88m

rook-discover-lhwvf                   1/1     Running   0          88m

rook-discover-nl5m2                   1/1     Running   0          88m

rook-discover-qmbx7                   1/1     Running   0          88m

kubectl create -f cluster.yaml

[root@dev-86-201 ~]# kubectl get pod -n rook-ceph

NAME                               READY   STATUS    RESTARTS   AGE

rook-ceph-mgr-a-8649f78d9b-jklbv   1/1     Running   0          64m

rook-ceph-mon-a-5d7fcfb6ff-2wq9l   1/1     Running   0          81m

rook-ceph-mon-b-7cfcd567d8-lkqff   1/1     Running   0          80m

rook-ceph-mon-d-65cd79df44-66rgz   1/1     Running   0          79m

rook-ceph-osd-0-56bd7545bd-5k9xk   1/1     Running   0          63m

rook-ceph-osd-1-77f56cd549-7rm4l   1/1     Running   0          63m

rook-ceph-osd-2-6cf58ddb6f-wkwp6   1/1     Running   0          63m

rook-ceph-osd-3-6f8b78c647-8xjzv   1/1     Running   0          63m

apiVersion: ceph.rook.io/v1

kind: CephCluster

metadata:

  name: rook-ceph

  namespace: rook-ceph

spec:

  cephVersion:

# For the latest ceph images, see https://hub.docker.com/r/ceph/ceph/tags

    image: ceph/ceph:v13.2.2-20181023

  dataDirHostPath: /var/lib/rook # 数据盘目录

  mon:

    count: 3

    allowMultiplePerNode: true

  dashboard:

    enabled: true

  storage:

    useAllNodes: true

    useAllDevices: false

    config:

      databaseSizeMB: "1024"

      journalSizeMB: "1024"

[root@dev-86-201 ~]# kubectl get svc -n rook-ceph

NAME                      TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)          AGE

rook-ceph-mgr             ClusterIP   10.98.183.33     <none>        9283/TCP         66m

rook-ceph-mgr-dashboard   NodePort    10.103.84.48     <none>        8443:31631/TCP   66m  # 把这个改成NodePort模式

rook-ceph-mon-a           ClusterIP   10.99.71.227     <none>        6790/TCP         83m

rook-ceph-mon-b           ClusterIP   10.110.245.119   <none>        6790/TCP         82m

rook-ceph-mon-d           ClusterIP   10.101.79.159    <none>        6790/TCP         81m

kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o yaml | grep "password:" | awk '{print $2}' | base64 --decode

apiVersion: ceph.rook.io/v1

kind: CephBlockPool

metadata:

  name: replicapool   # operator会监听并创建一个pool，执行完后界面上也能看到对应的pool

  namespace: rook-ceph

spec:

  failureDomain: host

  replicated:

    size: 3

---

apiVersion: storage.k8s.io/v1

kind: StorageClass

metadata:

   name: rook-ceph-block    # 这里创建一个storage class, 在pvc中指定这个storage class即可实现动态创建PV

provisioner: ceph.rook.io/block

parameters:

  blockPool: replicapool

# The value of "clusterNamespace" MUST be the same as the one in which your rook cluster exist

  clusterNamespace: rook-ceph

# Specify the filesystem type of the volume. If not specified, it will use `ext4`.

  fstype: xfs

# Optional, default reclaimPolicy is "Delete". Other options are: "Retain", "Recycle" as documented in https://kubernetes.io/docs/concepts/storage/storage-classes/

reclaimPolicy: Retain

kubectl create -f mysql.yaml

kubectl create -f wordpress.yaml

[root@dev-86-201 ~]# kubectl get pvc

NAME             STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE

mysql-pv-claim   Bound    pvc-a910f8c2-1ee9-11e9-84fc-becbfc415cde   20Gi       RWO            rook-ceph-block   144m

wp-pv-claim      Bound    pvc-af2dfbd4-1ee9-11e9-84fc-becbfc415cde   20Gi       RWO            rook-ceph-block   144m

[root@dev-86-201 ~]# kubectl get pv

NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                    STORAGECLASS      REASON   AGE

pvc-a910f8c2-1ee9-11e9-84fc-becbfc415cde   20Gi       RWO            Retain           Bound    default/mysql-pv-claim   rook-ceph-block            145m

pvc-af2dfbd4-1ee9-11e9-84fc-becbfc415cde   20Gi       RWO            Retain           Bound    default/wp-pv-claim      rook-ceph-block            145m

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: mysql-pv-claim

  labels:

    app: wordpress

spec:

  storageClassName: rook-ceph-block   # 指定storage class

  accessModes:

  - ReadWriteOnce

  resources:

    requests:

      storage: 20Gi  # 需要一个20G的盘

...

        volumeMounts:

        - name: mysql-persistent-storage

          mountPath: /var/lib/mysql

      volumes:

      - name: mysql-persistent-storage

        persistentVolumeClaim:

          claimName: mysql-pv-claim  # 指定上面定义的PVC

kubectl edit svc wordpress

[root@dev-86-201 kubernetes]# kubectl get svc

NAME              TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE

wordpress         NodePort    10.109.30.99   <none>        80:30130/TCP   148m