kubernetes使用ceph存储卷

kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: rbd
provisioner: ceph.com/rbd
parameters:
  monitors: xx.xx.xx.xx:6789
  pool: kube
  adminId: admin
  adminSecretNamespace: kube-system
  adminSecretName: ceph-admin-secret
  userId: kube
  userSecretNamespace: kube-system
  userSecretName: ceph-secret
  imageFormat: "2"
  imageFeatures: layering
allowVolumeExpansion: true

--feature-gates ExpandPersistentVolumes=true,PersistentVolumeClaimResize=true,ExpandInUsePersistentVolumes=true

$ kubectl edit pvc resize
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  annotations:
    volume.beta.kubernetes.io/storage-provisioner: ceph.com/rbd
  name: resize
spec:
  resources:
    requests:
      storage: 5Gi
  storageClassName: sata

// ExpandImage runs rbd resize command to resize the specified image.
func (util *RBDUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
	var output []byte
	var err error

	// Convert to MB that rbd defaults on.
	sz := int(volumehelpers.RoundUpToMiB(newSize))
	newVolSz := fmt.Sprintf("%d", sz)
	newSizeQuant := resource.MustParse(fmt.Sprintf("%dMi", sz))

	// Check the current size of rbd image, if equals to or greater that the new request size, do nothing.
	curSize, infoErr := util.rbdInfo(rbdExpander.rbdMounter)
	if infoErr != nil {
		return oldSize, fmt.Errorf("rbd info failed, error: %v", infoErr)
	}
	if curSize >= sz {
		return newSizeQuant, nil
	}

	// rbd resize.
	mon := util.kernelRBDMonitorsOpt(rbdExpander.rbdMounter.Mon)
	klog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key %s", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminId, rbdExpander.rbdMounter.adminSecret)
	output, err = rbdExpander.exec.Run("rbd",
		"resize", rbdExpander.rbdMounter.Image, "--size", newVolSz, "--pool", rbdExpander.rbdMounter.Pool, "--id", rbdExpander.rbdMounter.adminId, "-m", mon, "--key="+rbdExpander.rbdMounter.adminSecret)
	if err == nil {
		return newSizeQuant, nil
	}

	klog.Errorf("failed to resize rbd image: %v, command output: %s", err, string(output))
	return oldSize, err
}

$ cat test.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: test
spec:
  storageClassName: cephfs
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
---
apiVersion: v1
kind: Pod
metadata:
  name: test-pod
spec:
  volumes:
    - name: data
      persistentVolumeClaim:
        claimName: test
  containers:
    - name: nginx-server
      image: nginx
      ports:
        - containerPort: 80
          name: "http-server"
      volumeMounts:
        - mountPath: "/usr/share/nginx/html"
          name: data

# pvc的容量只有1G，竟然可以写入2G的数据到存储卷
$ kubectl exec -ti test-pod -- dd if=/dev/zero of=/usr/share/nginx/html/testfile.dat count=2018 bs=1048576

enableQuota                   = flag.Bool("enable-quota", false, "Enable PVC quota")

kubectl edit deployment cephfs-provisioner

spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: cephfs-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: cephfs-provisioner
    spec:
      containers:
      - args:
        ...
        - -enable-quota=true
        command:
        - /usr/local/bin/cephfs-provisioner

E0831 12:27:01.347130       1 cephfs-provisioner.go:158] failed to provision share "kubernetes-dynamic-pvc-a1f36dc2-cbea-11e9-97cf-6639535e4727" for "kubernetes-dynamic-user-a1f36ec8-cbea-11e9-97cf-6639535e4727", err: exit status 1, output: Traceback (most recent call last):
  File "/usr/local/bin/cephfs_provisioner", line 364, in <module>
    main()
  File "/usr/local/bin/cephfs_provisioner", line 358, in main
    print cephfs.create_share(share, user, size=size)
  File "/usr/local/bin/cephfs_provisioner", line 228, in create_share
    volume = self.volume_client.create_volume(volume_path, size=size, namespace_isolated=not self.ceph_namespace_isolation_disabled)
  File "/lib/python2.7/site-packages/ceph_volume_client.py", line 622, in create_volume
    self.fs.setxattr(path, 'ceph.quota.max_bytes', size.__str__(), 0)
  File "cephfs.pyx", line 988, in cephfs.LibCephFS.setxattr (/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/13.2.1/rpm/el7/BUILD/ceph-13.2.1/build/src/pybind/cephfs/pyrex/cephfs.c:10498)
cephfs.OperationNotSupported: [Errno 95] error in setxattr

$ mkdir test
$ ceph-fuse -c /etc/ceph/ceph.conf -m xx.xx.xx.xx:6789 -r /xxxx test
$ getfattr -n ceph.quota.max_bytes test
1073741824

	// check whether it belongs to fuse, if not, default to use kernel mount.
	if cephfsVolume.checkFuseMount() {
		klog.V(4).Info("CephFS fuse mount.")
		err = cephfsVolume.execFuseMount(dir)
		// cleanup no matter if fuse mount fail.
		keyringPath := cephfsVolume.GetKeyringPath()
		_, StatErr := os.Stat(keyringPath)
		if !os.IsNotExist(StatErr) {
			os.RemoveAll(keyringPath)
		}
		if err == nil {
			// cephfs fuse mount succeeded.
			return nil
		}
		// if cephfs fuse mount failed, fallback to kernel mount.
		klog.V(2).Infof("CephFS fuse mount failed: %v, fallback to kernel mount.", err)

	}
	klog.V(4).Info("CephFS kernel mount.")

	err = cephfsVolume.execMount(dir)
	if err != nil {
		// cleanup upon failure.
		mount.CleanupMountPoint(dir, cephfsVolume.mounter, false)
		return err
	}