Pod疑难杂症（3）：Device or resource busy

unlinkat /var/run/netns/cni-49ddd103-d374-1f86-7324-13abaeb9c910: device or resource busy

$ cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: sleepping
spec:
  replicas: 1
  selector:
    matchLabels:
      app: sleepping
  template:
    metadata:
      labels:
        app: sleepping
    spec:
      containers:
        - name: test
          image: busybox
          args: ["sleep", "1h"]
EOF

$ cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rootfsmount
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rootfsmount
  template:
    metadata:
      labels:
        app: rootfsmount
    spec:
      containers:
      - name: rootfsmount
        image: busybox
        args: ["sleep", "1h"]
        volumeMounts:
        - mountPath: /rootfs
          name: host-rootfs
      volumes:
      - hostPath:
          path: /
          type: ""
        name: host-rootfs
EOF

nsenter --net=/var/run/netns/cni-49ddd103-d374-1f86-7324-13abaeb9c910

$ strace -f -e trace=mount -p <pid>
...
[pid 15532] mount("/", "/run/containerd/io.containerd.runtime.v2.task/k8s.io/5b498caf152857cf1c797761e1f52d64c2ce7d4602b72304da7e154ed31043c8/rootfs/rootfs", 0xc0000f7500, MS_BIND|MS_REC, NULL) = 0
[pid 15532] mount("", "/run/containerd/io.containerd.runtime.v2.task/k8s.io/5b498caf152857cf1c797761e1f52d64c2ce7d4602b72304da7e154ed31043c8/rootfs/rootfs", 0xc0000f7506, MS_REC|MS_PRIVATE, NULL) = 0
...

$ crictl inspect <container-id>
...
{
  "destination": "/rootfs",
  "type": "bind",
  "source": "/",
  "options": [
    "rbind",
    "rprivate",
    "rw"
  ]
},
...

$ cat /proc/self/mountinfo
...
# 对应pod的volumeMount设置，将宿主机上的主目录/ 挂载到了容器内的/rootfs目录下
651 633 253:1 / /rootfs rw,relatime - ext4 /dev/vda1 rw,data=ordered
695 677 0:3 / /rootfs/run/netns/cni-49ddd103-d374-1f86-7324-13abaeb9c910 rw,nosuid,nodev,noexec,relatime - proc proc rw
...

apiVersion: apps/v1
kind: Deployment
metadata:
  name: rootfsmount
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rootfsmount
  template:
    metadata:
      labels:
        app: rootfsmount
    spec:
      containers:
      - name: rootfsmount
        image: busybox
        args: ["sleep", "1h"]
        volumeMounts:
        - mountPath: /rootfs
          name: host-rootfs
          mountPropagation: HostToContainer # 这里显示声明mountPropagation为HostToContainer 或者 Bidirectional
      volumes:
      - hostPath:
          path: /
          type: ""
        name: host-rootfs

$ echo 1 > /proc/sys/fs/may_detach_mounts

// This is used to allow removal of mountpoints that may be mounted in other
// namespaces on RHEL based kernels starting from RHEL 7.4.
// Without this setting, removals on these RHEL based kernels may fail with
// "device or resource busy".
// This setting is not available in upstream kernels as it is not configurable,
// but has been in the upstream kernels since 3.15.
func setMayDetachMounts() error {
    f, err := os.OpenFile("/proc/sys/fs/may_detach_mounts", os.O_WRONLY, 0)
    if err != nil {
        if os.IsNotExist(err) {
            return nil
        }
        return errors.Wrap(err, "error opening may_detach_mounts kernel config file")
    }
    defer f.Close()
 
    _, err = f.WriteString("1")
    if os.IsPermission(err) {
        // Setting may_detach_mounts does not work in an
        // unprivileged container. Ignore the error, but log
        // it if we appear not to be in that situation.
        if !rsystem.RunningInUserNS() {
            logrus.Debugf("Permission denied writing %q to /proc/sys/fs/may_detach_mounts", "1")
        }
        return nil
    }
    return err
}

// validateBindDaemonRoot ensures that if a given mountpoint's source is within
// the daemon root path, that the propagation is setup to prevent a container
// from holding private refereneces to a mount within the daemon root, which
// can cause issues when the daemon attempts to remove the mountpoint.
func (daemon *Daemon) validateBindDaemonRoot(m mount.Mount) (bool, error) {
    if m.Type != mount.TypeBind {
        return false, nil
    }
 
    // check if the source is within the daemon root, or if the daemon root is within the source
    if !strings.HasPrefix(m.Source, daemon.root) && !strings.HasPrefix(daemon.root, m.Source) {
        return false, nil
    }
 
    if m.BindOptions == nil {
        return true, nil
    }
 
    switch m.BindOptions.Propagation {
    case mount.PropagationRSlave, mount.PropagationRShared, "":
        return m.BindOptions.Propagation == "", nil
    default:
    }
 
    return false, errdefs.InvalidParameter(errors.Errorf(`invalid mount config: must use either propagation mode "rslave" or "rshared" when mount source is within the daemon root, daemon root: %q, bind mount source: %q, propagation: %q`, daemon.root, m.Source, m.BindOptions.Propagation))
}