Kubernetes如何通过Devi

 Register接口描述如下：
 	pkg/kubelet/apis/deviceplugin/v1beta1/api.pb.go:440 	type RegistrationServer interface { 		Register(context.Context, *RegisterRequest) (*Empty, error) 	}   	pkg/kubelet/apis/deviceplugin/v1beta1/api.pb.go:87 	type RegisterRequest struct { 		// Version of the API the Device Plugin was built against 		Version string `protobuf:"bytes,1,opt,name=version,proto3"json:"version,omitempty"` 		// Name of the unix socket the device plugin is listening on// PATH = path.Join(DevicePluginPath, endpoint) 		Endpoint string `protobuf:"bytes,2,opt,name=endpoint,proto3"json:"endpoint,omitempty"` 		// Schedulable resource name. As of now it's expected to be a DNS Label 		ResourceName string `protobuf:"bytes,3,opt,name=resource_name,json=resourceName,proto3"json:"resource_name,omitempty"` 		// Options to be communicated with Device Manager 		Options *DevicePluginOptions `protobuf:"bytes,4,opt,name=options"json:"options,omitempty"` 	}

vendor/k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1/api.pb.go:71func(m *NvidiaDevicePlugin)GetDevicePluginOptions(context.Context, *pluginapi.Empty)(*pluginapi.DevicePluginOptions, error) { 		return &pluginapi.DevicePluginOptions{}, nil 	}  	github.com/NVIDIA/k8s-device-plugin/server.go:80type DevicePluginOptions struct { 		// Indicates if PreStartContainer call is required before each container start 		PreStartRequired bool`protobuf:"varint,1,opt,name=pre_start_required,json=preStartRequired,proto3" json:"pre_start_required,omitempty"` 	}

 /deviceplugin.Registration/Register
 	pkg/kubelet/apis/deviceplugin/v1alpha/api.pb.go:374 	var _Registration_serviceDesc = grpc.ServiceDesc{ 		ServiceName: "deviceplugin.Registration", 		HandlerType: (*RegistrationServer)(nil), 		Methods: []grpc.MethodDesc{ 			{ 				MethodName: "Register", 				Handler:    _Registration_Register_Handler, 			}, 		}, 		Streams:  []grpc.StreamDesc{}, 		Metadata: "api.proto", 	}
 

 /deviceplugin.DevicePlugin/ListAndWatch
 	pkg/kubelet/apis/deviceplugin/v1alpha/api.pb.go:505 	var _DevicePlugin_serviceDesc = grpc.ServiceDesc{ 		ServiceName: "deviceplugin.DevicePlugin", 		HandlerType: (*DevicePluginServer)(nil), 		Methods: []grpc.MethodDesc{ 			{ 				MethodName: "Allocate", 				Handler:    _DevicePlugin_Allocate_Handler, 			}, 		}, 		Streams: []grpc.StreamDesc{ 			{ 				StreamName:    "ListAndWatch", 				Handler:       _DevicePlugin_ListAndWatch_Handler, 				ServerStreams: true, 			}, 		}, 		Metadata: "api.proto", 	}
 

 /v1beta1.Registration/Register
 	/v1beta1.Registration/Register  	pkg/kubelet/apis/deviceplugin/v1beta1/api.pb.go:466 	var _Registration_serviceDesc = grpc.ServiceDesc{ 		ServiceName: "v1beta1.Registration", 		HandlerType: (*RegistrationServer)(nil), 		Methods: []grpc.MethodDesc{ 			{ 				MethodName: "Register", 				Handler:    _Registration_Register_Handler, 			}, 		}, 		Streams:  []grpc.StreamDesc{}, 		Metadata: "api.proto", 	}
 

 /v1beta1.DevicePlugin/GetDevicePluginOptions
 	pkg/kubelet/apis/deviceplugin/v1beta1/api.pb.go:665 	var _DevicePlugin_serviceDesc = grpc.ServiceDesc{ 		ServiceName: "v1beta1.DevicePlugin", 		HandlerType: (*DevicePluginServer)(nil), 		Methods: []grpc.MethodDesc{ 			{ 				MethodName: "GetDevicePluginOptions", 				Handler:    _DevicePlugin_GetDevicePluginOptions_Handler, 			}, 			{ 				MethodName: "Allocate", 				Handler:    _DevicePlugin_Allocate_Handler, 			}, 			{ 				MethodName: "PreStartContainer", 				Handler:    _DevicePlugin_PreStartContainer_Handler, 			}, 		}, 		Streams: []grpc.StreamDesc{ 			{ 				StreamName:    "ListAndWatch", 				Handler:       _DevicePlugin_ListAndWatch_Handler, 				ServerStreams: true, 			}, 		}, 		Metadata: "api.proto", 	}
 

 Allocate
 	pkg/kubelet/apis/deviceplugin/v1alpha/api.proto 	// DevicePlugin is the service advertised by Device Plugins 	service DevicePlugin { 		// ListAndWatch returns a stream of List of Devices 		// Whenever a Device state changes or a Device disappears, ListAndWatch 		// returns the new list 		rpc ListAndWatch(Empty) returns (stream ListAndWatchResponse) {}  		// Allocate is called during container creation so that the Device 		// Plugin can run device specific operations and instruct Kubelet 		// of the steps to make the Device available in the container 		rpc Allocate(AllocateRequest) returns (AllocateResponse) {} 	}
 

 PreStartContainer
 	pkg/kubelet/apis/deviceplugin/v1beta1/api.proto 	// DevicePlugin is the service advertised by Device Plugins 	service DevicePlugin { 		// GetDevicePluginOptions returns options to be communicated with Device 	        // Manager 		rpc GetDevicePluginOptions(Empty) returns (DevicePluginOptions) {}  		// ListAndWatch returns a stream of List of Devices 		// Whenever a Device state change or a Device disapears, ListAndWatch 		// returns the new list 		rpc ListAndWatch(Empty) returns (stream ListAndWatchResponse) {}  		// Allocate is called during container creation so that the Device 		// Plugin can run device specific operations and instruct Kubelet 		// of the steps to make the Device available in the container 		rpc Allocate(AllocateRequest) returns (AllocateResponse) {}      // PreStartContainer is called, if indicated by Device Plugin during registeration phase,     // before each container start. Device plugin can run device specific operations     // such as reseting the device before making devices available to the container 		rpc PreStartContainer(PreStartContainerRequest) returns (PreStartContainerResponse) {} 	}
 

 ListAndWatch：监控对应Devices的状态变更或者Disappear事件，返回ListAndWatchResponse给kubelet, ListAndWatchResponse就是Device列表。
 	type ListAndWatchResponse struct { 		Devices []*Device `protobuf:"bytes,1,rep,name=devices" json:"devices,omitempty"` 	}  	type Device struct { 		// A unique ID assigned by the device plugin used 		// to identify devices during the communication 		// Max length of this field is 63 characters 		ID string `protobuf:"bytes,1,opt,name=ID,json=iD,proto3" json:"ID,omitempty"` 		// Health of the device, can be healthy or unhealthy, see constants.go 		Health string `protobuf:"bytes,2,opt,name=health,proto3" json:"health,omitempty"` 	}
 

 Allocate allows Device Plugin to run device specific operations on the Devices requested
 	type AllocateRequest struct { 		ContainerRequests []*ContainerAllocateRequest `protobuf:"bytes,1,rep,name=container_requests,json=containerRequests" json:"container_requests,omitempty"` 	}  	type ContainerAllocateRequest struct { 		DevicesIDs []string `protobuf:"bytes,1,rep,name=devicesIDs" json:"devicesIDs,omitempty"` 	}  	// AllocateResponse includes the artifacts that needs to be injected into 	// a container for accessing 'deviceIDs' that were mentioned as part of 	// 'AllocateRequest'. 	// Failure Handling: 	// if Kubelet sends an allocation request for dev1 and dev2. 	// Allocation on dev1 succeeds but allocation on dev2 fails. 	// The Device plugin should send a ListAndWatch update and fail the 	// Allocation request 	type AllocateResponse struct { 		ContainerResponses []*ContainerAllocateResponse `protobuf:"bytes,1,rep,name=container_responses,json=containerResponses" json:"container_responses,omitempty"` 	}  	type ContainerAllocateResponse struct { 		// List of environment variable to be set in the container to access one of more devices. 		Envs map[string]string `protobuf:"bytes,1,rep,name=envs" json:"envs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` 		// Mounts for the container. 		Mounts []*Mount `protobuf:"bytes,2,rep,name=mounts" json:"mounts,omitempty"` 		// Devices for the container. 		Devices []*DeviceSpec `protobuf:"bytes,3,rep,name=devices" json:"devices,omitempty"` 		// Container annotations to pass to the container runtime 		Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` 	}  	// DeviceSpec specifies a host device to mount into a container. 	type DeviceSpec struct { 		// Path of the device within the container. 		ContainerPath string `protobuf:"bytes,1,opt,name=container_path,json=containerPath,proto3" json:"container_path,omitempty"` 		// Path of the device on the host. 		HostPath string `protobuf:"bytes,2,opt,name=host_path,json=hostPath,proto3" json:"host_path,omitempty"` 		// Cgroups permissions of the device, candidates are one or more of 		// * r - allows container to read from the specified device. 		// * w - allows container to write to the specified device. 		// * m - allows container to create device files that do not yet exist. 		Permissions string `protobuf:"bytes,3,opt,name=permissions,proto3" json:"permissions,omitempty"` 	}
 

 PreStartContainer allows Device Plugin to run device specific operations on the Devices requested.
 	type PreStartContainerRequest struct { 		DevicesIDs []string `protobuf:"bytes,1,rep,name=devicesIDs" json:"devicesIDs,omitempty"` 	}  	// PreStartContainerResponse will be send by plugin in response to PreStartContainerRequest 	type PreStartContainerResponse struct { 	}
 

 GetDevicePluginOptions: 目前只有PreStartRequired这一个field。
 type DevicePluginOptions struct { 	// Indicates if PreStartContainer call is required before each container start 	PreStartRequired bool `protobuf:"varint,1,opt,name=pre_start_required,json=preStartRequired,proto3" json:"pre_start_required,omitempty"` }
 

github.com/NVIDIA/k8s-device-plugin/main.go:15

func main() {
	...
	
	log.Println("Starting FS watcher.")
	watcher, err := newFSWatcher(pluginapi.DevicePluginPath)
	
    ...

	restart := true
	var devicePlugin *NvidiaDevicePlugin

L:
	for {
		if restart {
			if devicePlugin != nil {
				devicePlugin.Stop()
			}

			devicePlugin = NewNvidiaDevicePlugin()
			if err := devicePlugin.Serve(); err != nil {
				log.Println("Could not contact Kubelet, retrying. Did you enable the device plugin feature gate?")
				log.Printf("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites")
				log.Printf("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start")
			} else {
				restart = false
			}
		}

		select {
		case event := <-watcher.Events:
			if event.Name == pluginapi.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create {
				log.Printf("inotify: %s created, restarting.", pluginapi.KubeletSocket)
				restart = true
			}

		case err := <-watcher.Errors:
			log.Printf("inotify: %s", err)

		case s := <-sigs:
			switch s {
			case syscall.SIGHUP:
				log.Println("Received SIGHUP, restarting.")
				restart = true
			default:
				log.Printf("Received signal \"%v\", shutting down.", s)
				devicePlugin.Stop()
				break L
			}
		}
	}
}	

select {
    case event := <-watcher.Events:
    	if event.Name == pluginapi.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create {
    		log.Printf("inotify: %s created, restarting.", pluginapi.KubeletSocket)
    		restart = true
    	}
    	
    	// 增加对nvidia.sock的删除事件监控
    	if event.Name == serverSocket && event.Op&fsnotify.Delete == fsnotify.Delete {
    		log.Printf("inotify: %s deleted, restarting.", serverSocket)
    		restart = true
    	}
    	
    	...
}

# Label your nodes with the accelerator type they have.
kubectl label nodes <node-with-k80> accelerator=nvidia-tesla-k80
kubectl label nodes <node-with-p100> accelerator=nvidia-tesla-p100

apiVersion: v1
kind: Pod
metadata:
  name: cuda-vector-add
spec:
  restartPolicy: OnFailure
  containers:
    - name: cuda-vector-add
      # https://github.com/kubernetes/kubernetes/blob/v1.7.11/test/images/nvidia-cuda/Dockerfile
      image: "k8s.gcr.io/cuda-vector-add:v0.1"
      resources:
        limits:
          nvidia.com/gpu: 1
  nodeSelector:
    accelerator: nvidia-tesla-p100 # or nvidia-tesla-k80 etc.

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: nvidia-device-plugin-daemonset
spec:
  template:
    metadata:
      labels:
        name: nvidia-device-plugin-ds
    spec:
      containers:
      - image: nvidia/k8s-device-plugin:1.8
        name: nvidia-device-plugin-ctr
        securityContext:
          privileged: true
        volumeMounts:
          - name: device-plugin
            mountPath: /var/lib/kubelet/device-plugins
      volumes:
        - name: device-plugin
          hostPath:
            path: /var/lib/kubelet/device-plugins

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: nvidia-device-plugin-daemonset
  namespace: kube-system
spec:
  template:
    metadata:
      # Mark this pod as a critical add-on; when enabled, the critical add-on scheduler
      # reserves resources for critical add-on pods so that they can be rescheduled after
      # a failure.  This annotation works in tandem with the toleration below.
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ""
      labels:
        name: nvidia-device-plugin-ds
    spec:
      tolerations:
      # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode.
      # This, along with the annotation above marks this pod as a critical add-on.
      - key: CriticalAddonsOnly
        operator: Exists
      containers:
      - image: nvidia/k8s-device-plugin:1.10
        name: nvidia-device-plugin-ctr
        securityContext:
          privileged: true
        volumeMounts:
          - name: device-plugin
            mountPath: /var/lib/kubelet/device-plugins
      volumes:
        - name: device-plugin
          hostPath:
            path: /var/lib/kubelet/device-plugins