1.Containerd容器运行时初识与尝试

# - 当我们执行 pstree 命令时可以看到如下的进程关系containerd-shim是独立于containerd进程运行的。
pstree
systemd─┬─VGAuthService
        ├─containerd───15*[{containerd}]
        ├─containerd-shim─┬─sh
        │                 └─13*[{containerd-shim}]
        ├─2*[containerd-shim─┬─sh]
        │                    └─12*[{containerd-shim}]]

# - OS
$ lsb_release -a
  # Description:    Ubuntu 20.04.2 LTS focal
$ uname -a
  # Linux containerd 5.4.0-78-generic #87-Ubuntu SMP Fri Jun 18 16:29:09 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux

# - Container : 1.4.6-1
$ apt-cache madison containerd.io | cut -d '|' -f 2 |  sed 's/^[ \t]*//g'
  # 1.4.6-1
  # 1.4.4-1
  # 1.4.3-2
  # 1.4.3-1
  # 1.3.9-1
  # 1.3.7-1
  # 1.2.13-2

# 安装和配置先决条件：
# 模块加载
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
sudo modprobe overlay
sudo modprobe br_netfilter

# Setup required sysctl params, these persist across reboots.
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

# Apply sysctl params without reboot
sudo sysctl --system

# - 卸载原有Docker 以及 containerd
sudo apt-get remove docker docker-engine docker.io containerd runc
# - 更新apt程序包索引并安装程序包，以允许apt通过HTTPS使用存储库
sudo apt-get update
sudo apt-get install \
  apt-transport-https \
  ca-certificates \
  curl \
  gnupg \
  lsb-release

# - 添加Docker的官方GPG密钥：
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

# - 使用以下命令设置稳定存储库。要添加nightly或test存储库，请在下面的命令中的单词stable后面添加单词nightly或test（或两者）。
echo \
  "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

# - 更新apt包索引，安装最新版本的containerd或进入下一步安装特定版本：
sudo apt-get update
# 安装前可查看containerd.io可用的版本: apt-cache madison containerd.io && apt install containerd.io=1.4.6-1
apt install containerd.io

# - 查看安装Containerd的版本
$ ctr --version
  # ctr containerd.io 1.4.6

# - 初始化配置
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml
# - 替换默认的 sandbox_image = "k8s.gcr.io/pause:3.2" 镜像因为GFW的原因导致无法访问我们可以替换为阿里云镜像源。
sed -ir 's#sandbox_image = \S\w+.*$#sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0"#g' /etc/containerd/config.toml
sed -ir "s#https://registry-1.docker.io#https://xlx9erfu.mirror.aliyuncs.com#g"  /etc/containerd/config.toml

# - 应用配置并重新运行containerd服务并查看服务状态
systemctl daemon-reload && systemctl restart containerd
systemctl status containerd.service
  # ● containerd.service - containerd container runtime
  # ......
  #     CGroup: /system.slice/containerd.service
  #             └─3097 /usr/bin/containerd

$ cat /etc/containerd/config.toml
version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  tcp_address = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[ttrpc]
  address = ""
  uid = 0
  gid = 0

[debug]
  address = ""
  uid = 0
  gid = 0
  level = ""

[metrics]
  address = ""
  grpc_histogram = false

[cgroup]
  path = ""

[timeouts]
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[plugins]
  [plugins."io.containerd.gc.v1.scheduler"]
    pause_threshold = 0.02
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = "0s"
    startup_delay = "100ms"
  [plugins."io.containerd.grpc.v1.cri"]
    disable_tcp_service = true
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    stream_idle_timeout = "4h0m0s"
    enable_selinux = false
    selinux_category_range = 1024
    sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0"
    stats_collect_period = 10
    systemd_cgroup = false
    enable_tls_streaming = false
    max_container_log_line_size = 16384
    disable_cgroup = false
    disable_apparmor = false
    restrict_oom_score_adj = false
    max_concurrent_downloads = 3
    disable_proc_mount = false
    unset_seccomp_profile = ""
    tolerate_missing_hugetlb_controller = true
    disable_hugetlb_controller = true
    ignore_image_defined_volumes = false
    [plugins."io.containerd.grpc.v1.cri".containerd]
      snapshotter = "overlayfs"
      default_runtime_name = "runc"
      no_pivot = false
      disable_snapshot_annotations = true
      discard_unpacked_layers = false
      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          runtime_type = "io.containerd.runc.v2"
          runtime_engine = ""
          runtime_root = ""
          privileged_without_host_devices = false
          base_runtime_spec = ""
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      max_conf_num = 1
      conf_template = ""
    [plugins."io.containerd.grpc.v1.cri".registry]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://registry-1.docker.io"]
    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = ""
    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""
  [plugins."io.containerd.internal.v1.opt"]
    path = "/opt/containerd"
  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"
  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"
  [plugins."io.containerd.monitor.v1.cgroups"]
    no_prometheus = false
  [plugins."io.containerd.runtime.v1.linux"]
    shim = "containerd-shim"
    runtime = "runc"
    runtime_root = ""
    no_shim = false
    shim_debug = false
  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["linux/amd64"]
  [plugins."io.containerd.service.v1.diff-service"]
    default = ["walking"]
  [plugins."io.containerd.snapshotter.v1.devmapper"]
    root_path = ""
    pool_name = ""
    base_image_size = ""
    async_remove = false

$ ctr version
Client:
  Version:  1.4.6
  Revision: d71fcd7d8303cbf684402823e425e9dd2e99285d
  Go version: go1.13.15

Server:
  Version:  1.4.6
  Revision: d71fcd7d8303cbf684402823e425e9dd2e99285d
  UUID: 71a28bbb-6ed6-408d-a873-e394d48b35d8

# - 1.将 hello-world:latest 镜像拉取到默认的名称空间中,将busybox:latest镜像拉取到k8s.io名称空间中。
ctr image pull docker.io/library/hello-world:latest
ctr -n k8s.io image pull docker.io/library/busybox:latest
ctr -n k8s.io image pull registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0

# - 2.分别查看拉取的镜像
ctr image ls && ctr -n k8s.io image ls
  # REF                                  TYPE                                                      DIGEST                                                                  SIZE    PLATFORMS                                                                                                             LABELS
  # docker.io/library/hello-world:latest application/vnd.docker.distribution.manifest.list.v2+json sha256:9f6ad537c5132bcce57f7a0a20e317228d382c3cd61edae14650eec68b2b345c 6.5 KiB linux/386,linux/amd64,linux/arm/v5,linux/arm/v7,linux/arm64/v8,linux/mips64le,linux/ppc64le,linux/s390x,windows/amd64 


# - 3.运行hello-world:latest和分步运行busybox镜像
ctr run docker.io/library/hello-world:latest hello-world
# Hello from Docker!
# ......
# For more examples and ideas, visit:
#  https://docs.docker.com/get-started/

# 创建一个container(此时还未运行)
ctr -n k8s.io container create docker.io/library/busybox:latest busybox
# 创建一个task并后台运行
ctr -n k8s.io task start -d busybox
# 一步到位: ctr -n k8s.io run -d docker.io/library/busybox:latest busybox


# - 4.查看你创建的容器与Task
ctr container ls && ctr -n k8s.io container ls
  # CONTAINER      IMAGE                                   RUNTIME
  # hello-world    docker.io/library/hello-world:latest    io.containerd.runc.v2
  # CONTAINER    IMAGE                               RUNTIME
  # busybox      docker.io/library/busybox:latest    io.containerd.runc.v2
# 查看该容器在宿主机的PID
ctr -n k8s.io task ls
  # TASK       PID      STATUS
  # busybox    25856    RUNNING
ps ajxf|grep "containerd-shim-runc\|25856"|grep -v grep
      1   25828   25828    3097 ?             -1 Sl       0   0:00 /usr/bin/containerd-shim-runc-v2 -namespace k8s.io -id busybox -address /run/containerd/containerd.sock
  25828   25856   25856   25856 ?             -1 Ss       0   0:00  \_ sh


# - 5.进入到容器内部执行shell命令
ctr -n k8s.io t exec --exec-id $RANDOM -t busybox sh
  # / # uname -a
  # Linux containerd 3.10.0-1062.el7.x86_64 #1 SMP Wed Aug 7 18:08:02 UTC 2019 x86_64 GNU/Linux
  # ---  此处你会发现只有lo网络卡后续我们可以通过CNI插件的形式添加网卡实现外部网络通信 --- 
  # / # ip a
  # 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000
  #     inet 127.0.0.1/8 scope host lo
  #  ...

# - 6.删除创建的Task与容器
ctr -n k8s.io t kill -s SIGKILL busybox  # 发送SIGKILL信号量杀死该容器
ctr -n k8s.io task rm busybox
ctr -n k8s.io container rm busybox
ctr container rm hello-world

# - 1.从containernetworking/plugins的release页面下载最新版本并解压到/usr/local/cni-plugins:
wget https://github.com/containernetworking/plugins/releases/download/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
mkdir -vp /usr/local/cni-plugins && tar xvf cni-plugins-linux-amd64-v0.9.1.tgz -C /usr/local/cni-plugins

# - 2.从containernetworking/cni的release页面下载最新版本并解压到/usr/local/cni-{version}:
wget https://github.com/containernetworking/cni/archive/refs/tags/v0.8.1.tar.gz
tar -zxvf v0.8.1.tar.gz -C /usr/local/

# - 使用bridge插件创建一个网卡依赖于一下面的配置文件。
mkdir -p /etc/cni/net.d
cat >/etc/cni/net.d/10-mynet.conf <<EOF
{
  "cniVersion": "0.2.0",
  "name": "mynet",
  "type": "bridge",
  "bridge": "cni0",
  "isGateway": true,
  "ipMasq": true,
  "ipam": {
    "type": "host-local",
    "subnet": "10.22.0.0/16",
    "routes": [
      { "dst": "0.0.0.0/0" }
    ]
  }
}
EOF
 
cat >/etc/cni/net.d/99-loopback.conf <<EOF
{
  "cniVersion": "0.2.0",
  "name": "lo",
  "type": "loopback"
}
EOF

# - 注意json字符串解析需要依赖jq命令所以我们需要安装(已安装可以跳过)
apt install jq -y

# - 利用CNI网络插件进行激活cni0网卡
/usr/local/cni-0.8.1/scripts# CNI\_PATH=/usr/local/cni-plugins ./priv-net-run.sh echo "Hello World! CNI Plugins."
Hello World! CNI Plugins.

# - 宿主机执行ip a命令即可看到一个cni0的网卡
/usr/local/cni-0.8.1/scripts# ip addr show cni0
3: cni0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default qlen 1000
    inet 10.22.0.1/16 brd 10.22.255.255 scope global cni0
       valid\_lft forever preferred\_lft forever

# - 将busybox容器实现网络通信
ctr -n k8s.io run -d docker.io/library/busybox:latest busybox
ctr -n k8s.io task ls
  # TASK       PID      STATUS
  # busybox    43908    RUNNING
pid=$(ctr -n k8s.io t ls|grep busybox|awk '{print $2}')
netnspath=/proc/$pid/ns/net
CNI_PATH=/usr/local/cni-plugins /usr/local/cni-0.8.1/scripts/exec-plugins.sh add $pid $netnspath

# - 随后进入busybox容器我们将会发现其新增了一张网卡并可以实现外部网络访问：
ctr -n k8s.io task exec --exec-id $RANDOM -t busybox  sh -
  # / # ip addr
  # 3: eth0@if5: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
  #     inet 10.22.0.3/16 brd 10.22.255.255 scope global eth0
  # / # ping 223.6.6.6   # 外网没有问题。
  PING 223.6.6.6 (223.6.6.6): 56 data bytes
  64 bytes from 223.6.6.6: seq=0 ttl=113 time=7.902 ms


# - 在创建一个busybox-1实现互通。
ctr -n k8s.io run -d docker.io/library/busybox:latest busybox-1
pid=$(ctr -n k8s.io t ls|grep busybox-1|awk '{print $2}')
netnspath=/proc/$pid/ns/net
CNI_PATH=/usr/local/cni-plugins /usr/local/cni-0.8.1/scripts/exec-plugins.sh add $pid $netnspath

# - 查看你创建容器的pid进程相关信息
ps ajxf|egrep "containerd-shim-runc|43908|48850"|grep -v grep
  #     1   43883   43883   39042 ?             -1 Sl       0   0:00 /usr/bin/containerd-shim-runc-v2 -namespace k8s.io -id busybox -address /run/containerd/containerd.sock
  # 43883   43908   43908   43908 ?             -1 Ss       0   0:00  \_ sh
  #     1   48825   48825   39042 ?             -1 Sl       0   0:00 /usr/bin/containerd-shim-runc-v2 -namespace k8s.io -id busybox-1 -address /run/containerd/containerd.sock
  # 48825   48850   48850   48850 ?             -1 Ss       0   0:00  \_ sh

# - 查看创建的busybox-1相关信息
ctr -n k8s.io task exec --exec-id $RANDOM -t busybox-1 sh -
  # 3: eth0@if6: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
  # ......
  #     inet 10.22.0.4/16 brd 10.22.255.255 scope global eth0

ctr -n k8s.io c create docker.io/library/busybox:latest busybox-2 --mount type=bind,src=/tmp,dst=/host,options=rbind:rw

ctr -n k8s.io t start -d busybox-2 sh

ctr -n k8s.io t exec -t --exec-id $RANDOM busybox-2 sh
  # / # echo "WeiyiGeek" > /host/name
  # / # 
root@containerd:~# cat /tmp/name
WeiyiGeek

[root@docker scripts]# docker run --rm -it -d busybox sh
687c80243ee15e0a2171027260e249400feeeee2607f88d1f029cc270402cdd1
[root@docker scripts]# docker run --rm -it -d --pid="container:687c80243ee15e0a2171027260e249400feeeee2607f88d1f029cc270402cdd1" busybox cat
fa2.....d2
[root@docker scripts]# docker ps -a
CONTAINER ID        IMAGE               COMMAND             CREATED             STATUS              PORTS               NAMES
fa2c09bd9c04        busybox             "cat"               2 seconds ago       Up 1 second                             pedantic_goodall
687c80243ee1        busybox             "sh"                22 seconds ago      Up 21 seconds                           hopeful_franklin
[root@docker scripts]# docker exec -it 687c80243ee1 sh
/ # ps aux
PID   USER     TIME  COMMAND
    1 root      0:00 sh
    8 root      0:00 cat
   15 root      0:00 sh

$ ctr -n k8s.io t ls
  # TASK         PID      STATUS
  # busybox      43908    RUNNING 
  # busybox-1    48850    RUNNING  # 这里的48850即为已有task运行时的pid号
  # busybox-2    50314    RUNNING

$ ctr -n k8s.io c create --with-ns "pid:/proc/48850/ns/pid" v4ehxdz8.mirror.aliyuncs.com/library/python:3.6-slim python # 将 容器中运行 的进程共享注入到 48850 pid之中

$ ctr -n k8s.io t start -d python python  # 启动了一个python的命令

$ ctr -n k8s.io t exec -t --exec-id $RANDOM busybox-1 sh # 至此可以在busybox-1容器中看见刚执行的python命令
/ # ps aux
PID   USER     TIME  COMMAND
    1 root      0:00 sh
   34 root      0:00 python3

echo \
  "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
apt-update
# - 查看可用版本
apt-cache madison docker-ce 
# - 安装指定版本 (20.10.X 版本支持 Docker Scan 特性)
sudo apt-get install docker-ce=5:20.10.7~3-0~ubuntu-focal docker-ce-cli=5:20.10.7~3-0~ubuntu-focal

sudo gpasswd -a ${USER} docker
systemctl enable docker && systemctl status docker
  # ● docker.service - Docker Application Container Engine
  #     Loaded: loaded (/lib/systemd/system/docker.service; enabled; vendor preset: enabled)
  # TriggeredBy: ● docker.socket
  #   Main PID: 129606 (dockerd)
  #       Tasks: 12
  #     Memory: 45.0M
  #     CGroup: /system.slice/docker.service
  #             └─129606 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --debug

# 或者查看 Dockerd 服务端执行的命令
ps aux|grep docker
  # root      129606  0.0  4.4 1021016 89792 ?       Ssl  12:47   0:00 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --debug

# 新增docker0网卡信息
# 7: docker0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
#   inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
#       valid_lft forever preferred_lft forever

$ docker run --rm -it busybox sh
  # / # ip addr
  # 8: eth0@if9: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
  #     inet 172.17.0.2/16 brd 172.17.255.255 scope global eth0   # 关键点
  #       valid_lft forever preferred_lft forever
  # / # cat /etc/hosts
  # 127.0.0.1       localhost
  # 172.17.0.2      51dcdf68d3b8

# 采用 ctr 在 moby 名称空间下查看docker创建的容器以及Task PID
$ ctr -n moby c ls
  # CONTAINER                                                           IMAGE    RUNTIME
  # 51dc......4b    -        io.containerd.runc.v2
$ ctr -n moby t ls
  # TASK                                                                PID       STATUS
  # 51d....4b    131926    RUNNING

# 验证获取到的 Task PID 下其 net 相关信息
$ grep "172.17.0.2" /proc/131926/net/fib_trie
  |-- 172.17.0.2
  |-- 172.17.0.2

# 采用 ctr 进入 Docker 创建的容器
$ ctr -n moby t exec -t --exec-id $RANDOM 
 51d....64b sh
  # / # ip addr | grep "172.17.0.2"
  #   inet 172.17.0.2/16 brd 172.17.255.255 scope global eth0
  # / # hostname
  #   51dcdf68d3b8

#USAGE:
  ctr [global options] command [command options] [arguments...]

# GLOBAL OPTIONS:
  --debug                      enable debug output in logs
  --address value, -a value    address for containerds GRPC server (default: "/run/containerd/containerd.sock") [$CONTAINERD_ADDRESS]
  --timeout value              total timeout for ctr commands (default: 0s)
  --connect-timeout value      timeout for connecting to containerd (default: 0s)
  --namespace value, -n value  namespace to use with commands (default: "default") [$CONTAINERD_NAMESPACE]
  --version, -v                print the version

# COMMANDS:
  plugins, plugin            provides information about containerd plugins
  version                    print the client and server versions
  containers, c, container   manage containers
  content                    manage content
  events, event              display containerd events
  images, image, i           manage images
  leases                     manage leases
  namespaces, namespace, ns  manage namespaces
  pprof                      provide golang pprof outputs for containerd
  run                        run a container
  snapshots, snapshot        manage snapshots
  tasks, t, task             manage tasks
  install                    install a new package
  oci                        OCI tools
  shim                       interact with a shim directly (直接与shim交互)
  help, h                    Shows a list of commands or help for one command

# 子命令 #
# Images COMMANDS:
  check       check that an image has all content available locally(找到发生故障的容器后，可以使用以下工具检查其日志)
  export      export images
  import      import images
  list, ls    list images known to containerd
  mount       mount an image to a target path
  unmount     unmount the image from the target
  pull        pull an image from a remote
  push        push an image to a remote
  remove, rm  remove one or more images by reference
  tag         tag an image
  label       set and clear labels for an image

# Container COMMANDS:
  create           create container
  delete, del, rm  delete one or more existing containers
  info             get info about a container
  list, ls         list containers
  label            set and clear labels for a container
  checkpoint       checkpoint a container (检查点容器)
  restore          restore a container from checkpoint(从检查点还原容器)

# run OPTIONS:
  --rm                                    remove the container after running
  --null-io                               send all IO to /dev/null (将容器内标准输出重定向到/dev/null)
  --log-uri value                         log uri
  --detach, -d                            detach from the task after it has started execution(在任务开始执行后从中分离,如没有选项则会等待用户输入并定向到容器内)
  --fifo-dir value                        directory used for storing IO FIFOs
  --cgroup value                          cgroup path (To disable use of cgroup, set to "" explicitly)
  --platform value                        run image for specific platform
  --runc-binary value                     specify runc-compatible binary
  --runc-systemd-cgroup                   start runc with systemd cgroup manager
  --uidmap container-uid:host-uid:length  run inside a user namespace with the specified UID mapping range; specified with the format container-uid:host-uid:length
  --gidmap container-gid:host-gid:length  run inside a user namespace with the specified GID mapping range; specified with the format container-gid:host-gid:length
  --remap-labels                          provide the user namespace ID remapping to the snapshotter via label options; requires snapshotter support
  --cpus value                            set the CFS cpu qouta (default: 0)
  --snapshotter value                     snapshotter name. Empty value stands for the default value. [$CONTAINERD_SNAPSHOTTER]
  --config value, -c value                path to the runtime-specific spec config file
  --cwd value                             specify the working directory of the process
  --env value                             specify additional container environment variables (i.e. FOO=bar)
  --env-file value                        specify additional container environment variables in a file(i.e. FOO=bar, one per line)
  --label value                           specify additional labels (i.e. foo=bar)
  --mount value                           specify additional container mount (ex: type=bind,src=/tmp,dst=/host,options=rbind:ro)
  --net-host                              enable host networking for the container
  --privileged                            run privileged container
  --read-only                             set the containers filesystem as readonly
  --runtime value                         runtime name (default: "io.containerd.runc.v2")
  --tty, -t                               allocate a TTY for the container
  --with-ns value                         specify existing Linux namespaces to join at container runtime (format '<nstype>:<path>')
  --pid-file value                        file path to write the task's pid
  --gpus value                            add gpus to the container (default: 0)
  --allow-new-privs                       turn off OCI spec's NoNewPrivileges feature flag
  --memory-limit value                    memory limit (in bytes) for the container (default: 0)
  --device value                          add a device to a container
  --seccomp                               enable the default seccomp profile
  --rootfs                                use custom rootfs that is not managed by containerd snapshotter
  --no-pivot                              disable use of pivot-root (linux only)
  --cpu-quota value                       Limit CPU CFS quota (default: -1)
  --cpu-period value                      Limit CPU CFS period (default: 0)

# Tasks COMMANDS:
  attach           attach to the IO of a running container
  checkpoint       checkpoint a container
  delete, rm       delete one or more tasks
  exec             execute additional processes in an existing container
  list, ls         list tasks
  kill             signal a container (default: SIGTERM)
  pause            pause an existing container
  ps               list processes for container
  resume           resume a paused container
  start            start a container that has been created
  metrics, metric  get a single data point of metrics for a task with the built-in Linux runtime

# 1) 名称空间查看创建或删除
ctr namespace create devops
ctr namespace ls -q
  # devops
  # k8s.io
ctr namespace remove devops


# 2) 镜像查看及其操作
ctr -n k8s.io images ls | grep "busybox"
  # docker.io/library/busybox:1.33.1 application/vnd.docker.distribution.manifest.list.v2+json sha256:93049.....d 752.6 KiB linux/386,linux/amd64,linux/arm/v5,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/mips64le,linux/ppc64le,linux/s390x io.cri-containerd.image=managed
ctr -n k8s.io images ls -q | grep "busybox"   # docker.io/library/busybox:1.33.1
ctr -n k8s.io i rm k8s.gcr.io/pause:3.2       # 删除镜像
ctr -n k8s.io i pull -k k8s.gcr.io/pause:3.2  # 拉取镜像 
ctr -n k8s.io i push -k k8s.gcr.io/pause:3.2  # 推送镜像
ctr -n k8s.io i export pause.tar k8s.gcr.io/pause:3.2 # 导出镜像
ctr -n k8s.io i import pause.tar              # 导入镜像    
ctr -n k8s.io images check  | grep "busybox"  # 镜像检查
  # docker.io/library/busybox:1.33.1  application/vnd.docker.distribution.manifest.list.v2+json sha256:93049.......0d complete (2/2) 750.1 KiB/750.1 KiB true
ctr -n k8s.io i tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2 k8s.gcr.io/pause:3.2         # 镜像标记 Tag 更改
ctr -n k8s.io i tag --force registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2 k8s.gcr.io/pause:3.2 # 若新镜像 reference 已存在需要先删除新 reference 或者如下方式强制替换
ctr -n k8s.io i label docker.io/library/busybox:1.33.1 name=weiyigeek-test  # 设置镜像标签
  # io.cri-containerd.image=managed,name=weiyigeek-test
ctr -n k8s.io i label docker.io/library/busybox:1.33.1 name=""              # 清除镜像标签
  # io.cri-containerd.image=managed


# 3) 容器查看及其操作
ctr -n k8s.io container ls
  # CONTAINER                                                           IMAGE                                                            RUNTIME
  # 3d840c5cf157c322c1c0839d0166d80550fe49ca53544e1d589bd44f422b1f81    registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2    io.containerd.runc.v2
ctr -n k8s.io container create docker.io/library/busybox:latest busybox                       # 创建容器
ctr -n k8s.io container info 39d36e......cb5 # 查看创建的容器详情（注意这个必须是完整的容器ID）
ctr -n k8s.io container rm 39d36e......cb5   # 删除容器
ctr -n k8s.io container label busybox name=weiyigeek-test     # 设置容器标签
  # io.containerd.image.config.stop-signal=SIGTERM,name=weiyigeek-test


# 4) 容器运行参数演示
ctr -n k8s.io run --null-io --net-host -d
–env PASSWORD=$drone_password
–mount type=bind,src=/etc,dst=/host-etc,options=rbind:rw
–mount type=bind,src=/root/.kube,dst=/root/.kube,options=rbind:rw
-log-uri file:///var/log/xx.log
WeiyiGeek:Test sysreport bash /sysreport/run.sh


# 5) 容器Task查看以及相关操作
ctr -n k8s.io tasks ls
  # TASK                                                                PID      STATUS
  # 4cdf.....66c81    12708    RUNNING
ctr -n k8s.io tasks kill -a -s 9 {id}  # 停止容器内的Task
ctr -n k8s.io task metric 37d0f.....b98 # - 使用内置的Linux运行时获取任务的单个度量数据点
  # ID                                                                  TIMESTAMP
  # 37d0ff9d8d.....46151fffd99b98    2021-07-11 03:33:18.421996083 +0000 UTC
  # METRIC                   VALUE                                                                                                            
  # memory.usage_in_bytes    65269760            
  # memory.limit_in_byte     9223372036854771712 
  # memory.stat.cache        4157440 
  # cpuacct.usage            44769155237                                                                                              
  # cpuacct.usage_percpu     [22375115908 22394039329 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  # pids.current             43          
  # pids.limit               4582         

# crictl by default connects on Unix to:
unix:///var/run/dockershim.sock or
unix:///run/containerd/containerd.sock or
unix:///run/crio/crio.sock

# or on Windows to:
npipe:////./pipe/dockershim or
npipe:////./pipe/containerd or
npipe:////./pipe/crio

# For other runtimes, use:
frakti: unix:///var/run/frakti.sock

$ cat /etc/crictl.yaml
runtime-endpoint: /run/containerd/containerd.sock
image-endpoint: "/run/containerd/containerd.sock"
timeout: 0
debug: false
pull-image-on-create: false

# 1.通过在配置文件中设置端点并通过 --config 参数指定配置文件
crictl --config=/etc/crictl-demo.yaml

# 2.通过设置参数 --runtime-endpoint 和 --image-endpoint
crictl --runtime-endpoint="/run/containerd/containerd.sock" --image-endpoint="/run/containerd/containerd.sock"

# USAGE:
  crictl [global options] command [command options] [arguments...]

# COMMANDS:
  attach        Attach to a running container
  create        Create a new container (创建容器这里需要先创建 sandbox, 获取sandbox容器的id后再用此id创建业务容器)
  exec          Run a command in a running container
  version       Display runtime version information
  images        List images
  inspect       Display the status of one or more containers (显示一个或多个容器的状态)
  inspecti      Return the status of one or more images (返回一个或者多个镜像的状态)
  inspectp      Display the status of one or more pods (返回一个或者多个Pod的状态)
  imagefsinfo: Return image filesystem info
  logs          Fetch the logs of a container
  port-forward  Forward local port to a pod
  ps            List containers (列出在 k8s.io 命名空间下的业务容器)
  pull          Pull an image from a registry
  runp          Run a new pod
  rm            Remove one or more containers
  rmi           Remove one or more images
  rmp           Remove one or more pods
  pods          List pods (列出在 k8s.io 命名空间下的 sandbox 容器, 在k8s里通常是pause容器)
  start         Start one or more created containers
  info          Display information of the container runtime
  stop          Stop one or more running containers
  stopp         Stop one or more running pods
  update        Update one or more running containers
  config        Get and set crictl options
  stats         List container(s) resource usage statistics
  completion    Output bash shell completion code
  help, h       Shows a list of commands or help for one command

# GLOBAL OPTIONS:
  --config value, -c value            Location of the client config file (default: "/etc/crictl.yaml") [$CRI_CONFIG_FILE]
  --debug, -D                         Enable debug mode
  --image-endpoint value, -i value    Endpoint of CRI image manager service [$IMAGE_SERVICE_ENDPOINT]
  --runtime-endpoint value, -r value  Endpoint of CRI container runtime service (default: "unix:///var/run/dockershim.sock") [$CONTAINER_RUNTIME_ENDPOINT]
  --timeout value, -t value           Timeout of connecting to the server (default: 10s)

# 0) 镜像查看及其操作
crictl images --digests
  # IMAGE                                                                         TAG                 DIGEST              IMAGE ID            SIZE
  # docker.io/calico/cni                                                          v3.18.4             278fd825d089e       021ecb3cb5348       44.7MB
crictl images nginx   # - 打印某个镜像
crictl images -q      # - 只打印镜像ID
  # sha256:8c811b....6a472a
crictl images -o [json|yaml|table]   # - 镜像详细信息
crictl pull busybox                  # - 拉取 busybox 镜像
crictl push weiyigeek.top/busybox    # - 推送 busybox 镜像
crictl rmi docker.io/library/busybox # - 容器删除
  # Deleted: docker.io/library/busybox:1.33.1
  # Deleted: docker.io/library/busybox:latest


# 1) 容器查看及其操作
crictl ps -a  # - 打印在 k8s.io 命名空间下的业务容器
crictl --runtime-endpoint /run/containerd/containerd.sock ps -a | grep kube | grep -v pause # - 查看过滤指定的容器相关信息
  # CONTAINER ID        IMAGE               CREATED             STATE               NAME                      ATTEMPT             POD ID
  # 8db74c2bf7595       ebc659140e762       3 days ago          Running             calico-node               0                   5af3d484b32c0
crictl create f84dd......e8c9a54f[创建的 Pod 的 ID] container-config.json pod-config.json # - 创建容器
crictl start 3e025dd.....57d60  # - 启动容器
crictl stop 3e025dd....57d60   # - 删除容器
crictl exec -i -t 1f73f2d81bf98 ls # - 容器上执行命令


# 2) Pod查看与操作
$ crictl pods
  # POD ID              CREATED             STATE               NAME                                   NAMESPACE           ATTEMPT
  # 5af3d484b32c0       4 days ago          Ready               calico-node-zvst7                      kube-system         0
$ crictl pods --name nginx-65899c769f-wv2gp  # - 打印某个固定pod.
$ crictl pods --label run=nginx              # - 根据标签筛选pod.
$ crictl runp pod-config.json                # - 使用 crictl runp 命令应用 JSON 文件并运行沙盒。
$ crictl inspectp --output table $POD_ID     # - 用crictl查看pod的信息
$ crictl stopp $POD_ID  # - 停止Pod
$ crictl rmp  $POD_ID   # - 删除Pod


# 3) 日志查看
crictl --runtime-endpoint /run/containerd/containerd.sock logs --tail 50 8db74c2bf7595


# 4) 容器占用资源状态查看
crictl stats
  # CONTAINER           CPU %               MEM                 DISK                INODES
  # 8db74c2bf7595       0.69                53.37MB             274.4kB             81

$ cat pod-config.json
{
    "metadata": {
        "name": "nginx-sandbox",
        "namespace": "default",
        "attempt": 1,
        "uid": "hdishd83djaidwnduwk28bcsb"
    },
    "log_directory": "/tmp",
    "linux": {
    }
}

$ crictl runp pod-config.json
f84dd361f8......e8c9a54f

$ crictl pods
POD ID              CREATED             STATE               NAME                NAMESPACE           ATTEMPT
f84dd361f8dc5       17 seconds ago      Ready               busybox-sandbox     default             1

$ cat pod-config.json
{
    "metadata": {
        "name": "nginx-runsc-sandbox",
        "namespace": "default",
        "attempt": 1,
        "uid": "hdishd83djaidwnduwk28bcsb"
    },
    "log_directory": "/tmp",
    "linux": {
    }
}

$ crictl runp --runtime=runsc pod-config.json
c112976cb6caa43a967293e2c62a2e0d9d8191d5109afef230f403411147548c

$ crictl inspectp c112976cb6caa43a967293e2c62a2e0d9d8191d5109afef230f403411147548c
...
    "runtime": {
      "runtimeType": "io.containerd.runtime.v1.linux",
      "runtimeEngine": "/usr/local/sbin/runsc",
      "runtimeRoot": "/run/containerd/runsc"
    },
...

crictl pull busybox:1.33.1
 # Image is up to date for sha256:69593048aa3acfee0f75f20b77acb549de2472063053f6730c4091b53f2dfb02

$ cat pod-config.json
{
    "metadata": {
        "name": "nginx-sandbox",
        "namespace": "default",
        "attempt": 1,
        "uid": "hdishd83djaidwnduwk28bcsb"
    },
    "log_directory": "/tmp",
    "linux": {
    }
}

$ cat container-config.json
{
  "metadata": {
      "name": "busybox"
  },
  "image":{
      "image": "busybox:1.33.1"
  },
  "command": [
      "top"
  ],
  "log_path":"busybox.log",
  "linux": {
  }
}

$ crictl create f84dd361f8......e8c9a54f container-config.json pod-config.json
3e025dd50a72d956c4f14881fbb5b1080c9275674e95fb67f965f6478a957d60

$ crictl ps -a
CONTAINER ID        IMAGE               CREATED             STATE               NAME                ATTEMPT
3e025dd50a72d       busybox             32 seconds ago      Created             busybox             0

$ crictl start 3e025dd50a72d956c4f14881fbb5b1080c9275674e95fb67f965f6478a957d60
3e025dd50a72d956c4f14881fbb5b1080c9275674e95fb67f965f6478a957d60

$ crictl ps
CONTAINER ID        IMAGE               CREATED              STATE               NAME                ATTEMPT
3e025dd50a72d       busybox             About a minute ago   Running             busybox             0

crictl exec -i -t 3e025dd50a72d956c4f14881fbb5b1080c9275674e95fb67f965f6478a957d60 ls
bin   dev   etc   home  proc  root  sys   tmp   usr   var