使用sniff 轻松抓取kubernetes pod的数据报文

(
  set -x; cd "$(mktemp -d)" &&
  curl -fsSLO "https://github.com/kubernetes-sigs/krew/releases/latest/download/krew.{tar.gz,yaml}" &&
  tar zxvf krew.tar.gz &&
  KREW=./krew-"$(uname | tr '[:upper:]' '[:lower:]')_amd64" &&
  "$KREW" install --manifest=krew.yaml --archive=krew.tar.gz &&
  "$KREW" update
)

然后，记得加到环境变量里面
export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"

$ kubectl sniff --help
Perform network sniffing on a container running in a kubernetes cluster.
Usage:
  sniff pod [-n namespace] [-c container] [-f filter] [-o output-file] [-l local-tcpdump-path] [-r remote-tcpdump-path] [flags]
Examples:
kubectl sniff hello-minikube-7c77b68cff-qbvsd -c hello-minikube
Flags:
  -c, --container string             container (optional)  # 可选参数，建议显式声明下，不加的话默认就是抓取的第一个container
  -f, --filter string                tcpdump filter (optional)   # 可选参数，等效于tcpdump里面的filter过滤
  -h, --help                         help for sniff
      --image string                 the privileged container image (optional) (default "docker")
  -i, --interface string             pod interface to packet capture (optional) (default "any")   #抓取的网卡接口，建议保持默认的全部接口
  -l, --local-tcpdump-path string    local static tcpdump binary path (optional)
  -n, --namespace string             namespace (optional) (default "default")   # 待抓取的pod所在namespace
  -o, --output-file string           output file path, tcpdump output will be redirect to this file instead of wireshark (optional) ('-' stdout)   # 抓包数据输出的路径或文件
  -p, --privileged                   if specified, ksniff will deploy another pod that have privileges to attach target pod network namespace  # 是否要使用特权模式的ksniff pod进行抓包
  -r, --remote-tcpdump-path string   remote static tcpdump binary path (optional) (default "/tmp/static-tcpdump")
  -v, --verbose                      if specified, ksniff output will include debug information (optional)

$ kubectl sniff -n default nginx-test-69b668b75c-w8pxr -c nginx  -o ./sb-test.cap      # 导出为文件
$ kubectl sniff -n default nginx-test-69b668b75c-w8pxr -c nginx   -o  - | tshark -r -   # tshark需要安装wireshark

# 说明： 启动上面的命令后，会自动在对应的namespace下起一个ksniff-xxxx 的pod，它不会自动销毁，需要我们在抓包完后人工去delete掉这个pod

kubectl sniff -n lens-metrics prometheus-0 -c prometheus  -p -o ./sb-123.cap
kubectl sniff -n lens-metrics prometheus-0 -c prometheus  -p   -o  - | tshark -r -

$ kubectl sniff -n lens-metrics node-exporter-7sgzd -c node-exporter  -p -o - | tshark -r -
INFO[0000] sniffing method: privileged pod              
INFO[0000] using tcpdump path at: '/root/.krew/store/sniff/v1.4.1/static-tcpdump' 
INFO[0000] selected container: 'node-exporter'          

INFO[0000] sniffing on pod: 'node-exporter-7sgzd' [namespace: 'lens-metrics', container: 'node-exporter', filter: '', interface: 'any'] 
INFO[0000] creating privileged pod on node: '192.168.2.164' 
INFO[0000] pod created: &Pod{ObjectMeta:{ksniff-qnk8j ksniff- lens-metrics /api/v1/namespaces/lens-metrics/pods/ksniff-qnk8j 0b297002-e5ca-4657-b069-9937d89050b3 5473644 0 2020-04-26 22:03:11 +0800 CST <nil> <nil> map[] map[] [] []  []},Spec:PodSpec{Volumes:[]Volume{Volume{Name:docker-sock,VolumeSource:VolumeSource{HostPath:&HostPathVolumeSource{Path:/var/run/docker.sock,Type:*File,},EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:nil,StorageOS:nil,CSI:nil,},},Volume{Name:default-token-vtln8,VolumeSource:VolumeSource{HostPath:nil,EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:&SecretVolumeSource{SecretName:default-token-vtln8,Items:[]KeyToPath{},DefaultMode:*420,Optional:nil,},NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:nil,StorageOS:nil,CSI:nil,},},},Containers:[]Container{Container{Name:ksniff-privileged,Image:docker,Command:[sh -c sleep 10000000],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:docker-sock,ReadOnly:true,MountPath:/var/run/docker.sock,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:default-token-vtln8,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Always,SecurityContext:&SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,},},RestartPolicy:Never,TerminationGracePeriodSeconds:*30,ActiveDeadlineSeconds:nil,DNSPolicy:ClusterFirst,NodeSelector:map[string]string{},ServiceAccountName:default,DeprecatedServiceAccount:default,NodeName:192.168.2.164,HostNetwork:false,HostPID:false,HostIPC:false,SecurityContext:&PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,},ImagePullSecrets:[]LocalObjectReference{},Hostname:,Subdomain:,Affinity:nil,SchedulerName:default-scheduler,InitContainers:[]Container{},AutomountServiceAccountToken:nil,Tolerations:[]Toleration{Toleration{Key:node.kubernetes.io/not-ready,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},Toleration{Key:node.kubernetes.io/unreachable,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},},HostAliases:[]HostAlias{},PriorityClassName:,Priority:*0,DNSConfig:nil,ShareProcessNamespace:nil,ReadinessGates:[]PodReadinessGate{},RuntimeClassName:nil,EnableServiceLinks:*true,PreemptionPolicy:nil,Overhead:ResourceList{},TopologySpreadConstraints:[]TopologySpreadConstraint{},EphemeralContainers:[]EphemeralContainer{},},Status:PodStatus{Phase:Pending,Conditions:[]PodCondition{},Message:,Reason:,HostIP:,PodIP:,StartTime:<nil>,ContainerStatuses:[]ContainerStatus{},QOSClass:BestEffort,InitContainerStatuses:[]ContainerStatus{},NominatedNodeName:,PodIPs:[]PodIP{},EphemeralContainerStatuses:[]ContainerStatus{},},} 
INFO[0000] waiting for pod successful startup           
Running as user "root" and group "root". This could be dangerous.
INFO[0032] pod: 'ksniff-qnk8j' created successfully on node: '192.168.2.164' 
INFO[0032] output file option specified, storing output in: '-' 
INFO[0032] starting remote sniffing using privileged pod 
INFO[0032] executing command: '[docker run --rm --name=ksniff-container-sACYVKlC --net=container:88b5286a914c69c498e5d9c4cfe88763e8d89cb3314abfd0fa737b2079366705 corfr/tcpdump -i any -U -w - ]' on container: 'ksniff-privileged', pod: 'ksniff-qnk8j', namespace: 'lens-metrics' 
  1          0 172.20.0.202 -> 172.20.2.71  HTTP 313 GET /metrics HTTP/1.1 
  2          0  172.20.2.71 -> 172.20.0.202 TCP 68 hp-pdl-datastr > 43610 [ACK] Seq=1 Ack=246 Win=1401 Len=0 TSval=785475095 TSecr=785326265
  3          0  172.20.2.71 -> 172.20.0.202 TCP 4164 [TCP segment of a reassembled PDU]
  4          0  172.20.2.71 -> 172.20.0.202 TCP 4164 [TCP segment of a reassembled PDU]
  5          0 172.20.0.202 -> 172.20.2.71  TCP 68 43610 > hp-pdl-datastr [ACK] Seq=246 Ack=4097 Win=1381 Len=0 TSval=785326580 TSecr=785475410
  6          0  172.20.2.71 -> 172.20.0.202 TCP 4164 [TCP segment of a reassembled PDU]
  7          0 172.20.0.202 -> 172.20.2.71  TCP 68 43610 > hp-pdl-datastr [ACK] Seq=246 Ack=8193 Win=1358 Len=0 TSval=785326581 TSecr=785475410
  8          0  172.20.2.71 -> 172.20.0.202 HTTP 1873 HTTP/1.1 200 OK  (text/plain)
  9          0 172.20.0.202 -> 172.20.2.71  TCP 68 43610 > hp-pdl-datastr [ACK] Seq=246 Ack=12289 Win=1336 Len=0 TSval=785326581 TSecr=785475410
 10          0 172.20.0.202 -> 172.20.2.71  TCP 68 43610 > hp-pdl-datastr [ACK] Seq=246 Ack=14094 Win=1322 Len=0 TSval=785326581 TSecr=785475410
 11         15 172.20.0.202 -> 172.20.2.71  HTTP 313 GET /metrics HTTP/1.1 
 12         15  172.20.2.71 -> 172.20.0.202 TCP 68 hp-pdl-datastr > 43610 [ACK] Seq=14094 Ack=491 Win=1401 Len=0 TSval=785490097 TSecr=785341266
 13         15  172.20.2.71 -> 172.20.0.202 TCP 4164 [TCP segment of a reassembled PDU]
 14         15  172.20.2.71 -> 172.20.0.202 TCP 4164 [TCP segment of a reassembled PDU]
 15         15 172.20.0.202 -> 172.20.2.71  TCP 68 43610 > hp-pdl-datastr [ACK] Seq=491 Ack=18190 Win=1381 Len=0 TSval=785341685 TSecr=785490514
$ kubectl get pods   -n lens-metrics -o wide
NAME                                 READY   STATUS              RESTARTS   AGE     IP             NODE            NOMINATED NODE   READINESS GATES
ksniff-qnk8j                         0/1     ContainerCreating   0          27s     <none>         192.168.2.164   <none>           <none>              # 这是刚启动ksniff时候查看到的状态
kube-state-metrics-57d759779-4hdr4   1/1     Running             1          9d      172.20.1.179   192.168.2.162   <none>           <none>
node-exporter-7sgzd                  1/1     Running             6          18d     172.20.2.71    192.168.2.164   <none>           <none>
node-exporter-c6tmq                  1/1     Running             2          18d     172.20.1.175   192.168.2.162   <none>           <none>
node-exporter-dw4k2                  1/1     Running             3          18d     172.20.0.183   192.168.2.161   <none>           <none>
prometheus-0                         1/1     Running             0          5d5h    172.20.0.202   192.168.2.161   <none>           <none>