openstack tap as a service深入研究

惠伟

发布于 2021-02-24 11:21:54

4510

发布于 2021-02-24 11:21:54

惠伟：openstack tap as a service浅尝辄止zhuanlan.zhihu.com

前面写的这篇流表没有分析清楚，上次简单试用了一下，没有好好分析，遇到的问题也没有细想，又重新实践了一次，整理一下记录下来。

环境

所有vm都在同一个网段，vm1和vm2通信，图上绿色线所示，vm3也和vm2通信，图上蓝色线所示，vm4是流量镜像的目的，vm1进出的流量镜像到vm4的路径上图红色所示，vm3进出的流量镜像到vm4的路径上图黄色所示。

流表

重点提示

br-int上的vlan本地有效，跨vxlan tunnel之间的报文一般不需要携带vlan tag。vm1和vm2之间通信的报文src ip和mac分别是vm1的，dst ip和mac分别是vm2的，镜像的报文到了ostack1上的br-tun，不知道镜像的目的port在哪里，所以第一个报文采用了flood，打上vlan tag 1，ostack6上的br-tun收到flood过来的vlan tag是1报文后发一分给br-tap，然后再copy一份打上vlan tag 2从源tunnel发回到ostack1，ostack1上的br-tun收到vlan tag是2的报文后，学习一条流表，这条流表让vm1和vm2之间镜像的流量从ostack1上的tunnel发往ostack6上的tunnel，以后vm1和vm2之间镜像的流量就不在所有tunnel上flood了，携带的vlan tag是0。镜像流量的在br-int上vlan tag是taas_id，在tunnel上tunnel id也是taas_id。

ostack1

[root@ostack1 huiwei]#  ovs-ofctl dump-flows br-int
NXST_FLOW reply (xid=0x4):
 #vm1到vm2的报文正常转发，copy打上tag 3904发给br-tap
 cookie=0x0, duration=67469.740s, table=0, n_packets=5433, n_bytes=805554, idle_age=0, hard_age=65534, priority=20,in_port=21 actions=NORMAL,mod_vlan_vid:3904,output:18
 #vm2回到vm1的报文上一条一样的操作
 cookie=0x0, duration=67469.501s, table=0, n_packets=3185, n_bytes=299754, idle_age=0, hard_age=65534, priority=20,dl_dst=fa:16:3d:bf:69:87 actions=NORMAL,mod_vlan_vid:3904,output:18


[root@ostack1 huiwei]#  ovs-ofctl dump-flows br-tap
NXST_FLOW reply (xid=0x4):
 cookie=0x0, duration=66653.290s, table=0, n_packets=1206878592, n_bytes=417421937098, idle_age=0, hard_age=65534, priority=1,in_port=1 actions=resubmit(,1)
 cookie=0x0, duration=66653.051s, table=0, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=1,in_port=2 actions=resubmit(,2)
 cookie=0x0, duration=66652.807s, table=0, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=0 actions=drop
 cookie=0x0, duration=66652.563s, table=1, n_packets=1206866840, n_bytes=417417872406, idle_age=0, hard_age=65534, priority=0 actions=output:2
 cookie=0x0, duration=66652.315s, table=2, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=0 actions=drop


[root@ostack1 huiwei]#  ovs-ofctl dump-flows br-tun
NXST_FLOW reply (xid=0x4):
 #vm3和vm2的报文
 cookie=0x82d633d26ce3ac51, duration=69076.716s, table=0, n_packets=1258811058, n_bytes=435448250659, idle_age=0, hard_age=65534, priority=1,in_port=1 actions=resubmit(,2)
 #ostack6上反射回来的vlan tag是2的报文
 cookie=0x82d633d26ce3ac51, duration=68578.721s, table=0, n_packets=18305, n_bytes=1027855, idle_age=0, hard_age=65534, priority=1,in_port=2 actions=resubmit(,4)
 #从br-tap来的报文转到table 30
 cookie=0x0, duration=67349.352s, table=0, n_packets=1219221035, n_bytes=421690599704, idle_age=0, hard_age=65534, priority=1,in_port=3 actions=resubmit(,30)
 #ostack6上反射回来的vlan tag是2的报文，把vlan tag 2暂存在reg0中，转table 35
 cookie=0x0, duration=67187.480s, table=4, n_packets=810, n_bytes=173948, idle_age=4, hard_age=65534, priority=1,tun_id=0xf40 actions=move:NXM_OF_VLAN_TCI[0..11]->NXM_NX_REG0[0..11],move:NXM_NX_TUN_ID[0..11]->NXM_OF_VLAN_TCI[0..11],resubmit(,35)
 #这条是第一个报文学习到的，携带的vlan tag是0
 cookie=0x0, duration=4.965s, table=30, n_packets=8, n_bytes=784, hard_timeout=60, idle_age=0, priority=1,vlan_tci=0x0f40/0x0fff actions=load:0xf40->NXM_NX_TUN_ID[0..11],load:0->NXM_OF_VLAN_TCI[0..11],output:2
 #这一个报文转table 31
 cookie=0x0, duration=67349.108s, table=30, n_packets=1219209151, n_bytes=421688104819, idle_age=0, hard_age=65534, priority=0 actions=resubmit(,31)
 #打vlan tag 1然后flood，我的环境只有两个节点，所以只有一个output
 cookie=0x0, duration=67348.371s, table=31, n_packets=1219194996, n_bytes=421683208776, idle_age=0, hard_age=58331, priority=0 actions=move:NXM_OF_VLAN_TCI[0..11]->NXM_NX_TUN_ID[0..11],mod_vlan_vid:1,output:2

 cookie=0x0, duration=67348.111s, table=35, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=2,reg0=0 actions=resubmit(,36)
 cookie=0x0, duration=67347.870s, table=35, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=1,reg0=0x1 actions=resubmit(,36)
 #reg0是2
 cookie=0x0, duration=67347.626s, table=35, n_packets=810, n_bytes=173948, idle_age=4, hard_age=65534, priority=1,reg0=0x2 actions=resubmit(,37)
 cookie=0x0, duration=67347.383s, table=36, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=0 actions=drop
 #转table 39
 cookie=0x0, duration=67187.001s, table=37, n_packets=810, n_bytes=173948, idle_age=4, hard_age=65534, priority=1,tun_id=0xf40 actions=resubmit(,39)
 cookie=0x0, duration=67347.140s, table=37, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=0 actions=drop
 cookie=0x0, duration=67346.894s, table=38, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=2,reg0=0 actions=output:3
 cookie=0x0, duration=67346.650s, table=38, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=1,reg0=0x1 actions=output:3,move:NXM_OF_VLAN_TCI[0..11]->NXM_NX_TUN_ID[0..11],mod_vlan_vid:2,IN_PORT
 #学习一条流表到table 30，丢报文
 cookie=0x0, duration=67346.405s, table=39, n_packets=810, n_bytes=173948, idle_age=4, hard_age=65534, priority=1 actions=learn(table=30,hard_timeout=60,priority=1,NXM_OF_VLAN_TCI[0..11],load:NXM_OF_VLAN_TCI[0..11]->NXM_NX_TUN_ID[0..11],load:0->NXM_OF_VLAN_TCI[0..11],output:NXM_OF_IN_PORT[])

ostack6

[root@ostack6 huiwei]# ovs-ofctl dump-flows br-tun
NXST_FLOW reply (xid=0x4):
 #收到镜像的流量
 cookie=0xbdce641f892fed7b, duration=68486.657s, table=0, n_packets=1217580998, n_bytes=421118108309, idle_age=0, hard_age=65534, priority=1,in_port=2 actions=resubmit(,4)
 #暂存vlan tag到reg0
 cookie=0x0, duration=67102.776s, table=3, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=1,tun_id=0xf40 actions=move:NXM_OF_VLAN_TCI[0..11]->NXM_NX_REG0[0..11],move:NXM_NX_TUN_ID[0..11]->NXM_OF_VLAN_TCI[0..11],resubmit(,35)
 #tag是0单播来的
 cookie=0x0, duration=67260.501s, table=35, n_packets=6997, n_bytes=884737, idle_age=0, hard_age=65534, priority=2,reg0=0 actions=resubmit(,36)
 #tag是1 flood过来的，需要回复
 cookie=0x0, duration=67260.274s, table=35, n_packets=807, n_bytes=173515, idle_age=34, hard_age=65534, priority=1,reg0=0x1 actions=resubmit(,36)
 cookie=0x0, duration=67260.049s, table=35, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=1,reg0=0x2 actions=resubmit(,37)
 #是这个tap service的转table 38
 cookie=0x0, duration=67102.085s, table=36, n_packets=7804, n_bytes=1058252, idle_age=0, hard_age=65534, priority=1,tun_id=0xf40 actions=resubmit(,38)
 cookie=0x0, duration=67259.822s, table=36, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=0 actions=drop
 cookie=0x0, duration=67259.595s, table=37, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=0 actions=drop
 #单播来的发往br-tap
 cookie=0x0, duration=67259.365s, table=38, n_packets=6997, n_bytes=884737, idle_age=0, hard_age=65534, priority=2,reg0=0 actions=output:3
 #flood过来的发往br-tap，同时打vlan tag 2，反射回去
 cookie=0x0, duration=67259.136s, table=38, n_packets=807, n_bytes=173515, idle_age=34, hard_age=65534, priority=1,reg0=0x1 actions=output:3,move:NXM_OF_VLAN_TCI[0..11]->NXM_NX_TUN_ID[0..11],mod_vlan_vid:2,IN_PORT


[root@ostack6 huiwei]# ovs-ofctl dump-flows br-tap
NXST_FLOW reply (xid=0x4):
 cookie=0x0, duration=66717.382s, table=0, n_packets=17278, n_bytes=760532, idle_age=43, hard_age=65534, priority=1,in_port=1 actions=resubmit(,1)
 cookie=0x0, duration=66717.160s, table=0, n_packets=7055, n_bytes=982353, idle_age=28, hard_age=65534, priority=1,in_port=2 actions=resubmit(,2)
 cookie=0x0, duration=66716.927s, table=0, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=0 actions=drop
 #是这个host上的tap service，发往br-tin
 cookie=0x0, duration=66557.818s, table=1, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=1,dl_vlan=3904 actions=IN_PORT
 #否则发往br-tun
 cookie=0x0, duration=66716.699s, table=1, n_packets=17278, n_bytes=760532, idle_age=43, hard_age=65534, priority=0 actions=output:2
 #从br-tun来的报文发往br-int
 cookie=0x0, duration=66557.588s, table=2, n_packets=7055, n_bytes=982353, idle_age=28, hard_age=65534, priority=1,dl_vlan=3904 actions=output:1
 cookie=0x0, duration=66716.472s, table=2, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=0 actions=drop


[root@ostack6 huiwei]# ovs-ofctl dump-flows br-int
NXST_FLOW reply (xid=0x4):
 #br-tap上来的报文,修改vlan tag是vm4的tag，output到vm4
 cookie=0x0, duration=64831.196s, table=0, n_packets=3329, n_bytes=609525, idle_age=11, priority=3,in_port=16,dl_vlan=3904 actions=mod_vlan_vid:5,output:19

问题

Q：qbr4桥上mac learning问题

A：vm1,vm2,vm3和vm4在同一个网段，qbr4上学习到的vm1,vm2和vm3的mac都在qvo口，那么目的mac是vm1,vm2或者vm3的报文来了，理论上qbr4会丢掉报文，不会转给vm4。我个人的想法是br-int上把报文的目的mac修改成vm4的。

Q：br-int上output action和port tag问题

A：到vm4最后流表是actions=mod_vlan_vid:5,output:19，那么到vm4的报文到底带不带vlan tag，抓包看了一下是带的tag的，openvswitch版本是2.5.5，按理解port vlan类型是access，如果是normal转发出来的报文一定不带tag，但action output就不太清楚，还没有搞清楚，不知道和ovs版本有没有关系，请大神指点。综合上一个问题，我个人觉得把流表修改成这样比较好，actions=pop_vlan,mod_dl_dst:fa:16:3d:96:9c:13,output:19，但问题是目的mac被修改了，不知道vm4里的软件需要不需要分析原来的目的MAC。

Gerrit Code Reviewreview.opendev.org

Q：没有classify

A：tap-as-a-service功能简单粗暴，分in和out两个方向把镜像流量，如果能加更多过滤条件会更实用，如果目的IP是8.8.8.8的流量镜像过来。

本文参与腾讯云自媒体同步曝光计划，分享自作者个人站点/博客。

如有侵权请联系 cloudcommunity@tencent.com 删除

编程算法