dpdk X710 symmetric RSS

dpdk-vpp源码解读

发布于 2023-03-07 17:09:00

1.1K0

发布于 2023-03-07 17:09:00

文章被收录于专栏：DPDK VPP源码分析

RSS(Receive Side Scaling)是一种能够在多处理器系统下使接收报文在多个CPU之间高效分发的网卡驱动技术。

网卡对接收到的报文进行解析，获取IP地址、协议和端口五元组信息
网卡通过配置的HASH函数根据五元组信息计算出HASH值,也可以根据二、三或四元组进行计算。
取HASH值的低几位(这个具体网卡可能不同)作为RETA(redirection table)的索引
根据RETA中存储的值分发到对应的CPU

下图描述了完整的处理流程：

Symmetric RSS

对称RSS的目的是为了保证来自一个TCP连接的两个方向的报文被hash到相同的Rx队列上。这样相同的TCP连接的统计信息可以存储在每个队列数据结构中，从而多线程处理中避免了加锁。

最近项目中需要在处理tcp 会话中，需要保证tcp连接的两个方向的报文被hash到一个接收队列上。在dpdk官网测试用例76.3。从代码上分析x710网卡是支持的。

下图命令在test_pmd使能port0的对称哈希功能。

#port id 0 设置ip4-other 使能hash函数t
set_hash_global_config 0 toeplitz ipv4-other enable
#port id 0的使能对称哈希
set_sym_hash_ena_per_port 0 enable
#多队列模式下，下面会被hash到一个队列上
sendp([Ether(dst="90:e2:ba:36:99:3c")/IP(src="192.168.0.4", dst="192.168.0.5")], iface="eth3")
sendp([Ether(dst="90:e2:ba:36:99:3c")/IP(src="192.168.0.5", dst="192.168.0.4")], iface="eth3")

具体的代码在dpdk/app/test-pmd/cmdline.c文件中如下：

其中新增了一个 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ类型的hash函数，在dpdk19.5中还不支持。没有具体测试过。

static void
cmd_set_hash_global_config_parsed(void *parsed_result,
          __rte_unused struct cmdline *cl,
          __rte_unused void *data)
{
  struct cmd_set_hash_global_config_result *res = parsed_result;
  struct rte_eth_hash_filter_info info;
  uint32_t ftype, idx, offset;
  int ret;

  if (rte_eth_dev_filter_supported(res->port_id,
        RTE_ETH_FILTER_HASH) < 0) {
    printf("RTE_ETH_FILTER_HASH not supported on port %d\n",
              res->port_id);
    return;
  }
  memset(&info, 0, sizeof(info));
  info.info_type = RTE_ETH_HASH_FILTER_GLOBAL_CONFIG;
  if (!strcmp(res->hash_func, "toeplitz"))
    info.info.global_conf.hash_func =
      RTE_ETH_HASH_FUNCTION_TOEPLITZ;
  else if (!strcmp(res->hash_func, "simple_xor"))
    info.info.global_conf.hash_func =
      RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
  else if (!strcmp(res->hash_func, "symmetric_toeplitz"))
    info.info.global_conf.hash_func =
      RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
  else if (!strcmp(res->hash_func, "default"))
    info.info.global_conf.hash_func =
      RTE_ETH_HASH_FUNCTION_DEFAULT;

  ftype = str2flowtype(res->flow_type);
  idx = ftype / UINT64_BIT;
  offset = ftype % UINT64_BIT;
  info.info.global_conf.valid_bit_mask[idx] |= (1ULL << offset);
  if (!strcmp(res->enable, "enable"))
    info.info.global_conf.sym_hash_enable_mask[idx] |=
            (1ULL << offset);
  ret = rte_eth_dev_filter_ctrl(res->port_id, RTE_ETH_FILTER_HASH,
          RTE_ETH_FILTER_SET, &info);
  if (ret < 0)
    printf("Cannot set global hash configurations by port %d\n",
              res->port_id);
  else
    printf("Global hash configurations have been set "
      "successfully by port %d\n", res->port_id);
}
static void
cmd_set_sym_hash_per_port_parsed(void *parsed_result,
         __rte_unused struct cmdline *cl,
         __rte_unused void *data)
{
  struct cmd_set_sym_hash_ena_per_port_result *res = parsed_result;
  struct rte_eth_hash_filter_info info;
  int ret;
    /*82599网卡是不支持的，X710网卡是支持的*/
  if (rte_eth_dev_filter_supported(res->port_id,
        RTE_ETH_FILTER_HASH) < 0) {
    printf("RTE_ETH_FILTER_HASH not supported on port: %d\n",
              res->port_id);
    return;
  }
  memset(&info, 0, sizeof(info));
  info.info_type = RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT;
  if (!strcmp(res->enable, "enable"))
    info.info.enable = 1;
  ret = rte_eth_dev_filter_ctrl(res->port_id, RTE_ETH_FILTER_HASH,
          RTE_ETH_FILTER_SET, &info);
  if (ret < 0) {
    printf("Cannot set symmetric hash enable per port on "
          "port %u\n", res->port_id);
    return;
  }
  printf("Symmetric hash has been set to %s on port %u\n",
          res->enable, res->port_id);
}

下面是X710网卡具体的设置代码？：

struct rte_eth_conf port_conf = {
  .rxmode = {
    .mq_mode = ETH_MQ_RX_RSS,
  },
  .rx_adv_conf = {
    .rss_conf = {
        .rss_hf = ETH_RSS_IP |
              ETH_RSS_TCP |
              ETH_RSS_UDP |
              ETH_RSS_SCTP,
    }
   },
};

rte_eth_dev_configure(port_id, rx_queue_num, tx_queue_num, &port_conf);

int sym_hash_enable(int port_id, uint32_t ftype, enum rte_eth_hash_function function)
{
    struct rte_eth_hash_filter_info info;
    int ret = 0;
    uint32_t idx = 0;
    uint32_t offset = 0;

    memset(&info, 0, sizeof(info));

    ret = rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH);
    if (ret < 0) {
        DPDK_ERROR("RTE_ETH_FILTER_HASH not supported on port: %d",
                         port_id);
        return ret;
    }

    info.info_type = RTE_ETH_HASH_FILTER_GLOBAL_CONFIG;
    info.info.global_conf.hash_func = function;

    idx = ftype / UINT64_BIT;
    offset = ftype % UINT64_BIT;
    info.info.global_conf.valid_bit_mask[idx] |= (1ULL << offset);
    info.info.global_conf.sym_hash_enable_mask[idx] |=
                        (1ULL << offset);

    ret = rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH,
                                  RTE_ETH_FILTER_SET, &info);
    if (ret < 0)
    {
        DPDK_ERROR("Cannot set global hash configurations"
                        "on port %u", port_id);
        return ret;
    }

    return 0;
}

int sym_hash_set(int port_id, int enable)
{
    int ret = 0;
    struct rte_eth_hash_filter_info info;

    memset(&info, 0, sizeof(info));

    ret = rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH);
    if (ret < 0) {
        DPDK_ERROR("RTE_ETH_FILTER_HASH not supported on port: %d",
                         port_id);
        return ret;
    }

    info.info_type = RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT;
    info.info.enable = enable;
    ret = rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH,
                        RTE_ETH_FILTER_SET, &info);

    if (ret < 0)
    {
        DPDK_ERROR("Cannot set symmetric hash enable per port "
                        "on port %u", port_id);
        return ret;
    }

    return 0;
}

sym_hash_enable(port_id, RTE_ETH_FLOW_NONFRAG_IPV4_TCP, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
sym_hash_enable(port_id, RTE_ETH_FLOW_NONFRAG_IPV4_UDP, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
sym_hash_enable(port_id, RTE_ETH_FLOW_FRAG_IPV4, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
sym_hash_enable(port_id, RTE_ETH_FLOW_NONFRAG_IPV4_SCTP, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
sym_hash_enable(port_id, RTE_ETH_FLOW_NONFRAG_IPV4_OTHER, RTE_ETH_HASH_FUNCTION_TOEPLITZ);

sym_hash_set(port_id, 1);

上面代码来自文章：

https://haryachyy.wordpress.com/2019/01/18/learning-dpdk-symmetric-rss/

另外一篇博客上关于对称RSS，也很详细，并且有对应的事例代码：

https://www.yuque.com/zzqcn/opensource/ca9run

本文参与腾讯云自媒体同步曝光计划，分享自微信公众号。

原始发表：2020-09-23，如有侵权请联系 cloudcommunity@tencent.com 删除

开源