Since ethdev patch is factored out from the patch series the rest could
go to mlx5 maintainers.
On 6/7/22 15:59, Spike Du wrote:
> Add command line options to support host shaper configure.
> - Command syntax:
> mlx5 set port <port_id> host_shaper avail_thresh_triggered <0|1> rate
> <rate_num>
>
> - Example commands:
> To enable avail_thresh_triggered on port 1 and disable current host
> shaper:
> testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 1 rate 0
>
> To disable avail_thresh_triggered and current host shaper on port 1:
> testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 0 rate 0
>
> The rate unit is 100Mbps.
> To disable avail_thresh_triggered and configure a shaper of 5Gbps on
> port 1:
> testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 0 rate 50
>
> Add sample code to handle rxq available descriptor threshold event, it
> delays a while so that rxq empties, then disables host shaper and
> rearms available descriptor threshold event.
>
> Signed-off-by: Spike Du <spiked@nvidia.com>
[snip]
> +
> +static uint8_t host_shaper_avail_thresh_triggered[RTE_MAX_ETHPORTS];
> +#define SHAPER_DISABLE_DELAY_US 100000 /* 100ms */
> +
> +/**
> + * Disable the host shaper and re-arm available descriptor threshold event.
> + *
> + * @param[in] args
> + * uint32_t integer combining port_id and rxq_id.
> + */
> +static void
> +mlx5_test_host_shaper_disable(void *args)
> +{
> + uint32_t port_rxq_id = (uint32_t)(uintptr_t)args;
> + uint16_t port_id = port_rxq_id & 0xffff;
> + uint16_t qid = (port_rxq_id >> 16) & 0xffff;
> + struct rte_eth_rxq_info qinfo;
> +
> + printf("%s disable shaper\n", __func__);
> + if (rte_eth_rx_queue_info_get(port_id, qid, &qinfo)) {
> + printf("rx_queue_info_get returns error\n");
> + return;
> + }
> + /* Rearm the available descriptor threshold event. */
> + if (rte_eth_rx_avail_thresh_set(port_id, qid, qinfo.avail_thresh)) {
> + printf("config avail_thresh returns error\n");
> + return;
> + }
> + /* Only disable the shaper when avail_thresh_triggered is set. */
> + if (host_shaper_avail_thresh_triggered[port_id] &&
> + rte_pmd_mlx5_host_shaper_config(port_id, 0, 0))
> + printf("%s disable shaper returns error\n", __func__);
> +}
> +
> +void
> +mlx5_test_avail_thresh_event_handler(uint16_t port_id, uint16_t rxq_id)
> +{
> + uint32_t port_rxq_id = port_id | (rxq_id << 16);
Nobody guarantees here that port_id refers to an mlx5 port.
It could be any port with avail_thres support.
> +
> + rte_eal_alarm_set(SHAPER_DISABLE_DELAY_US,
> + mlx5_test_host_shaper_disable,
> + (void *)(uintptr_t)port_rxq_id);
> + printf("%s port_id:%u rxq_id:%u\n", __func__, port_id, rxq_id);
> +}
[snip]
> +cmdline_parse_token_string_t cmd_port_host_shaper_mlx5 =
> + TOKEN_STRING_INITIALIZER(struct cmd_port_host_shaper_result,
> + mlx5, "mlx5");
I think it lucks 'static' keyword (many cases below).
[snip]
> -----Original Message-----
> From: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> Sent: Thursday, June 9, 2022 3:55 PM
> To: Spike Du <spiked@nvidia.com>; Matan Azrad <matan@nvidia.com>;
> Slava Ovsiienko <viacheslavo@nvidia.com>; Ori Kam <orika@nvidia.com>;
> NBU-Contact-Thomas Monjalon (EXTERNAL) <thomas@monjalon.net>;
> Xiaoyun Li <xiaoyun.li@intel.com>; Aman Singh
> <aman.deep.singh@intel.com>; Yuying Zhang <yuying.zhang@intel.com>
> Cc: stephen@networkplumber.org; mb@smartsharesystems.com;
> dev@dpdk.org; Raslan Darawsheh <rasland@nvidia.com>
> Subject: Re: [PATCH v5 7/7] app/testpmd: add Host Shaper command
>
> External email: Use caution opening links or attachments
>
>
> Since ethdev patch is factored out from the patch series the rest could go to
> mlx5 maintainers.
>
> On 6/7/22 15:59, Spike Du wrote:
> > Add command line options to support host shaper configure.
> > - Command syntax:
> > mlx5 set port <port_id> host_shaper avail_thresh_triggered <0|1>
> > rate <rate_num>
> >
> > - Example commands:
> > To enable avail_thresh_triggered on port 1 and disable current host
> > shaper:
> > testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 1 rate 0
> >
> > To disable avail_thresh_triggered and current host shaper on port 1:
> > testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 0 rate 0
> >
> > The rate unit is 100Mbps.
> > To disable avail_thresh_triggered and configure a shaper of 5Gbps on
> > port 1:
> > testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 0 rate 50
> >
> > Add sample code to handle rxq available descriptor threshold event, it
> > delays a while so that rxq empties, then disables host shaper and
> > rearms available descriptor threshold event.
> >
> > Signed-off-by: Spike Du <spiked@nvidia.com>
>
> [snip]
>
> > +
> > +static uint8_t host_shaper_avail_thresh_triggered[RTE_MAX_ETHPORTS];
> > +#define SHAPER_DISABLE_DELAY_US 100000 /* 100ms */
> > +
> > +/**
> > + * Disable the host shaper and re-arm available descriptor threshold event.
> > + *
> > + * @param[in] args
> > + * uint32_t integer combining port_id and rxq_id.
> > + */
> > +static void
> > +mlx5_test_host_shaper_disable(void *args) {
> > + uint32_t port_rxq_id = (uint32_t)(uintptr_t)args;
> > + uint16_t port_id = port_rxq_id & 0xffff;
> > + uint16_t qid = (port_rxq_id >> 16) & 0xffff;
> > + struct rte_eth_rxq_info qinfo;
> > +
> > + printf("%s disable shaper\n", __func__);
> > + if (rte_eth_rx_queue_info_get(port_id, qid, &qinfo)) {
> > + printf("rx_queue_info_get returns error\n");
> > + return;
> > + }
> > + /* Rearm the available descriptor threshold event. */
> > + if (rte_eth_rx_avail_thresh_set(port_id, qid, qinfo.avail_thresh)) {
> > + printf("config avail_thresh returns error\n");
> > + return;
> > + }
> > + /* Only disable the shaper when avail_thresh_triggered is set. */
> > + if (host_shaper_avail_thresh_triggered[port_id] &&
> > + rte_pmd_mlx5_host_shaper_config(port_id, 0, 0))
> > + printf("%s disable shaper returns error\n", __func__); }
> > +
> > +void
> > +mlx5_test_avail_thresh_event_handler(uint16_t port_id, uint16_t
> > +rxq_id) {
> > + uint32_t port_rxq_id = port_id | (rxq_id << 16);
>
> Nobody guarantees here that port_id refers to an mlx5 port.
> It could be any port with avail_thres support.
I think at the beginning of this function, we can add below check:
if (rte_eth_dev_info_get(port_id, &dev_info) != 0 ||
(strncmp(dev_info.driver_name, "mlx5", 4) != 0))
return;
is it ok?
>
> > +
> > + rte_eal_alarm_set(SHAPER_DISABLE_DELAY_US,
> > + mlx5_test_host_shaper_disable,
> > + (void *)(uintptr_t)port_rxq_id);
> > + printf("%s port_id:%u rxq_id:%u\n", __func__, port_id, rxq_id);
> > +}
>
> [snip]
>
> > +cmdline_parse_token_string_t cmd_port_host_shaper_mlx5 =
> > + TOKEN_STRING_INITIALIZER(struct cmd_port_host_shaper_result,
> > + mlx5, "mlx5");
>
> I think it lucks 'static' keyword (many cases below).
Sure, will add 'static'.
>
> [snip]
@@ -69,6 +69,9 @@
#ifdef RTE_NET_BOND
#include <rte_eth_bond.h>
#endif
+#ifdef RTE_NET_MLX5
+#include "mlx5_testpmd.h"
+#endif
#include "testpmd.h"
@@ -3659,6 +3662,9 @@ struct pmd_test_command {
break;
printf("Received avail_thresh event, port:%d rxq_id:%d\n",
port_id, rxq_id);
+#ifdef RTE_NET_MLX5
+ mlx5_test_avail_thresh_event_handler(port_id, rxq_id);
+#endif
}
break;
default:
@@ -1727,3 +1727,49 @@ which can be installed from OFED mstflint package.
Meson detects ``libmtcr_ul`` existence at configure stage.
If the library is detected, the application must link with ``-lmtcr_ul``,
as done by the pkg-config file libdpdk.pc.
+
+How to use available descriptor threshold and Host Shaper
+------------------------------
+
+There are sample command lines to configure available descriptor threshold in testpmd.
+Testpmd also contains sample logic to handle available descriptor threshold event.
+The typical workflow is: testpmd configure available descriptor threshold for Rx queues, enable
+avail_thresh_triggered in host shaper and register a callback, when traffic from host is
+too high and Rx queue emptiness is below available descriptor threshold, PMD receives an event and
+firmware configures a 100Mbps shaper on host port automatically, then PMD call
+the callback registered previously, which will delay a while to let Rx queue
+empty, then disable host shaper.
+
+Let's assume we have a simple Blue Field 2 setup: port 0 is uplink, port 1
+is VF representor. Each port has 2 Rx queues.
+In order to control traffic from host to ARM, we can enable available descriptor threshold in testpmd by:
+
+.. code-block:: console
+
+ testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 1 rate 0
+ testpmd> set port 1 rxq 0 avail_thresh 70
+ testpmd> set port 1 rxq 1 avail_thresh 70
+
+The first command disables current host shaper, and enables available descriptor threshold triggered mode.
+The left commands configure available descriptor threshold to 70% of Rx queue size for both Rx queues,
+When traffic from host is too high, you can see testpmd console prints log
+about available descriptor threshold event receiving, then host shaper is disabled.
+The traffic rate from host is controlled and less drop happens in Rx queues.
+
+When disable available descriptor threshold and avail_thresh_triggered, we can invoke below commands in testpmd:
+
+.. code-block:: console
+
+ testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 0 rate 0
+ testpmd> set port 1 rxq 0 avail_thresh 0
+ testpmd> set port 1 rxq 1 avail_thresh 0
+
+It's recommended an application disables available descriptor threshold and avail_thresh_triggered before exit,
+if it enables them before.
+
+We can also configure the shaper with a value, the rate unit is 100Mbps, below
+command sets current shaper to 5Gbps and disables avail_thresh_triggered.
+
+.. code-block:: console
+
+ testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 0 rate 50
@@ -68,4 +68,8 @@ if get_option('buildtype').contains('debug')
else
cflags += [ '-UPEDANTIC' ]
endif
+
+testpmd_sources += files('mlx5_testpmd.c')
+testpmd_drivers_deps += 'net_mlx5'
+
subdir(exec_env)
new file mode 100644
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 6WIND S.A.
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include <rte_prefetch.h>
+#include <rte_common.h>
+#include <rte_branch_prediction.h>
+#include <rte_ether.h>
+#include <rte_alarm.h>
+#include <rte_pmd_mlx5.h>
+#include <rte_ethdev.h>
+#include "mlx5_testpmd.h"
+#include "testpmd.h"
+
+static uint8_t host_shaper_avail_thresh_triggered[RTE_MAX_ETHPORTS];
+#define SHAPER_DISABLE_DELAY_US 100000 /* 100ms */
+
+/**
+ * Disable the host shaper and re-arm available descriptor threshold event.
+ *
+ * @param[in] args
+ * uint32_t integer combining port_id and rxq_id.
+ */
+static void
+mlx5_test_host_shaper_disable(void *args)
+{
+ uint32_t port_rxq_id = (uint32_t)(uintptr_t)args;
+ uint16_t port_id = port_rxq_id & 0xffff;
+ uint16_t qid = (port_rxq_id >> 16) & 0xffff;
+ struct rte_eth_rxq_info qinfo;
+
+ printf("%s disable shaper\n", __func__);
+ if (rte_eth_rx_queue_info_get(port_id, qid, &qinfo)) {
+ printf("rx_queue_info_get returns error\n");
+ return;
+ }
+ /* Rearm the available descriptor threshold event. */
+ if (rte_eth_rx_avail_thresh_set(port_id, qid, qinfo.avail_thresh)) {
+ printf("config avail_thresh returns error\n");
+ return;
+ }
+ /* Only disable the shaper when avail_thresh_triggered is set. */
+ if (host_shaper_avail_thresh_triggered[port_id] &&
+ rte_pmd_mlx5_host_shaper_config(port_id, 0, 0))
+ printf("%s disable shaper returns error\n", __func__);
+}
+
+void
+mlx5_test_avail_thresh_event_handler(uint16_t port_id, uint16_t rxq_id)
+{
+ uint32_t port_rxq_id = port_id | (rxq_id << 16);
+
+ rte_eal_alarm_set(SHAPER_DISABLE_DELAY_US,
+ mlx5_test_host_shaper_disable,
+ (void *)(uintptr_t)port_rxq_id);
+ printf("%s port_id:%u rxq_id:%u\n", __func__, port_id, rxq_id);
+}
+
+/**
+ * Configure host shaper's avail_thresh_triggered and current rate.
+ *
+ * @param[in] avail_thresh_triggered
+ * Disable/enable avail_thresh_triggered.
+ * @param[in] rate
+ * Configure current host shaper rate.
+ * @return
+ * On success, returns 0.
+ * On failure, returns < 0.
+ */
+static int
+mlx5_test_set_port_host_shaper(uint16_t port_id, uint16_t avail_thresh_triggered, uint8_t rate)
+{
+ struct rte_eth_link link;
+ bool port_id_valid = false;
+ uint16_t pid;
+ int ret;
+
+ RTE_ETH_FOREACH_DEV(pid)
+ if (port_id == pid) {
+ port_id_valid = true;
+ break;
+ }
+ if (!port_id_valid)
+ return -EINVAL;
+ ret = rte_eth_link_get_nowait(port_id, &link);
+ if (ret < 0)
+ return ret;
+ host_shaper_avail_thresh_triggered[port_id] = avail_thresh_triggered ? 1 : 0;
+ if (!avail_thresh_triggered) {
+ ret = rte_pmd_mlx5_host_shaper_config(port_id, 0,
+ RTE_BIT32(MLX5_HOST_SHAPER_FLAG_AVAIL_THRESH_TRIGGERED));
+ } else {
+ ret = rte_pmd_mlx5_host_shaper_config(port_id, 1,
+ RTE_BIT32(MLX5_HOST_SHAPER_FLAG_AVAIL_THRESH_TRIGGERED));
+ }
+ if (ret)
+ return ret;
+ ret = rte_pmd_mlx5_host_shaper_config(port_id, rate, 0);
+ if (ret)
+ return ret;
+ return 0;
+}
+
+/* *** SET HOST_SHAPER FOR A PORT *** */
+struct cmd_port_host_shaper_result {
+ cmdline_fixed_string_t mlx5;
+ cmdline_fixed_string_t set;
+ cmdline_fixed_string_t port;
+ uint16_t port_num;
+ cmdline_fixed_string_t host_shaper;
+ cmdline_fixed_string_t avail_thresh_triggered;
+ uint16_t fr;
+ cmdline_fixed_string_t rate;
+ uint8_t rate_num;
+};
+
+static void cmd_port_host_shaper_parsed(void *parsed_result,
+ __rte_unused struct cmdline *cl,
+ __rte_unused void *data)
+{
+ struct cmd_port_host_shaper_result *res = parsed_result;
+ int ret = 0;
+
+ if ((strcmp(res->mlx5, "mlx5") == 0) &&
+ (strcmp(res->set, "set") == 0) &&
+ (strcmp(res->port, "port") == 0) &&
+ (strcmp(res->host_shaper, "host_shaper") == 0) &&
+ (strcmp(res->avail_thresh_triggered, "avail_thresh_triggered") == 0) &&
+ (strcmp(res->rate, "rate") == 0))
+ ret = mlx5_test_set_port_host_shaper(res->port_num, res->fr,
+ res->rate_num);
+ if (ret < 0)
+ printf("cmd_port_host_shaper error: (%s)\n", strerror(-ret));
+}
+
+cmdline_parse_token_string_t cmd_port_host_shaper_mlx5 =
+ TOKEN_STRING_INITIALIZER(struct cmd_port_host_shaper_result,
+ mlx5, "mlx5");
+cmdline_parse_token_string_t cmd_port_host_shaper_set =
+ TOKEN_STRING_INITIALIZER(struct cmd_port_host_shaper_result,
+ set, "set");
+cmdline_parse_token_string_t cmd_port_host_shaper_port =
+ TOKEN_STRING_INITIALIZER(struct cmd_port_host_shaper_result,
+ port, "port");
+cmdline_parse_token_num_t cmd_port_host_shaper_portnum =
+ TOKEN_NUM_INITIALIZER(struct cmd_port_host_shaper_result,
+ port_num, RTE_UINT16);
+cmdline_parse_token_string_t cmd_port_host_shaper_host_shaper =
+ TOKEN_STRING_INITIALIZER(struct cmd_port_host_shaper_result,
+ host_shaper, "host_shaper");
+cmdline_parse_token_string_t cmd_port_host_shaper_avail_thresh_triggered =
+ TOKEN_STRING_INITIALIZER(struct cmd_port_host_shaper_result,
+ avail_thresh_triggered, "avail_thresh_triggered");
+cmdline_parse_token_num_t cmd_port_host_shaper_fr =
+ TOKEN_NUM_INITIALIZER(struct cmd_port_host_shaper_result,
+ fr, RTE_UINT16);
+cmdline_parse_token_string_t cmd_port_host_shaper_rate =
+ TOKEN_STRING_INITIALIZER(struct cmd_port_host_shaper_result,
+ rate, "rate");
+cmdline_parse_token_num_t cmd_port_host_shaper_rate_num =
+ TOKEN_NUM_INITIALIZER(struct cmd_port_host_shaper_result,
+ rate_num, RTE_UINT8);
+cmdline_parse_inst_t mlx5_test_cmd_port_host_shaper = {
+ .f = cmd_port_host_shaper_parsed,
+ .data = (void *)0,
+ .help_str = "mlx5 set port <port_id> host_shaper avail_thresh_triggered <0|1> "
+ "rate <rate_num>: Set HOST_SHAPER avail_thresh_triggered and rate with port_id",
+ .tokens = {
+ (void *)&cmd_port_host_shaper_mlx5,
+ (void *)&cmd_port_host_shaper_set,
+ (void *)&cmd_port_host_shaper_port,
+ (void *)&cmd_port_host_shaper_portnum,
+ (void *)&cmd_port_host_shaper_host_shaper,
+ (void *)&cmd_port_host_shaper_avail_thresh_triggered,
+ (void *)&cmd_port_host_shaper_fr,
+ (void *)&cmd_port_host_shaper_rate,
+ (void *)&cmd_port_host_shaper_rate_num,
+ NULL,
+ }
+};
+
+struct testpmd_driver_commands mlx5_driver_cmds = {
+ .commands = {
+ {
+ .ctx = &mlx5_test_cmd_port_host_shaper,
+ .help = "mlx5 set port (port_id) host_shaper avail_thresh_triggered (on|off)"
+ "rate (rate_num):\n"
+ " Set HOST_SHAPER avail_thresh_triggered and rate with port_id\n\n",
+ },
+ {
+ .ctx = NULL,
+ },
+ }
+};
+TESTPMD_ADD_DRIVER_COMMANDS(mlx5_driver_cmds);
+
new file mode 100644
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 6WIND S.A.
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_TEST_H_
+#define RTE_PMD_MLX5_TEST_H_
+
+#include <cmdline_parse.h>
+#include <cmdline_parse_num.h>
+#include <cmdline_parse_string.h>
+
+/**
+ * RTE_ETH_EVENT_RX_AVAIL_THRESH handler sample code.
+ * It's called in testpmd, the work flow here is delay a while until
+ * RX queueu is empty, then disable host shaper.
+ *
+ * @param[in] port_id
+ * Port identifier.
+ * @param[in] rxq_id
+ * Rx queue identifier.
+ */
+void
+mlx5_test_avail_thresh_event_handler(uint16_t port_id, uint16_t rxq_id);
+
+#endif