net/mlx5: support metadata as flow rule criteria
Checks
Commit Message
As described in series starting at [1], it adds option to set metadata
value as match pattern when creating a new flow rule.
This patch adds metadata support in mlx5 driver, in several parts:
- Add the setting of metadata value in matcher when creating
a new flow rule.
- Add the passing of metadata value from mbuf to wqe when
indicated by ol_flag, in different burst functions.
- Allow flow rule with attribute egress in specific cases.
This patch must be built together with the files in series [1].
[1] "ethdev: support metadata as flow rule criteria"
Signed-off-by: Dekel Peled <dekelp@mellanox.com>
---
drivers/net/mlx5/mlx5_flow.c | 60 +++++++++++++----
drivers/net/mlx5/mlx5_flow.h | 22 +++++--
drivers/net/mlx5/mlx5_flow_dv.c | 120 ++++++++++++++++++++++++++++++----
drivers/net/mlx5/mlx5_flow_verbs.c | 21 ++++--
drivers/net/mlx5/mlx5_prm.h | 2 +-
drivers/net/mlx5/mlx5_rxtx.c | 34 ++++++++--
drivers/net/mlx5/mlx5_rxtx_vec.c | 28 ++++++--
drivers/net/mlx5/mlx5_rxtx_vec.h | 1 +
drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 4 +-
drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 4 +-
drivers/net/mlx5/mlx5_txq.c | 6 ++
11 files changed, 255 insertions(+), 47 deletions(-)
Comments
> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Dekel Peled
> Sent: Sunday, September 16, 2018 9:42 PM
> To: dev@dpdk.org; Shahaf Shuler <shahafs@mellanox.com>; Yongseok Koh <yskoh@mellanox.com>
> Cc: Ori Kam <orika@mellanox.com>
> Subject: [dpdk-dev] [PATCH] net/mlx5: support metadata as flow rule criteria
>
> As described in series starting at [1], it adds option to set metadata value as match pattern when
> creating a new flow rule.
>
> This patch adds metadata support in mlx5 driver, in several parts:
> - Add the setting of metadata value in matcher when creating a new flow rule.
> - Add the passing of metadata value from mbuf to wqe when indicated by ol_flag, in different burst
> functions.
> - Allow flow rule with attribute egress in specific cases.
>
> This patch must be built together with the files in series [1].
>
> [1] "ethdev: support metadata as flow rule criteria"
>
> Signed-off-by: Dekel Peled <dekelp@mellanox.com>
> ---
> drivers/net/mlx5/mlx5_flow.c | 60 +++++++++++++----
> drivers/net/mlx5/mlx5_flow.h | 22 +++++--
> drivers/net/mlx5/mlx5_flow_dv.c | 120 ++++++++++++++++++++++++++++++----
> drivers/net/mlx5/mlx5_flow_verbs.c | 21 ++++--
> drivers/net/mlx5/mlx5_prm.h | 2 +-
> drivers/net/mlx5/mlx5_rxtx.c | 34 ++++++++--
> drivers/net/mlx5/mlx5_rxtx_vec.c | 28 ++++++--
> drivers/net/mlx5/mlx5_rxtx_vec.h | 1 +
> drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 4 +-
> drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 4 +-
> drivers/net/mlx5/mlx5_txq.c | 6 ++
> 11 files changed, 255 insertions(+), 47 deletions(-)
>
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 4234be6..7932e0f
> 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -402,7 +402,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev,
> * @return
> * 0 on success, a negative errno value otherwise and rte_errno is set.
> */
> -static int
> +int
> mlx5_flow_item_acceptable(const struct rte_flow_item *item,
> const uint8_t *mask,
> const uint8_t *nic_mask,
> @@ -602,7 +602,8 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev,
> * 0 on success, a negative errno value otherwise and rte_ernno is set.
> */
> int mlx5_flow_validate_action_flag(uint64_t action_flags,
> - struct rte_flow_error *error)
> + const struct rte_flow_attr *attr,
> + struct rte_flow_error *error)
> {
>
> if (action_flags & MLX5_ACTION_DROP)
> @@ -624,6 +625,11 @@ int mlx5_flow_validate_action_flag(uint64_t action_flags,
> NULL,
> "can't have 2 flag"
> " actions in same flow");
> + if (attr->egress)
> + return rte_flow_error_set(error, ENOTSUP,
> + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> + NULL,
> + "flag action not supported for egress");
> return 0;
> }
>
> @@ -642,8 +648,9 @@ int mlx5_flow_validate_action_flag(uint64_t action_flags,
> * 0 on success, a negative errno value otherwise and rte_ernno is set.
> */
> int mlx5_flow_validate_action_mark(uint64_t action_flags,
> - const struct rte_flow_action *action,
> - struct rte_flow_error *error)
> + const struct rte_flow_action *action,
> + const struct rte_flow_attr *attr,
> + struct rte_flow_error *error)
> {
> const struct rte_flow_action_mark *mark = action->conf;
>
> @@ -677,6 +684,11 @@ int mlx5_flow_validate_action_mark(uint64_t action_flags,
> NULL,
> "can't have 2 flag actions in same"
> " flow");
> + if (attr->egress)
> + return rte_flow_error_set(error, ENOTSUP,
> + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> + NULL,
> + "mark action not supported for egress");
> return 0;
> }
>
> @@ -693,7 +705,8 @@ int mlx5_flow_validate_action_mark(uint64_t action_flags,
> * 0 on success, a negative errno value otherwise and rte_ernno is set.
> */
> int mlx5_flow_validate_action_drop(uint64_t action_flags,
> - struct rte_flow_error *error)
> + const struct rte_flow_attr *attr,
> + struct rte_flow_error *error)
> {
> if (action_flags & MLX5_ACTION_FLAG)
> return rte_flow_error_set(error,
> @@ -715,6 +728,11 @@ int mlx5_flow_validate_action_drop(uint64_t action_flags,
> NULL,
> "can't have 2 fate actions in"
> " same flow");
> + if (attr->egress)
> + return rte_flow_error_set(error, ENOTSUP,
> + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> + NULL,
> + "drop action not supported for egress");
> return 0;
> }
>
> @@ -735,9 +753,10 @@ int mlx5_flow_validate_action_drop(uint64_t action_flags,
> * 0 on success, a negative errno value otherwise and rte_ernno is set.
> */
> int mlx5_flow_validate_action_queue(uint64_t action_flags,
> - struct rte_eth_dev *dev,
> - const struct rte_flow_action *action,
> - struct rte_flow_error *error)
> + struct rte_eth_dev *dev,
> + const struct rte_flow_action *action,
> + const struct rte_flow_attr *attr,
> + struct rte_flow_error *error)
> {
> struct priv *priv = dev->data->dev_private;
> const struct rte_flow_action_queue *queue = action->conf; @@ -760,6 +779,11 @@ int
> mlx5_flow_validate_action_queue(uint64_t action_flags,
> RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> &queue->index,
> "queue is not configured");
> + if (attr->egress)
> + return rte_flow_error_set(error, ENOTSUP,
> + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> + NULL,
> + "queue action not supported for egress");
> return 0;
> }
>
> @@ -780,9 +804,10 @@ int mlx5_flow_validate_action_queue(uint64_t action_flags,
> * 0 on success, a negative errno value otherwise and rte_ernno is set.
> */
> int mlx5_flow_validate_action_rss(uint64_t action_flags,
> - struct rte_eth_dev *dev,
> - const struct rte_flow_action *action,
> - struct rte_flow_error *error)
> + struct rte_eth_dev *dev,
> + const struct rte_flow_action *action,
> + const struct rte_flow_attr *attr,
> + struct rte_flow_error *error)
> {
> struct priv *priv = dev->data->dev_private;
> const struct rte_flow_action_rss *rss = action->conf; @@ -839,6 +864,11 @@ int
> mlx5_flow_validate_action_rss(uint64_t action_flags,
> &rss->queue[i],
> "queue is not configured");
> }
> + if (attr->egress)
> + return rte_flow_error_set(error, ENOTSUP,
> + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> + NULL,
> + "rss action not supported for egress");
> return 0;
> }
>
> @@ -855,7 +885,8 @@ int mlx5_flow_validate_action_rss(uint64_t action_flags,
> * 0 on success, a negative errno value otherwise and rte_ernno is set.
> */
> int mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
> - struct rte_flow_error *error)
> + const struct rte_flow_attr *attr,
> + struct rte_flow_error *error)
> {
> struct priv *priv = dev->data->dev_private;
>
> @@ -864,6 +895,11 @@ int mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
> RTE_FLOW_ERROR_TYPE_ACTION,
> NULL,
> "flow counters are not supported.");
> + if (attr->egress)
> + return rte_flow_error_set(error, ENOTSUP,
> + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> + NULL,
> + "count action not supported for egress");
> return 0;
> }
>
> diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index f1a72d4..e0446c5
> 100644
> --- a/drivers/net/mlx5/mlx5_flow.h
> +++ b/drivers/net/mlx5/mlx5_flow.h
> @@ -43,6 +43,9 @@
> #define MLX5_FLOW_LAYER_GRE (1u << 14)
> #define MLX5_FLOW_LAYER_MPLS (1u << 15)
>
> +/* General pattern items bits. */
> +#define MLX5_FLOW_ITEM_METADATA (1u << 16)
> +
> /* Outer Masks. */
> #define MLX5_FLOW_LAYER_OUTER_L3 \
> (MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6) @@ -237,21 +240,27 @@ struct
> mlx5_flow_driver_ops {
>
> /* mlx5_flow.c */
> int mlx5_flow_validate_action_flag(uint64_t action_flags,
> - struct rte_flow_error *error);
> + const struct rte_flow_attr *attr,
> + struct rte_flow_error *error);
> int mlx5_flow_validate_action_mark(uint64_t action_flags,
> - const struct rte_flow_action *action,
> - struct rte_flow_error *error);
> + const struct rte_flow_action *action,
> + const struct rte_flow_attr *attr,
> + struct rte_flow_error *error);
> int mlx5_flow_validate_action_drop(uint64_t action_flags,
> - struct rte_flow_error *error);
> + const struct rte_flow_attr *attr,
> + struct rte_flow_error *error);
> int mlx5_flow_validate_action_queue(uint64_t action_flags,
> struct rte_eth_dev *dev,
> const struct rte_flow_action *action,
> + const struct rte_flow_attr *attr,
> struct rte_flow_error *error);
> int mlx5_flow_validate_action_rss(uint64_t action_flags,
> struct rte_eth_dev *dev,
> const struct rte_flow_action *action,
> + const struct rte_flow_attr *attr,
> struct rte_flow_error *error);
> int mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
> + const struct rte_flow_attr *attr,
> struct rte_flow_error *error);
> int mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
> const struct rte_flow_attr *attributes, @@ -294,6 +303,11 @@ int
> mlx5_flow_validate_item_mpls(uint64_t item_flags, void mlx5_flow_init_driver_ops(struct rte_eth_dev
> *dev); uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
> uint32_t subpriority);
> +int mlx5_flow_item_acceptable(const struct rte_flow_item *item,
> + const uint8_t *mask,
> + const uint8_t *nic_mask,
> + unsigned int size,
> + struct rte_flow_error *error);
Above invalid egress actions checks seems less relation to metadata, how about split this patch into:
1. egress support
2. metadata support
>
> /* mlx5_flow_dv.c */
> void mlx5_flow_dv_get_driver_ops(struct mlx5_flow_driver_ops *flow_ops); diff --git
> a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 4090c5f..3b52181 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -37,6 +37,47 @@
> #ifdef HAVE_IBV_FLOW_DV_SUPPORT
>
> /**
> + * Validate META item.
> + *
> + * @param[in] item
> + * Item specification.
> + * @param[out] error
> + * Pointer to error structure.
> + *
> + * @return
> + * 0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_validate_item_meta(const struct rte_flow_item *item,
> + const struct rte_flow_attr *attributes,
> + struct rte_flow_error *error)
> +{
> + const struct rte_flow_item_meta *mask = item->mask;
> +
> + const struct rte_flow_item_meta nic_mask = {
> + .data = RTE_BE32(UINT32_MAX)
> + };
> +
> + int ret;
> +
> + if (!mask)
> + mask = &rte_flow_item_meta_mask;
> + ret = mlx5_flow_item_acceptable
> + (item, (const uint8_t *)mask,
> + (const uint8_t *)&nic_mask,
> + sizeof(struct rte_flow_item_meta), error);
Please follow line wrapping format.
> + if (ret < 0)
> + return ret;
> +
> + if (attributes->ingress)
> + return rte_flow_error_set(error, ENOTSUP,
> + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
> + NULL,
> + "pattern not supported for ingress");
> + return 0;
What if spec not specified?
> +}
> +
> +/**
> * Verify the @p attributes will be correctly understood by the NIC and store
> * them in the @p flow if everything is correct.
> *
> @@ -68,21 +109,17 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
> RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
> NULL,
> "priority out of range");
> - if (attributes->egress)
> - return rte_flow_error_set(error, ENOTSUP,
> - RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> - NULL,
> - "egress is not supported");
> if (attributes->transfer)
> return rte_flow_error_set(error, ENOTSUP,
> RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
> NULL,
> "transfer is not supported");
> - if (!attributes->ingress)
> + if (!(attributes->egress ^ attributes->ingress))
> return rte_flow_error_set(error, ENOTSUP,
> - RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
> + RTE_FLOW_ERROR_TYPE_ATTR,
> NULL,
> - "ingress attribute is mandatory");
> + "must specify exactly one of "
> + "ingress or egress");
> return 0;
> }
>
> @@ -219,6 +256,12 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
> return ret;
> item_flags |= MLX5_FLOW_LAYER_MPLS;
> break;
> + case RTE_FLOW_ITEM_TYPE_META:
> + ret = mlx5_flow_validate_item_meta(items, attr, error);
> + if (ret < 0)
> + return ret;
> + item_flags |= MLX5_FLOW_ITEM_METADATA;
> + break;
> default:
> return rte_flow_error_set(error, ENOTSUP,
> RTE_FLOW_ERROR_TYPE_ITEM,
> @@ -233,6 +276,7 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
> break;
> case RTE_FLOW_ACTION_TYPE_FLAG:
> ret = mlx5_flow_validate_action_flag(action_flags,
> + attr,
> error);
> if (ret < 0)
> return ret;
> @@ -241,6 +285,7 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
> case RTE_FLOW_ACTION_TYPE_MARK:
> ret = mlx5_flow_validate_action_mark(action_flags,
> actions,
> + attr,
> error);
> if (ret < 0)
> return ret;
> @@ -248,27 +293,36 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
> break;
> case RTE_FLOW_ACTION_TYPE_DROP:
> ret = mlx5_flow_validate_action_drop(action_flags,
> + attr,
> error);
> if (ret < 0)
> return ret;
> action_flags |= MLX5_ACTION_DROP;
> break;
> case RTE_FLOW_ACTION_TYPE_QUEUE:
> - ret = mlx5_flow_validate_action_queue(action_flags, dev,
> - actions, error);
> + ret = mlx5_flow_validate_action_queue(action_flags,
> + dev,
> + actions,
> + attr,
> + error);
> if (ret < 0)
> return ret;
> action_flags |= MLX5_ACTION_QUEUE;
> break;
> case RTE_FLOW_ACTION_TYPE_RSS:
> - ret = mlx5_flow_validate_action_rss(action_flags, dev,
> - actions, error);
> + ret = mlx5_flow_validate_action_rss(action_flags,
> + dev,
> + actions,
> + attr,
> + error);
> if (ret < 0)
> return ret;
> action_flags |= MLX5_ACTION_RSS;
> break;
> case RTE_FLOW_ACTION_TYPE_COUNT:
> - ret = mlx5_flow_validate_action_count(dev, error);
> + ret = mlx5_flow_validate_action_count(dev,
> + attr,
> + error);
> if (ret < 0)
> return ret;
> action_flags |= MLX5_ACTION_COUNT;
> @@ -865,6 +919,43 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev, }
>
> /**
> + * Add META item to matcher
> + *
> + * @param[in, out] matcher
> + * Flow matcher.
> + * @param[in, out] key
> + * Flow matcher value.
> + * @param[in] item
> + * Flow pattern to translate.
> + * @param[in] inner
> + * Item is inner pattern.
> + */
> +static void
> +flow_dv_translate_item_meta(void *matcher, void *key,
> + const struct rte_flow_item *item)
> +{
> + const struct rte_flow_item_meta *metam;
> + const struct rte_flow_item_meta *metav;
> +
> + void *misc2_m =
> + MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
> + void *misc2_v =
> + MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
> +
> + metam = (const void *)item->mask;
> + if (!metam)
> + metam = &rte_flow_item_meta_mask;
> +
> + metav = (const void *)item->spec;
> + if (metav) {
> + MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
> + RTE_BE32((uint32_t)rte_be_to_cpu_64(metam->data)));
> + MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
> + RTE_BE32((uint32_t)rte_be_to_cpu_64(metav->data)));
Data size of rte_flow_item_meta.data is 32 bits in [1], no need to convert the size.
> + }
> +}
> +
> +/**
> *
> * Translate flow item.
> *
> @@ -946,6 +1037,9 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
> case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
> flow_dv_translate_item_vxlan(tmatcher->key, key, item, inner);
> break;
> + case RTE_FLOW_ITEM_TYPE_META:
> + flow_dv_translate_item_meta(tmatcher->key, key, item);
> + break;
> case RTE_FLOW_ITEM_TYPE_ICMP:
> case RTE_FLOW_ITEM_TYPE_MPLS:
> case RTE_FLOW_ITEM_TYPE_SCTP:
> diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
> index 48e816d..3c42016 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -1272,6 +1272,7 @@ struct ibv_spec_header {
> break;
> case RTE_FLOW_ACTION_TYPE_FLAG:
> ret = mlx5_flow_validate_action_flag(action_flags,
> + attr,
> error);
> if (ret < 0)
> return ret;
> @@ -1280,6 +1281,7 @@ struct ibv_spec_header {
> case RTE_FLOW_ACTION_TYPE_MARK:
> ret = mlx5_flow_validate_action_mark(action_flags,
> actions,
> + attr,
> error);
> if (ret < 0)
> return ret;
> @@ -1287,27 +1289,36 @@ struct ibv_spec_header {
> break;
> case RTE_FLOW_ACTION_TYPE_DROP:
> ret = mlx5_flow_validate_action_drop(action_flags,
> + attr,
> error);
> if (ret < 0)
> return ret;
> action_flags |= MLX5_ACTION_DROP;
> break;
> case RTE_FLOW_ACTION_TYPE_QUEUE:
> - ret = mlx5_flow_validate_action_queue(action_flags, dev,
> - actions, error);
> + ret = mlx5_flow_validate_action_queue(action_flags,
> + dev,
> + actions,
> + attr,
> + error);
> if (ret < 0)
> return ret;
> action_flags |= MLX5_ACTION_QUEUE;
> break;
> case RTE_FLOW_ACTION_TYPE_RSS:
> - ret = mlx5_flow_validate_action_rss(action_flags, dev,
> - actions, error);
> + ret = mlx5_flow_validate_action_rss(action_flags,
> + dev,
> + actions,
> + attr,
> + error);
> if (ret < 0)
> return ret;
> action_flags |= MLX5_ACTION_RSS;
> break;
> case RTE_FLOW_ACTION_TYPE_COUNT:
> - ret = mlx5_flow_validate_action_count(dev, error);
> + ret = mlx5_flow_validate_action_count(dev,
> + attr,
> + error);
> if (ret < 0)
> return ret;
> action_flags |= MLX5_ACTION_COUNT;
> diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h index 117cec7..2f33aef 100644
> --- a/drivers/net/mlx5/mlx5_prm.h
> +++ b/drivers/net/mlx5/mlx5_prm.h
> @@ -159,7 +159,7 @@ struct mlx5_wqe_eth_seg_small {
> uint8_t cs_flags;
> uint8_t rsvd1;
> uint16_t mss;
> - uint32_t rsvd2;
> + uint32_t flow_table_metadata;
> uint16_t inline_hdr_sz;
> uint8_t inline_hdr[2];
> } __rte_aligned(MLX5_WQE_DWORD_SIZE);
> diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index 2d14f8a..080de57
> 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.c
> +++ b/drivers/net/mlx5/mlx5_rxtx.c
> @@ -523,6 +523,7 @@
> uint8_t tso = txq->tso_en && (buf->ol_flags & PKT_TX_TCP_SEG);
> uint32_t swp_offsets = 0;
> uint8_t swp_types = 0;
> + uint32_t metadata = 0;
> uint16_t tso_segsz = 0;
> #ifdef MLX5_PMD_SOFT_COUNTERS
> uint32_t total_length = 0;
> @@ -566,6 +567,9 @@
> cs_flags = txq_ol_cksum_to_cs(buf);
> txq_mbuf_to_swp(txq, buf, (uint8_t *)&swp_offsets, &swp_types);
> raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE;
> + /* Copy metadata from mbuf if valid */
> + if (buf->ol_flags & PKT_TX_METADATA)
> + metadata = buf->hash.fdir.hi;
> /* Replace the Ethernet type by the VLAN if necessary. */
> if (buf->ol_flags & PKT_TX_VLAN_PKT) {
> uint32_t vlan = rte_cpu_to_be_32(0x81000000 | @@ -781,7 +785,7 @@
> swp_offsets,
> cs_flags | (swp_types << 8) |
> (rte_cpu_to_be_16(tso_segsz) << 16),
> - 0,
> + rte_cpu_to_be_32(metadata),
> (ehdr << 16) | rte_cpu_to_be_16(tso_header_sz),
> };
> } else {
> @@ -795,7 +799,7 @@
> wqe->eseg = (rte_v128u32_t){
> swp_offsets,
> cs_flags | (swp_types << 8),
> - 0,
> + rte_cpu_to_be_32(metadata),
> (ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz),
> };
> }
> @@ -861,7 +865,7 @@
> mpw->wqe->eseg.inline_hdr_sz = 0;
> mpw->wqe->eseg.rsvd0 = 0;
> mpw->wqe->eseg.rsvd1 = 0;
> - mpw->wqe->eseg.rsvd2 = 0;
> + mpw->wqe->eseg.flow_table_metadata = 0;
> mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) |
> (txq->wqe_ci << 8) |
> MLX5_OPCODE_TSO);
> @@ -971,6 +975,8 @@
> if ((mpw.state == MLX5_MPW_STATE_OPENED) &&
> ((mpw.len != length) ||
> (segs_n != 1) ||
> + (mpw.wqe->eseg.flow_table_metadata !=
> + rte_cpu_to_be_32(buf->hash.fdir.hi)) ||
> (mpw.wqe->eseg.cs_flags != cs_flags)))
> mlx5_mpw_close(txq, &mpw);
> if (mpw.state == MLX5_MPW_STATE_CLOSED) { @@ -984,6 +990,8 @@
> max_wqe -= 2;
> mlx5_mpw_new(txq, &mpw, length);
> mpw.wqe->eseg.cs_flags = cs_flags;
> + mpw.wqe->eseg.flow_table_metadata =
> + rte_cpu_to_be_32(buf->hash.fdir.hi);
> }
> /* Multi-segment packets must be alone in their MPW. */
> assert((segs_n == 1) || (mpw.pkts_n == 0)); @@ -1082,7 +1090,7 @@
> mpw->wqe->eseg.cs_flags = 0;
> mpw->wqe->eseg.rsvd0 = 0;
> mpw->wqe->eseg.rsvd1 = 0;
> - mpw->wqe->eseg.rsvd2 = 0;
> + mpw->wqe->eseg.flow_table_metadata = 0;
> inl = (struct mlx5_wqe_inl_small *)
> (((uintptr_t)mpw->wqe) + 2 * MLX5_WQE_DWORD_SIZE);
> mpw->data.raw = (uint8_t *)&inl->raw;
> @@ -1199,12 +1207,16 @@
> if (mpw.state == MLX5_MPW_STATE_OPENED) {
> if ((mpw.len != length) ||
> (segs_n != 1) ||
> + (mpw.wqe->eseg.flow_table_metadata !=
> + rte_cpu_to_be_32(buf->hash.fdir.hi)) ||
Shall we check buf->ol_flags & PKT_TX_METADATA here? Same question to similar code below.
> (mpw.wqe->eseg.cs_flags != cs_flags))
> mlx5_mpw_close(txq, &mpw);
> } else if (mpw.state == MLX5_MPW_INL_STATE_OPENED) {
> if ((mpw.len != length) ||
> (segs_n != 1) ||
> (length > inline_room) ||
> + (mpw.wqe->eseg.flow_table_metadata !=
> + rte_cpu_to_be_32(buf->hash.fdir.hi)) ||
> (mpw.wqe->eseg.cs_flags != cs_flags)) {
> mlx5_mpw_inline_close(txq, &mpw);
> inline_room =
> @@ -1224,12 +1236,20 @@
> max_wqe -= 2;
> mlx5_mpw_new(txq, &mpw, length);
> mpw.wqe->eseg.cs_flags = cs_flags;
> + /* Copy metadata from mbuf if valid */
> + if (buf->ol_flags & PKT_TX_METADATA)
> + mpw.wqe->eseg.flow_table_metadata =
> + rte_cpu_to_be_32(buf->hash.fdir.hi);
> } else {
> if (unlikely(max_wqe < wqe_inl_n))
> break;
> max_wqe -= wqe_inl_n;
> mlx5_mpw_inline_new(txq, &mpw, length);
> mpw.wqe->eseg.cs_flags = cs_flags;
> + /* Copy metadata from mbuf if valid */
> + if (buf->ol_flags & PKT_TX_METADATA)
> + mpw.wqe->eseg.flow_table_metadata =
> + rte_cpu_to_be_32(buf->hash.fdir.hi);
> }
> }
> /* Multi-segment packets must be alone in their MPW. */ @@ -1482,6 +1502,8 @@
> (length <= txq->inline_max_packet_sz &&
> inl_pad + sizeof(inl_hdr) + length >
> mpw_room) ||
> + (mpw.wqe->eseg.flow_table_metadata !=
> + rte_cpu_to_be_32(buf->hash.fdir.hi)) ||
> (mpw.wqe->eseg.cs_flags != cs_flags))
> max_wqe -= mlx5_empw_close(txq, &mpw);
> }
> @@ -1505,6 +1527,10 @@
> sizeof(inl_hdr) + length <= mpw_room &&
> !txq->mpw_hdr_dseg;
> mpw.wqe->eseg.cs_flags = cs_flags;
> + /* Copy metadata from mbuf if valid */
> + if (buf->ol_flags & PKT_TX_METADATA)
> + mpw.wqe->eseg.flow_table_metadata =
> + rte_cpu_to_be_32(buf->hash.fdir.hi);
> } else {
> /* Evaluate whether the next packet can be inlined.
> * Inlininig is possible when:
> diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
> index 0a4aed8..db36699 100644
> --- a/drivers/net/mlx5/mlx5_rxtx_vec.c
> +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
> @@ -48,25 +48,39 @@
> * Number of packets.
> * @param cs_flags
> * Pointer of flags to be returned.
> + * @param txq_offloads
> + * Offloads enabled on Tx queue
> *
> * @return
> * Number of packets having same ol_flags.
> + * If PKT_TX_METADATA is set in ol_flags, packets must have same metadata
> + * as well.
Please move such function description above member description.
> */
> static inline unsigned int
> -txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags)
> +txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n,
> + uint8_t *cs_flags, const uint64_t txq_offloads)
> {
> unsigned int pos;
> const uint64_t ol_mask =
> PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
> PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
> - PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
> + PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM | PKT_TX_METADATA;
>
> if (!pkts_n)
> return 0;
> /* Count the number of packets having same ol_flags. */
> - for (pos = 1; pos < pkts_n; ++pos)
> - if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
> + for (pos = 1; pos < pkts_n; ++pos) {
> + if ((txq_offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) &&
> + ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask))
> break;
> + /* If the metadata ol_flag is set,
> + * metadata must be same in all packets.
> + */
> + if ((txq_offloads & DEV_TX_OFFLOAD_MATCH_METADATA) &&
> + (pkts[pos]->ol_flags & PKT_TX_METADATA) &&
> + pkts[0]->hash.fdir.hi != pkts[pos]->hash.fdir.hi)
> + break;
> + }
> *cs_flags = txq_ol_cksum_to_cs(pkts[0]);
> return pos;
> }
> @@ -137,8 +151,10 @@
> n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
> if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
> n = txq_count_contig_single_seg(&pkts[nb_tx], n);
> - if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP)
> - n = txq_calc_offload(&pkts[nb_tx], n, &cs_flags);
> + if (txq->offloads & (MLX5_VEC_TX_CKSUM_OFFLOAD_CAP |
> + DEV_TX_OFFLOAD_MATCH_METADATA))
> + n = txq_calc_offload(&pkts[nb_tx], n,
> + &cs_flags, txq->offloads);
> ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
> nb_tx += ret;
> if (!ret)
> diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h
> index fb884f9..fda7004 100644
> --- a/drivers/net/mlx5/mlx5_rxtx_vec.h
> +++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
> @@ -22,6 +22,7 @@
> /* HW offload capabilities of vectorized Tx. */ #define MLX5_VEC_TX_OFFLOAD_CAP \
> (MLX5_VEC_TX_CKSUM_OFFLOAD_CAP | \
> + DEV_TX_OFFLOAD_MATCH_METADATA | \
> DEV_TX_OFFLOAD_MULTI_SEGS)
>
> /*
> diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
> index b37b738..20c9427 100644
> --- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
> +++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
> @@ -237,6 +237,7 @@
> uint8x16_t *t_wqe;
> uint8_t *dseg;
> uint8x16_t ctrl;
> + uint32_t md; /* metadata */
>
> /* Make sure all packets can fit into a single WQE. */
> assert(elts_n > pkts_n);
> @@ -293,10 +294,11 @@
> ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m);
> vst1q_u8((void *)t_wqe, ctrl);
> /* Fill ESEG in the header. */
> + md = pkts[0]->hash.fdir.hi;
> vst1q_u8((void *)(t_wqe + 1),
> ((uint8x16_t) { 0, 0, 0, 0,
> cs_flags, 0, 0, 0,
> - 0, 0, 0, 0,
> + md >> 24, md >> 16, md >> 8, md,
> 0, 0, 0, 0 }));
> #ifdef MLX5_PMD_SOFT_COUNTERS
> txq->stats.opackets += pkts_n;
> diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
> index 54b3783..7c8535c 100644
> --- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
> +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
> @@ -236,6 +236,7 @@
> 0, 1, 2, 3 /* bswap32 */);
> __m128i *t_wqe, *dseg;
> __m128i ctrl;
> + uint32_t md; /* metadata */
>
> /* Make sure all packets can fit into a single WQE. */
> assert(elts_n > pkts_n);
> @@ -292,9 +293,10 @@
> ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
> _mm_store_si128(t_wqe, ctrl);
> /* Fill ESEG in the header. */
> + md = pkts[0]->hash.fdir.hi;
> _mm_store_si128(t_wqe + 1,
> _mm_set_epi8(0, 0, 0, 0,
> - 0, 0, 0, 0,
> + md, md >> 8, md >> 16, md >> 24,
> 0, 0, 0, cs_flags,
> 0, 0, 0, 0));
> #ifdef MLX5_PMD_SOFT_COUNTERS
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index f9bc473..7263fb1 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -128,6 +128,12 @@
> offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
> DEV_TX_OFFLOAD_GRE_TNL_TSO);
> }
> +
> +#ifdef HAVE_IBV_FLOW_DV_SUPPORT
> + if (config->dv_flow_en)
> + offloads |= DEV_TX_OFFLOAD_MATCH_METADATA; #endif
> +
> return offloads;
> }
>
> --
> 1.8.3.1
@@ -402,7 +402,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev,
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
-static int
+int
mlx5_flow_item_acceptable(const struct rte_flow_item *item,
const uint8_t *mask,
const uint8_t *nic_mask,
@@ -602,7 +602,8 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev,
* 0 on success, a negative errno value otherwise and rte_ernno is set.
*/
int mlx5_flow_validate_action_flag(uint64_t action_flags,
- struct rte_flow_error *error)
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error)
{
if (action_flags & MLX5_ACTION_DROP)
@@ -624,6 +625,11 @@ int mlx5_flow_validate_action_flag(uint64_t action_flags,
NULL,
"can't have 2 flag"
" actions in same flow");
+ if (attr->egress)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+ NULL,
+ "flag action not supported for egress");
return 0;
}
@@ -642,8 +648,9 @@ int mlx5_flow_validate_action_flag(uint64_t action_flags,
* 0 on success, a negative errno value otherwise and rte_ernno is set.
*/
int mlx5_flow_validate_action_mark(uint64_t action_flags,
- const struct rte_flow_action *action,
- struct rte_flow_error *error)
+ const struct rte_flow_action *action,
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error)
{
const struct rte_flow_action_mark *mark = action->conf;
@@ -677,6 +684,11 @@ int mlx5_flow_validate_action_mark(uint64_t action_flags,
NULL,
"can't have 2 flag actions in same"
" flow");
+ if (attr->egress)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+ NULL,
+ "mark action not supported for egress");
return 0;
}
@@ -693,7 +705,8 @@ int mlx5_flow_validate_action_mark(uint64_t action_flags,
* 0 on success, a negative errno value otherwise and rte_ernno is set.
*/
int mlx5_flow_validate_action_drop(uint64_t action_flags,
- struct rte_flow_error *error)
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error)
{
if (action_flags & MLX5_ACTION_FLAG)
return rte_flow_error_set(error,
@@ -715,6 +728,11 @@ int mlx5_flow_validate_action_drop(uint64_t action_flags,
NULL,
"can't have 2 fate actions in"
" same flow");
+ if (attr->egress)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+ NULL,
+ "drop action not supported for egress");
return 0;
}
@@ -735,9 +753,10 @@ int mlx5_flow_validate_action_drop(uint64_t action_flags,
* 0 on success, a negative errno value otherwise and rte_ernno is set.
*/
int mlx5_flow_validate_action_queue(uint64_t action_flags,
- struct rte_eth_dev *dev,
- const struct rte_flow_action *action,
- struct rte_flow_error *error)
+ struct rte_eth_dev *dev,
+ const struct rte_flow_action *action,
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error)
{
struct priv *priv = dev->data->dev_private;
const struct rte_flow_action_queue *queue = action->conf;
@@ -760,6 +779,11 @@ int mlx5_flow_validate_action_queue(uint64_t action_flags,
RTE_FLOW_ERROR_TYPE_ACTION_CONF,
&queue->index,
"queue is not configured");
+ if (attr->egress)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+ NULL,
+ "queue action not supported for egress");
return 0;
}
@@ -780,9 +804,10 @@ int mlx5_flow_validate_action_queue(uint64_t action_flags,
* 0 on success, a negative errno value otherwise and rte_ernno is set.
*/
int mlx5_flow_validate_action_rss(uint64_t action_flags,
- struct rte_eth_dev *dev,
- const struct rte_flow_action *action,
- struct rte_flow_error *error)
+ struct rte_eth_dev *dev,
+ const struct rte_flow_action *action,
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error)
{
struct priv *priv = dev->data->dev_private;
const struct rte_flow_action_rss *rss = action->conf;
@@ -839,6 +864,11 @@ int mlx5_flow_validate_action_rss(uint64_t action_flags,
&rss->queue[i],
"queue is not configured");
}
+ if (attr->egress)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+ NULL,
+ "rss action not supported for egress");
return 0;
}
@@ -855,7 +885,8 @@ int mlx5_flow_validate_action_rss(uint64_t action_flags,
* 0 on success, a negative errno value otherwise and rte_ernno is set.
*/
int mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
- struct rte_flow_error *error)
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error)
{
struct priv *priv = dev->data->dev_private;
@@ -864,6 +895,11 @@ int mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
"flow counters are not supported.");
+ if (attr->egress)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+ NULL,
+ "count action not supported for egress");
return 0;
}
@@ -43,6 +43,9 @@
#define MLX5_FLOW_LAYER_GRE (1u << 14)
#define MLX5_FLOW_LAYER_MPLS (1u << 15)
+/* General pattern items bits. */
+#define MLX5_FLOW_ITEM_METADATA (1u << 16)
+
/* Outer Masks. */
#define MLX5_FLOW_LAYER_OUTER_L3 \
(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
@@ -237,21 +240,27 @@ struct mlx5_flow_driver_ops {
/* mlx5_flow.c */
int mlx5_flow_validate_action_flag(uint64_t action_flags,
- struct rte_flow_error *error);
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error);
int mlx5_flow_validate_action_mark(uint64_t action_flags,
- const struct rte_flow_action *action,
- struct rte_flow_error *error);
+ const struct rte_flow_action *action,
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error);
int mlx5_flow_validate_action_drop(uint64_t action_flags,
- struct rte_flow_error *error);
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error);
int mlx5_flow_validate_action_queue(uint64_t action_flags,
struct rte_eth_dev *dev,
const struct rte_flow_action *action,
+ const struct rte_flow_attr *attr,
struct rte_flow_error *error);
int mlx5_flow_validate_action_rss(uint64_t action_flags,
struct rte_eth_dev *dev,
const struct rte_flow_action *action,
+ const struct rte_flow_attr *attr,
struct rte_flow_error *error);
int mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
struct rte_flow_error *error);
int mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
const struct rte_flow_attr *attributes,
@@ -294,6 +303,11 @@ int mlx5_flow_validate_item_mpls(uint64_t item_flags,
void mlx5_flow_init_driver_ops(struct rte_eth_dev *dev);
uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
uint32_t subpriority);
+int mlx5_flow_item_acceptable(const struct rte_flow_item *item,
+ const uint8_t *mask,
+ const uint8_t *nic_mask,
+ unsigned int size,
+ struct rte_flow_error *error);
/* mlx5_flow_dv.c */
void mlx5_flow_dv_get_driver_ops(struct mlx5_flow_driver_ops *flow_ops);
@@ -37,6 +37,47 @@
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
/**
+ * Validate META item.
+ *
+ * @param[in] item
+ * Item specification.
+ * @param[out] error
+ * Pointer to error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_validate_item_meta(const struct rte_flow_item *item,
+ const struct rte_flow_attr *attributes,
+ struct rte_flow_error *error)
+{
+ const struct rte_flow_item_meta *mask = item->mask;
+
+ const struct rte_flow_item_meta nic_mask = {
+ .data = RTE_BE32(UINT32_MAX)
+ };
+
+ int ret;
+
+ if (!mask)
+ mask = &rte_flow_item_meta_mask;
+ ret = mlx5_flow_item_acceptable
+ (item, (const uint8_t *)mask,
+ (const uint8_t *)&nic_mask,
+ sizeof(struct rte_flow_item_meta), error);
+ if (ret < 0)
+ return ret;
+
+ if (attributes->ingress)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+ NULL,
+ "pattern not supported for ingress");
+ return 0;
+}
+
+/**
* Verify the @p attributes will be correctly understood by the NIC and store
* them in the @p flow if everything is correct.
*
@@ -68,21 +109,17 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
NULL,
"priority out of range");
- if (attributes->egress)
- return rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
- NULL,
- "egress is not supported");
if (attributes->transfer)
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
NULL,
"transfer is not supported");
- if (!attributes->ingress)
+ if (!(attributes->egress ^ attributes->ingress))
return rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+ RTE_FLOW_ERROR_TYPE_ATTR,
NULL,
- "ingress attribute is mandatory");
+ "must specify exactly one of "
+ "ingress or egress");
return 0;
}
@@ -219,6 +256,12 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
return ret;
item_flags |= MLX5_FLOW_LAYER_MPLS;
break;
+ case RTE_FLOW_ITEM_TYPE_META:
+ ret = mlx5_flow_validate_item_meta(items, attr, error);
+ if (ret < 0)
+ return ret;
+ item_flags |= MLX5_FLOW_ITEM_METADATA;
+ break;
default:
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ITEM,
@@ -233,6 +276,7 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
break;
case RTE_FLOW_ACTION_TYPE_FLAG:
ret = mlx5_flow_validate_action_flag(action_flags,
+ attr,
error);
if (ret < 0)
return ret;
@@ -241,6 +285,7 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
case RTE_FLOW_ACTION_TYPE_MARK:
ret = mlx5_flow_validate_action_mark(action_flags,
actions,
+ attr,
error);
if (ret < 0)
return ret;
@@ -248,27 +293,36 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
break;
case RTE_FLOW_ACTION_TYPE_DROP:
ret = mlx5_flow_validate_action_drop(action_flags,
+ attr,
error);
if (ret < 0)
return ret;
action_flags |= MLX5_ACTION_DROP;
break;
case RTE_FLOW_ACTION_TYPE_QUEUE:
- ret = mlx5_flow_validate_action_queue(action_flags, dev,
- actions, error);
+ ret = mlx5_flow_validate_action_queue(action_flags,
+ dev,
+ actions,
+ attr,
+ error);
if (ret < 0)
return ret;
action_flags |= MLX5_ACTION_QUEUE;
break;
case RTE_FLOW_ACTION_TYPE_RSS:
- ret = mlx5_flow_validate_action_rss(action_flags, dev,
- actions, error);
+ ret = mlx5_flow_validate_action_rss(action_flags,
+ dev,
+ actions,
+ attr,
+ error);
if (ret < 0)
return ret;
action_flags |= MLX5_ACTION_RSS;
break;
case RTE_FLOW_ACTION_TYPE_COUNT:
- ret = mlx5_flow_validate_action_count(dev, error);
+ ret = mlx5_flow_validate_action_count(dev,
+ attr,
+ error);
if (ret < 0)
return ret;
action_flags |= MLX5_ACTION_COUNT;
@@ -865,6 +919,43 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
}
/**
+ * Add META item to matcher
+ *
+ * @param[in, out] matcher
+ * Flow matcher.
+ * @param[in, out] key
+ * Flow matcher value.
+ * @param[in] item
+ * Flow pattern to translate.
+ * @param[in] inner
+ * Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_meta(void *matcher, void *key,
+ const struct rte_flow_item *item)
+{
+ const struct rte_flow_item_meta *metam;
+ const struct rte_flow_item_meta *metav;
+
+ void *misc2_m =
+ MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
+ void *misc2_v =
+ MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
+
+ metam = (const void *)item->mask;
+ if (!metam)
+ metam = &rte_flow_item_meta_mask;
+
+ metav = (const void *)item->spec;
+ if (metav) {
+ MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
+ RTE_BE32((uint32_t)rte_be_to_cpu_64(metam->data)));
+ MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
+ RTE_BE32((uint32_t)rte_be_to_cpu_64(metav->data)));
+ }
+}
+
+/**
*
* Translate flow item.
*
@@ -946,6 +1037,9 @@ static int flow_dv_validate_attributes(struct rte_eth_dev *dev,
case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
flow_dv_translate_item_vxlan(tmatcher->key, key, item, inner);
break;
+ case RTE_FLOW_ITEM_TYPE_META:
+ flow_dv_translate_item_meta(tmatcher->key, key, item);
+ break;
case RTE_FLOW_ITEM_TYPE_ICMP:
case RTE_FLOW_ITEM_TYPE_MPLS:
case RTE_FLOW_ITEM_TYPE_SCTP:
@@ -1272,6 +1272,7 @@ struct ibv_spec_header {
break;
case RTE_FLOW_ACTION_TYPE_FLAG:
ret = mlx5_flow_validate_action_flag(action_flags,
+ attr,
error);
if (ret < 0)
return ret;
@@ -1280,6 +1281,7 @@ struct ibv_spec_header {
case RTE_FLOW_ACTION_TYPE_MARK:
ret = mlx5_flow_validate_action_mark(action_flags,
actions,
+ attr,
error);
if (ret < 0)
return ret;
@@ -1287,27 +1289,36 @@ struct ibv_spec_header {
break;
case RTE_FLOW_ACTION_TYPE_DROP:
ret = mlx5_flow_validate_action_drop(action_flags,
+ attr,
error);
if (ret < 0)
return ret;
action_flags |= MLX5_ACTION_DROP;
break;
case RTE_FLOW_ACTION_TYPE_QUEUE:
- ret = mlx5_flow_validate_action_queue(action_flags, dev,
- actions, error);
+ ret = mlx5_flow_validate_action_queue(action_flags,
+ dev,
+ actions,
+ attr,
+ error);
if (ret < 0)
return ret;
action_flags |= MLX5_ACTION_QUEUE;
break;
case RTE_FLOW_ACTION_TYPE_RSS:
- ret = mlx5_flow_validate_action_rss(action_flags, dev,
- actions, error);
+ ret = mlx5_flow_validate_action_rss(action_flags,
+ dev,
+ actions,
+ attr,
+ error);
if (ret < 0)
return ret;
action_flags |= MLX5_ACTION_RSS;
break;
case RTE_FLOW_ACTION_TYPE_COUNT:
- ret = mlx5_flow_validate_action_count(dev, error);
+ ret = mlx5_flow_validate_action_count(dev,
+ attr,
+ error);
if (ret < 0)
return ret;
action_flags |= MLX5_ACTION_COUNT;
@@ -159,7 +159,7 @@ struct mlx5_wqe_eth_seg_small {
uint8_t cs_flags;
uint8_t rsvd1;
uint16_t mss;
- uint32_t rsvd2;
+ uint32_t flow_table_metadata;
uint16_t inline_hdr_sz;
uint8_t inline_hdr[2];
} __rte_aligned(MLX5_WQE_DWORD_SIZE);
@@ -523,6 +523,7 @@
uint8_t tso = txq->tso_en && (buf->ol_flags & PKT_TX_TCP_SEG);
uint32_t swp_offsets = 0;
uint8_t swp_types = 0;
+ uint32_t metadata = 0;
uint16_t tso_segsz = 0;
#ifdef MLX5_PMD_SOFT_COUNTERS
uint32_t total_length = 0;
@@ -566,6 +567,9 @@
cs_flags = txq_ol_cksum_to_cs(buf);
txq_mbuf_to_swp(txq, buf, (uint8_t *)&swp_offsets, &swp_types);
raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE;
+ /* Copy metadata from mbuf if valid */
+ if (buf->ol_flags & PKT_TX_METADATA)
+ metadata = buf->hash.fdir.hi;
/* Replace the Ethernet type by the VLAN if necessary. */
if (buf->ol_flags & PKT_TX_VLAN_PKT) {
uint32_t vlan = rte_cpu_to_be_32(0x81000000 |
@@ -781,7 +785,7 @@
swp_offsets,
cs_flags | (swp_types << 8) |
(rte_cpu_to_be_16(tso_segsz) << 16),
- 0,
+ rte_cpu_to_be_32(metadata),
(ehdr << 16) | rte_cpu_to_be_16(tso_header_sz),
};
} else {
@@ -795,7 +799,7 @@
wqe->eseg = (rte_v128u32_t){
swp_offsets,
cs_flags | (swp_types << 8),
- 0,
+ rte_cpu_to_be_32(metadata),
(ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz),
};
}
@@ -861,7 +865,7 @@
mpw->wqe->eseg.inline_hdr_sz = 0;
mpw->wqe->eseg.rsvd0 = 0;
mpw->wqe->eseg.rsvd1 = 0;
- mpw->wqe->eseg.rsvd2 = 0;
+ mpw->wqe->eseg.flow_table_metadata = 0;
mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) |
(txq->wqe_ci << 8) |
MLX5_OPCODE_TSO);
@@ -971,6 +975,8 @@
if ((mpw.state == MLX5_MPW_STATE_OPENED) &&
((mpw.len != length) ||
(segs_n != 1) ||
+ (mpw.wqe->eseg.flow_table_metadata !=
+ rte_cpu_to_be_32(buf->hash.fdir.hi)) ||
(mpw.wqe->eseg.cs_flags != cs_flags)))
mlx5_mpw_close(txq, &mpw);
if (mpw.state == MLX5_MPW_STATE_CLOSED) {
@@ -984,6 +990,8 @@
max_wqe -= 2;
mlx5_mpw_new(txq, &mpw, length);
mpw.wqe->eseg.cs_flags = cs_flags;
+ mpw.wqe->eseg.flow_table_metadata =
+ rte_cpu_to_be_32(buf->hash.fdir.hi);
}
/* Multi-segment packets must be alone in their MPW. */
assert((segs_n == 1) || (mpw.pkts_n == 0));
@@ -1082,7 +1090,7 @@
mpw->wqe->eseg.cs_flags = 0;
mpw->wqe->eseg.rsvd0 = 0;
mpw->wqe->eseg.rsvd1 = 0;
- mpw->wqe->eseg.rsvd2 = 0;
+ mpw->wqe->eseg.flow_table_metadata = 0;
inl = (struct mlx5_wqe_inl_small *)
(((uintptr_t)mpw->wqe) + 2 * MLX5_WQE_DWORD_SIZE);
mpw->data.raw = (uint8_t *)&inl->raw;
@@ -1199,12 +1207,16 @@
if (mpw.state == MLX5_MPW_STATE_OPENED) {
if ((mpw.len != length) ||
(segs_n != 1) ||
+ (mpw.wqe->eseg.flow_table_metadata !=
+ rte_cpu_to_be_32(buf->hash.fdir.hi)) ||
(mpw.wqe->eseg.cs_flags != cs_flags))
mlx5_mpw_close(txq, &mpw);
} else if (mpw.state == MLX5_MPW_INL_STATE_OPENED) {
if ((mpw.len != length) ||
(segs_n != 1) ||
(length > inline_room) ||
+ (mpw.wqe->eseg.flow_table_metadata !=
+ rte_cpu_to_be_32(buf->hash.fdir.hi)) ||
(mpw.wqe->eseg.cs_flags != cs_flags)) {
mlx5_mpw_inline_close(txq, &mpw);
inline_room =
@@ -1224,12 +1236,20 @@
max_wqe -= 2;
mlx5_mpw_new(txq, &mpw, length);
mpw.wqe->eseg.cs_flags = cs_flags;
+ /* Copy metadata from mbuf if valid */
+ if (buf->ol_flags & PKT_TX_METADATA)
+ mpw.wqe->eseg.flow_table_metadata =
+ rte_cpu_to_be_32(buf->hash.fdir.hi);
} else {
if (unlikely(max_wqe < wqe_inl_n))
break;
max_wqe -= wqe_inl_n;
mlx5_mpw_inline_new(txq, &mpw, length);
mpw.wqe->eseg.cs_flags = cs_flags;
+ /* Copy metadata from mbuf if valid */
+ if (buf->ol_flags & PKT_TX_METADATA)
+ mpw.wqe->eseg.flow_table_metadata =
+ rte_cpu_to_be_32(buf->hash.fdir.hi);
}
}
/* Multi-segment packets must be alone in their MPW. */
@@ -1482,6 +1502,8 @@
(length <= txq->inline_max_packet_sz &&
inl_pad + sizeof(inl_hdr) + length >
mpw_room) ||
+ (mpw.wqe->eseg.flow_table_metadata !=
+ rte_cpu_to_be_32(buf->hash.fdir.hi)) ||
(mpw.wqe->eseg.cs_flags != cs_flags))
max_wqe -= mlx5_empw_close(txq, &mpw);
}
@@ -1505,6 +1527,10 @@
sizeof(inl_hdr) + length <= mpw_room &&
!txq->mpw_hdr_dseg;
mpw.wqe->eseg.cs_flags = cs_flags;
+ /* Copy metadata from mbuf if valid */
+ if (buf->ol_flags & PKT_TX_METADATA)
+ mpw.wqe->eseg.flow_table_metadata =
+ rte_cpu_to_be_32(buf->hash.fdir.hi);
} else {
/* Evaluate whether the next packet can be inlined.
* Inlininig is possible when:
@@ -48,25 +48,39 @@
* Number of packets.
* @param cs_flags
* Pointer of flags to be returned.
+ * @param txq_offloads
+ * Offloads enabled on Tx queue
*
* @return
* Number of packets having same ol_flags.
+ * If PKT_TX_METADATA is set in ol_flags, packets must have same metadata
+ * as well.
*/
static inline unsigned int
-txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags)
+txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n,
+ uint8_t *cs_flags, const uint64_t txq_offloads)
{
unsigned int pos;
const uint64_t ol_mask =
PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
- PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
+ PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM | PKT_TX_METADATA;
if (!pkts_n)
return 0;
/* Count the number of packets having same ol_flags. */
- for (pos = 1; pos < pkts_n; ++pos)
- if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
+ for (pos = 1; pos < pkts_n; ++pos) {
+ if ((txq_offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) &&
+ ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask))
break;
+ /* If the metadata ol_flag is set,
+ * metadata must be same in all packets.
+ */
+ if ((txq_offloads & DEV_TX_OFFLOAD_MATCH_METADATA) &&
+ (pkts[pos]->ol_flags & PKT_TX_METADATA) &&
+ pkts[0]->hash.fdir.hi != pkts[pos]->hash.fdir.hi)
+ break;
+ }
*cs_flags = txq_ol_cksum_to_cs(pkts[0]);
return pos;
}
@@ -137,8 +151,10 @@
n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
n = txq_count_contig_single_seg(&pkts[nb_tx], n);
- if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP)
- n = txq_calc_offload(&pkts[nb_tx], n, &cs_flags);
+ if (txq->offloads & (MLX5_VEC_TX_CKSUM_OFFLOAD_CAP |
+ DEV_TX_OFFLOAD_MATCH_METADATA))
+ n = txq_calc_offload(&pkts[nb_tx], n,
+ &cs_flags, txq->offloads);
ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
nb_tx += ret;
if (!ret)
@@ -22,6 +22,7 @@
/* HW offload capabilities of vectorized Tx. */
#define MLX5_VEC_TX_OFFLOAD_CAP \
(MLX5_VEC_TX_CKSUM_OFFLOAD_CAP | \
+ DEV_TX_OFFLOAD_MATCH_METADATA | \
DEV_TX_OFFLOAD_MULTI_SEGS)
/*
@@ -237,6 +237,7 @@
uint8x16_t *t_wqe;
uint8_t *dseg;
uint8x16_t ctrl;
+ uint32_t md; /* metadata */
/* Make sure all packets can fit into a single WQE. */
assert(elts_n > pkts_n);
@@ -293,10 +294,11 @@
ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m);
vst1q_u8((void *)t_wqe, ctrl);
/* Fill ESEG in the header. */
+ md = pkts[0]->hash.fdir.hi;
vst1q_u8((void *)(t_wqe + 1),
((uint8x16_t) { 0, 0, 0, 0,
cs_flags, 0, 0, 0,
- 0, 0, 0, 0,
+ md >> 24, md >> 16, md >> 8, md,
0, 0, 0, 0 }));
#ifdef MLX5_PMD_SOFT_COUNTERS
txq->stats.opackets += pkts_n;
@@ -236,6 +236,7 @@
0, 1, 2, 3 /* bswap32 */);
__m128i *t_wqe, *dseg;
__m128i ctrl;
+ uint32_t md; /* metadata */
/* Make sure all packets can fit into a single WQE. */
assert(elts_n > pkts_n);
@@ -292,9 +293,10 @@
ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
_mm_store_si128(t_wqe, ctrl);
/* Fill ESEG in the header. */
+ md = pkts[0]->hash.fdir.hi;
_mm_store_si128(t_wqe + 1,
_mm_set_epi8(0, 0, 0, 0,
- 0, 0, 0, 0,
+ md, md >> 8, md >> 16, md >> 24,
0, 0, 0, cs_flags,
0, 0, 0, 0));
#ifdef MLX5_PMD_SOFT_COUNTERS
@@ -128,6 +128,12 @@
offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
DEV_TX_OFFLOAD_GRE_TNL_TSO);
}
+
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ if (config->dv_flow_en)
+ offloads |= DEV_TX_OFFLOAD_MATCH_METADATA;
+#endif
+
return offloads;
}