@@ -1781,7 +1781,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
priv->drop_queue.hrxq = mlx5_drop_action_create(eth_dev);
if (!priv->drop_queue.hrxq)
goto error;
- /* Supported Verbs flow priority number detection. */
err = mlx5_flow_discover_priorities(eth_dev);
if (err < 0) {
err = -err;
@@ -9362,3 +9362,101 @@ mlx5_dbg__print_pattern(const struct rte_flow_item *item)
}
printf("END\n");
}
+
+/* Map of Verbs to Flow priority with 8 Verbs priorities. */
+static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
+ { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
+};
+
+/* Map of Verbs to Flow priority with 16 Verbs priorities. */
+static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
+ { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
+ { 9, 10, 11 }, { 12, 13, 14 },
+};
+
+/**
+ * Discover the number of available flow priorities.
+ *
+ * @param dev
+ * Ethernet device.
+ *
+ * @return
+ * On success, number of available flow priorities.
+ * On failure, a negative errno-style code and rte_errno is set.
+ */
+int
+mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
+{
+ static const uint16_t vprio[] = {8, 16};
+ const struct mlx5_priv *priv = dev->data->dev_private;
+ const struct mlx5_flow_driver_ops *fops;
+ enum mlx5_flow_drv_type type;
+ int ret;
+
+ type = mlx5_flow_os_get_type();
+ if (type == MLX5_FLOW_TYPE_MAX) {
+ type = MLX5_FLOW_TYPE_VERBS;
+ if (priv->config.devx && priv->config.dv_flow_en)
+ type = MLX5_FLOW_TYPE_DV;
+ }
+ fops = flow_get_drv_ops(type);
+ if (fops->discover_priorities == NULL) {
+ DRV_LOG(ERR, "Priority discovery not supported");
+ rte_errno = ENOTSUP;
+ return -rte_errno;
+ }
+ ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio));
+ if (ret < 0)
+ return ret;
+ switch (ret) {
+ case 8:
+ ret = RTE_DIM(priority_map_3);
+ break;
+ case 16:
+ ret = RTE_DIM(priority_map_5);
+ break;
+ default:
+ rte_errno = ENOTSUP;
+ DRV_LOG(ERR,
+ "port %u maximum priority: %d expected 8/16",
+ dev->data->port_id, ret);
+ return -rte_errno;
+ }
+ DRV_LOG(INFO, "port %u supported flow priorities:"
+ " 0-%d for ingress or egress root table,"
+ " 0-%d for non-root table or transfer root table.",
+ dev->data->port_id, ret - 2,
+ MLX5_NON_ROOT_FLOW_MAX_PRIO - 1);
+ return ret;
+}
+
+/**
+ * Adjust flow priority based on the highest layer and the request priority.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[in] priority
+ * The rule base priority.
+ * @param[in] subpriority
+ * The priority based on the items.
+ *
+ * @return
+ * The new priority.
+ */
+uint32_t
+mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
+ uint32_t subpriority)
+{
+ uint32_t res = 0;
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ switch (priv->config.flow_prio) {
+ case RTE_DIM(priority_map_3):
+ res = priority_map_3[priority][subpriority];
+ break;
+ case RTE_DIM(priority_map_5):
+ res = priority_map_5[priority][subpriority];
+ break;
+ }
+ return res;
+}
@@ -1229,6 +1229,9 @@ typedef int (*mlx5_flow_create_def_policy_t)
(struct rte_eth_dev *dev);
typedef void (*mlx5_flow_destroy_def_policy_t)
(struct rte_eth_dev *dev);
+typedef int (*mlx5_flow_discover_priorities_t)
+ (struct rte_eth_dev *dev,
+ const uint16_t *vprio, int vprio_n);
struct mlx5_flow_driver_ops {
mlx5_flow_validate_t validate;
@@ -1263,6 +1266,7 @@ struct mlx5_flow_driver_ops {
mlx5_flow_action_update_t action_update;
mlx5_flow_action_query_t action_query;
mlx5_flow_sync_domain_t sync_domain;
+ mlx5_flow_discover_priorities_t discover_priorities;
};
/* mlx5_flow.c */
@@ -17828,6 +17828,108 @@ flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
return 0;
}
+/**
+ * Discover the number of available flow priorities
+ * by trying to create a flow with the highest priority value
+ * for each possible number.
+ *
+ * @param[in] dev
+ * Ethernet device.
+ * @param[in] vprio
+ * List of possible number of available priorities.
+ * @param[in] vprio_n
+ * Size of @p vprio array.
+ * @return
+ * On success, number of available flow priorities.
+ * On failure, a negative errno-style code and rte_errno is set.
+ */
+static int
+flow_dv_discover_priorities(struct rte_eth_dev *dev,
+ const uint16_t *vprio, int vprio_n)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_indexed_pool *pool = priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW];
+ struct rte_flow_item_eth eth;
+ struct rte_flow_item item = {
+ .type = RTE_FLOW_ITEM_TYPE_ETH,
+ .spec = ð,
+ .mask = ð,
+ };
+ struct mlx5_flow_dv_matcher matcher = {
+ .mask = {
+ .size = sizeof(matcher.mask.buf),
+ },
+ };
+ union mlx5_flow_tbl_key tbl_key;
+ struct mlx5_flow flow;
+ void *action;
+ struct rte_flow_error error;
+ uint8_t misc_mask;
+ int i, err, ret = -ENOTSUP;
+
+ /*
+ * Prepare a flow with a catch-all pattern and a drop action.
+ * Use drop queue, because shared drop action may be unavailable.
+ */
+ action = priv->drop_queue.hrxq->action;
+ if (action == NULL) {
+ DRV_LOG(ERR, "Priority discovery requires a drop action");
+ rte_errno = ENOTSUP;
+ return -rte_errno;
+ }
+ memset(&flow, 0, sizeof(flow));
+ flow.handle = mlx5_ipool_zmalloc(pool, &flow.handle_idx);
+ if (flow.handle == NULL) {
+ DRV_LOG(ERR, "Cannot create flow handle");
+ rte_errno = ENOMEM;
+ return -rte_errno;
+ }
+ flow.ingress = true;
+ flow.dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
+ flow.dv.actions[0] = action;
+ flow.dv.actions_n = 1;
+ memset(ð, 0, sizeof(eth));
+ flow_dv_translate_item_eth(matcher.mask.buf, flow.dv.value.buf,
+ &item, /* inner */ false, /* group */ 0);
+ matcher.crc = rte_raw_cksum(matcher.mask.buf, matcher.mask.size);
+ for (i = 0; i < vprio_n; i++) {
+ /* Configure the next proposed maximum priority. */
+ matcher.priority = vprio[i] - 1;
+ memset(&tbl_key, 0, sizeof(tbl_key));
+ err = flow_dv_matcher_register(dev, &matcher, &tbl_key, &flow,
+ /* tunnel */ NULL,
+ /* group */ 0,
+ &error);
+ if (err != 0) {
+ /* This action is pure SW and must always succeed. */
+ DRV_LOG(ERR, "Cannot register matcher");
+ ret = -rte_errno;
+ break;
+ }
+ /* Try to apply the flow to HW. */
+ misc_mask = flow_dv_matcher_enable(flow.dv.value.buf);
+ __flow_dv_adjust_buf_size(&flow.dv.value.size, misc_mask);
+ err = mlx5_flow_os_create_flow
+ (flow.handle->dvh.matcher->matcher_object,
+ (void *)&flow.dv.value, flow.dv.actions_n,
+ flow.dv.actions, &flow.handle->drv_flow);
+ if (err == 0) {
+ claim_zero(mlx5_flow_os_destroy_flow
+ (flow.handle->drv_flow));
+ flow.handle->drv_flow = NULL;
+ }
+ claim_zero(flow_dv_matcher_release(dev, flow.handle));
+ if (err != 0)
+ break;
+ ret = vprio[i];
+ }
+ mlx5_ipool_free(pool, flow.handle_idx);
+ /* Set rte_errno if no expected priority value matched. */
+ if (ret < 0)
+ rte_errno = -ret;
+ return ret;
+}
+
const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
.validate = flow_dv_validate,
.prepare = flow_dv_prepare,
@@ -17861,6 +17963,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
.action_update = flow_dv_action_update,
.action_query = flow_dv_action_query,
.sync_domain = flow_dv_sync_domain,
+ .discover_priorities = flow_dv_discover_priorities,
};
#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
@@ -28,17 +28,6 @@
#define VERBS_SPEC_INNER(item_flags) \
(!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
-/* Map of Verbs to Flow priority with 8 Verbs priorities. */
-static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
- { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
-};
-
-/* Map of Verbs to Flow priority with 16 Verbs priorities. */
-static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
- { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
- { 9, 10, 11 }, { 12, 13, 14 },
-};
-
/* Verbs specification header. */
struct ibv_spec_header {
enum ibv_flow_spec_type type;
@@ -50,13 +39,17 @@ struct ibv_spec_header {
*
* @param[in] dev
* Pointer to the Ethernet device structure.
- *
+ * @param[in] vprio
+ * Expected result variants.
+ * @param[in] vprio_n
+ * Number of entries in @p vprio array.
* @return
- * number of supported flow priority on success, a negative errno
+ * Number of supported flow priority on success, a negative errno
* value otherwise and rte_errno is set.
*/
-int
-mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
+static int
+flow_verbs_discover_priorities(struct rte_eth_dev *dev,
+ const uint16_t *vprio, int vprio_n)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct {
@@ -79,7 +72,6 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
};
struct ibv_flow *flow;
struct mlx5_hrxq *drop = priv->drop_queue.hrxq;
- uint16_t vprio[] = { 8, 16 };
int i;
int priority = 0;
@@ -87,7 +79,7 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
rte_errno = ENOTSUP;
return -rte_errno;
}
- for (i = 0; i != RTE_DIM(vprio); i++) {
+ for (i = 0; i != vprio_n; i++) {
flow_attr.attr.priority = vprio[i] - 1;
flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
if (!flow)
@@ -95,59 +87,9 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
claim_zero(mlx5_glue->destroy_flow(flow));
priority = vprio[i];
}
- switch (priority) {
- case 8:
- priority = RTE_DIM(priority_map_3);
- break;
- case 16:
- priority = RTE_DIM(priority_map_5);
- break;
- default:
- rte_errno = ENOTSUP;
- DRV_LOG(ERR,
- "port %u verbs maximum priority: %d expected 8/16",
- dev->data->port_id, priority);
- return -rte_errno;
- }
- DRV_LOG(INFO, "port %u supported flow priorities:"
- " 0-%d for ingress or egress root table,"
- " 0-%d for non-root table or transfer root table.",
- dev->data->port_id, priority - 2,
- MLX5_NON_ROOT_FLOW_MAX_PRIO - 1);
return priority;
}
-/**
- * Adjust flow priority based on the highest layer and the request priority.
- *
- * @param[in] dev
- * Pointer to the Ethernet device structure.
- * @param[in] priority
- * The rule base priority.
- * @param[in] subpriority
- * The priority based on the items.
- *
- * @return
- * The new priority.
- */
-uint32_t
-mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
- uint32_t subpriority)
-{
- uint32_t res = 0;
- struct mlx5_priv *priv = dev->data->dev_private;
-
- switch (priv->config.flow_prio) {
- case RTE_DIM(priority_map_3):
- res = priority_map_3[priority][subpriority];
- break;
- case RTE_DIM(priority_map_5):
- res = priority_map_5[priority][subpriority];
- break;
- }
- return res;
-}
-
/**
* Get Verbs flow counter by index.
*
@@ -2093,4 +2035,5 @@ const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
.destroy = flow_verbs_destroy,
.query = flow_verbs_query,
.sync_domain = flow_verbs_sync_domain,
+ .discover_priorities = flow_verbs_discover_priorities,
};