patch 'vdpa/mlx5: workaround dirty bitmap MR creation' has been queued to stable release 20.11.4

[Xueming Li]

Hi,

FYI, your patch has been queued to stable release 20.11.4

Note it hasn't been pushed to http://dpdk.org/browse/dpdk-stable yet.
It will be pushed if I get no objections before 11/30/21. So please
shout if anyone has objections.

Also note that after the patch there's a diff of the upstream commit vs the
patch applied to the branch. This will indicate if there was any rebasing
needed to apply to the stable branch. If there were code changes for rebasing
(ie: not only metadata diffs), please double check that the rebase was
correctly done.

Queued patches are on a temporary branch at:
https://github.com/steevenlee/dpdk

This queued commit can be viewed at:
https://github.com/steevenlee/dpdk/commit/4f691f2c97079a5fe264c85d226a46a08e691cb7

Thanks.

Xueming Li <xuemingl at nvidia.com>

---
>From 4f691f2c97079a5fe264c85d226a46a08e691cb7 Mon Sep 17 00:00:00 2001
From: Matan Azrad <matan at nvidia.com>
Date: Tue, 9 Nov 2021 14:36:10 +0200
Subject: [PATCH] vdpa/mlx5: workaround dirty bitmap MR creation
Cc: Xueming Li <xuemingl at nvidia.com>

[ upstream commit 398ea8450c535a1385c7767d6c60c94e8f1c4c25 ]

Due to kernel driver/FW issues in direct MKEY creation using the DevX
API, this patch replaces the dirty bitmap MR creation to use wrapped
mkey instead.

Fixes: 9d39e57f21ac ("vdpa/mlx5: support live migration")

Signed-off-by: Michael Baum <michaelba at nvidia.com>
Signed-off-by: Matan Azrad <matan at nvidia.com>
---
 drivers/vdpa/mlx5/mlx5_vdpa.h     |  1 +
 drivers/vdpa/mlx5/mlx5_vdpa_lm.c  | 42 ++++++-------------------------
 drivers/vdpa/mlx5/mlx5_vdpa_mem.c |  2 ++
 3 files changed, 11 insertions(+), 34 deletions(-)

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index fefc7cfdad..954fd1e86a 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -160,6 +160,7 @@ struct mlx5_vdpa_priv {
 	struct mlx5_vdpa_steer steer;
 	struct mlx5dv_var *var;
 	void *virtq_db_addr;
+	struct mlx5_pmd_wrapped_mr lm_mr;
 	SLIST_HEAD(mr_list, mlx5_vdpa_query_mr) mr_list;
 	struct mlx5_vdpa_virtq virtqs[];
 };
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
index 6c4284f7f7..49d74a7e99 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
@@ -36,42 +36,22 @@ int
 mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
 			   uint64_t log_size)
 {
-	struct mlx5_devx_mkey_attr mkey_attr = {
-			.addr = (uintptr_t)log_base,
-			.size = log_size,
-			.pd = priv->pdn,
-			.pg_access = 1,
-			.klm_array = NULL,
-			.klm_num = 0,
-			.relaxed_ordering_read = 0,
-			.relaxed_ordering_write = 0,
-	};
 	struct mlx5_devx_virtq_attr attr = {
 		.type = MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS,
 		.dirty_bitmap_addr = log_base,
 		.dirty_bitmap_size = log_size,
 	};
-	struct mlx5_vdpa_query_mr *mr = rte_malloc(__func__, sizeof(*mr), 0);
 	int i;
+	int ret = mlx5_os_wrapped_mkey_create(priv->ctx, priv->pd,
+					      priv->pdn,
+					      (void *)(uintptr_t)log_base,
+					      log_size, &priv->lm_mr);
 
-	if (!mr) {
-		DRV_LOG(ERR, "Failed to allocate mem for lm mr.");
+	if (!ret) {
+		DRV_LOG(ERR, "Failed to allocate wrapped MR for lm.");
 		return -1;
 	}
-	mr->umem = mlx5_glue->devx_umem_reg(priv->ctx,
-					    (void *)(uintptr_t)log_base,
-					    log_size, IBV_ACCESS_LOCAL_WRITE);
-	if (!mr->umem) {
-		DRV_LOG(ERR, "Failed to register umem for lm mr.");
-		goto err;
-	}
-	mkey_attr.umem_id = mr->umem->umem_id;
-	mr->mkey = mlx5_devx_cmd_mkey_create(priv->ctx, &mkey_attr);
-	if (!mr->mkey) {
-		DRV_LOG(ERR, "Failed to create Mkey for lm.");
-		goto err;
-	}
-	attr.dirty_bitmap_mkey = mr->mkey->id;
+	attr.dirty_bitmap_mkey = priv->lm_mr.lkey;
 	for (i = 0; i < priv->nr_virtqs; ++i) {
 		attr.queue_index = i;
 		if (!priv->virtqs[i].virtq) {
@@ -82,15 +62,9 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
 			goto err;
 		}
 	}
-	mr->is_indirect = 0;
-	SLIST_INSERT_HEAD(&priv->mr_list, mr, next);
 	return 0;
 err:
-	if (mr->mkey)
-		mlx5_devx_cmd_destroy(mr->mkey);
-	if (mr->umem)
-		mlx5_glue->devx_umem_dereg(mr->umem);
-	rte_free(mr);
+	mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
 	return -1;
 }
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
index c5ab652a6b..f36ceabaec 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
@@ -31,6 +31,8 @@ mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
 		entry = next;
 	}
 	SLIST_INIT(&priv->mr_list);
+	if (priv->lm_mr.addr)
+		mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
 	if (priv->null_mr) {
 		claim_zero(mlx5_glue->dereg_mr(priv->null_mr));
 		priv->null_mr = NULL;
-- 
2.34.0

---
  Diff of the applied patch vs upstream commit (please double-check if non-empty:
---
--- -	2021-11-28 22:41:03.727391773 +0800
+++ 0003-vdpa-mlx5-workaround-dirty-bitmap-MR-creation.patch	2021-11-28 22:41:03.183543455 +0800
@@ -1 +1 @@
-From 398ea8450c535a1385c7767d6c60c94e8f1c4c25 Mon Sep 17 00:00:00 2001
+From 4f691f2c97079a5fe264c85d226a46a08e691cb7 Mon Sep 17 00:00:00 2001
@@ -4,0 +5,3 @@
+Cc: Xueming Li <xuemingl at nvidia.com>
+
+[ upstream commit 398ea8450c535a1385c7767d6c60c94e8f1c4c25 ]
@@ -11 +13,0 @@
-Cc: stable at dpdk.org
@@ -17 +19 @@
- drivers/vdpa/mlx5/mlx5_vdpa_lm.c  | 38 +++++++------------------------
+ drivers/vdpa/mlx5/mlx5_vdpa_lm.c  | 42 ++++++-------------------------
@@ -19 +21 @@
- 3 files changed, 11 insertions(+), 30 deletions(-)
+ 3 files changed, 11 insertions(+), 34 deletions(-)
@@ -22 +24 @@
-index 62498f87fd..15212a2b30 100644
+index fefc7cfdad..954fd1e86a 100644
@@ -25 +27 @@
-@@ -147,6 +147,7 @@ struct mlx5_vdpa_priv {
+@@ -160,6 +160,7 @@ struct mlx5_vdpa_priv {
@@ -34 +36 @@
-index 3e8d9eb9a2..e65e4faa47 100644
+index 6c4284f7f7..49d74a7e99 100644
@@ -37 +39 @@
-@@ -36,38 +36,22 @@ int
+@@ -36,42 +36,22 @@ int
@@ -44 +46 @@
--			.pd = priv->cdev->pdn,
+-			.pd = priv->pdn,
@@ -45,0 +48,4 @@
+-			.klm_array = NULL,
+-			.klm_num = 0,
+-			.relaxed_ordering_read = 0,
+-			.relaxed_ordering_write = 0,
@@ -54,2 +60,2 @@
-+	int ret = mlx5_os_wrapped_mkey_create(priv->cdev->ctx, priv->cdev->pd,
-+					      priv->cdev->pdn,
++	int ret = mlx5_os_wrapped_mkey_create(priv->ctx, priv->pd,
++					      priv->pdn,
@@ -65 +71 @@
--	mr->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
+-	mr->umem = mlx5_glue->devx_umem_reg(priv->ctx,
@@ -73 +79 @@
--	mr->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &mkey_attr);
+-	mr->mkey = mlx5_devx_cmd_mkey_create(priv->ctx, &mkey_attr);
@@ -83 +89 @@
-@@ -78,15 +62,9 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
+@@ -82,15 +62,9 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
@@ -101 +107 @@
-index f551a094cd..d7707bbd91 100644
+index c5ab652a6b..f36ceabaec 100644