patch 'vdpa/mlx5: workaround guest MR registrations' has been queued to stable release 20.11.4

[Xueming Li]

Hi,

FYI, your patch has been queued to stable release 20.11.4

Note it hasn't been pushed to http://dpdk.org/browse/dpdk-stable yet.
It will be pushed if I get no objections before 11/30/21. So please
shout if anyone has objections.

Also note that after the patch there's a diff of the upstream commit vs the
patch applied to the branch. This will indicate if there was any rebasing
needed to apply to the stable branch. If there were code changes for rebasing
(ie: not only metadata diffs), please double check that the rebase was
correctly done.

Queued patches are on a temporary branch at:
https://github.com/steevenlee/dpdk

This queued commit can be viewed at:
https://github.com/steevenlee/dpdk/commit/2e7f6f9336fe91e94e06274f6cf00410c1a8f55f

Thanks.

Xueming Li <xuemingl at nvidia.com>

---
>From 2e7f6f9336fe91e94e06274f6cf00410c1a8f55f Mon Sep 17 00:00:00 2001
From: Michael Baum <michaelba at nvidia.com>
Date: Tue, 9 Nov 2021 14:36:11 +0200
Subject: [PATCH] vdpa/mlx5: workaround guest MR registrations
Cc: Xueming Li <xuemingl at nvidia.com>

[ upstream commit 04b4e4cbc0352c6323f459d501d7bc0d82b2f1aa ]

Due to kernel issue in direct MKEY creation using the DevX API, this
patch replaces the virtio MR creation to use Verbs API.

Fixes: cc07a42da250 ("vdpa/mlx5: prepare memory regions")

Signed-off-by: Michael Baum <michaelba at nvidia.com>
Signed-off-by: Matan Azrad <matan at nvidia.com>
---
 drivers/vdpa/mlx5/mlx5_vdpa.h     |  8 +++----
 drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 40 ++++++++++---------------------
 2 files changed, 16 insertions(+), 32 deletions(-)

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 954fd1e86a..0ffe16e295 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -67,10 +67,10 @@ struct mlx5_vdpa_event_qp {
 
 struct mlx5_vdpa_query_mr {
 	SLIST_ENTRY(mlx5_vdpa_query_mr) next;
-	void *addr;
-	uint64_t length;
-	struct mlx5dv_devx_umem *umem;
-	struct mlx5_devx_obj *mkey;
+	union {
+		struct ibv_mr *mr;
+		struct mlx5_devx_obj *mkey;
+	};
 	int is_indirect;
 };
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
index f36ceabaec..97a5f1c1ea 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
@@ -23,9 +23,10 @@ mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
 	entry = SLIST_FIRST(&priv->mr_list);
 	while (entry) {
 		next = SLIST_NEXT(entry, next);
-		claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
-		if (!entry->is_indirect)
-			claim_zero(mlx5_glue->devx_umem_dereg(entry->umem));
+		if (entry->is_indirect)
+			claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
+		else
+			claim_zero(mlx5_glue->dereg_mr(entry->mr));
 		SLIST_REMOVE(&priv->mr_list, entry, mlx5_vdpa_query_mr, next);
 		rte_free(entry);
 		entry = next;
@@ -210,31 +211,18 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
 			DRV_LOG(ERR, "Failed to allocate mem entry memory.");
 			goto error;
 		}
-		entry->umem = mlx5_glue->devx_umem_reg(priv->ctx,
-					 (void *)(uintptr_t)reg->host_user_addr,
-					     reg->size, IBV_ACCESS_LOCAL_WRITE);
-		if (!entry->umem) {
-			DRV_LOG(ERR, "Failed to register Umem by Devx.");
-			ret = -errno;
-			goto error;
-		}
-		mkey_attr.addr = (uintptr_t)(reg->guest_phys_addr);
-		mkey_attr.size = reg->size;
-		mkey_attr.umem_id = entry->umem->umem_id;
-		mkey_attr.pd = priv->pdn;
-		mkey_attr.pg_access = 1;
-		mkey_attr.klm_array = NULL;
-		mkey_attr.klm_num = 0;
+		entry->mr = mlx5_glue->reg_mr_iova(priv->pd,
+				       (void *)(uintptr_t)(reg->host_user_addr),
+				       reg->size, reg->guest_phys_addr,
+				       IBV_ACCESS_LOCAL_WRITE);
+		if (!entry->mr) {
 		mkey_attr.relaxed_ordering_read = 0;
 		mkey_attr.relaxed_ordering_write = 0;
 		entry->mkey = mlx5_devx_cmd_mkey_create(priv->ctx, &mkey_attr);
-		if (!entry->mkey) {
 			DRV_LOG(ERR, "Failed to create direct Mkey.");
 			ret = -rte_errno;
 			goto error;
 		}
-		entry->addr = (void *)(uintptr_t)(reg->host_user_addr);
-		entry->length = reg->size;
 		entry->is_indirect = 0;
 		if (i > 0) {
 			uint64_t sadd;
@@ -264,12 +252,13 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
 		for (k = 0; k < reg->size; k += klm_size) {
 			klm_array[klm_index].byte_count = k + klm_size >
 					   reg->size ? reg->size - k : klm_size;
-			klm_array[klm_index].mkey = entry->mkey->id;
+			klm_array[klm_index].mkey = entry->mr->lkey;
 			klm_array[klm_index].address = reg->guest_phys_addr + k;
 			klm_index++;
 		}
 		SLIST_INSERT_HEAD(&priv->mr_list, entry, next);
 	}
+	memset(&mkey_attr, 0, sizeof(mkey_attr));
 	mkey_attr.addr = (uintptr_t)(mem->regions[0].guest_phys_addr);
 	mkey_attr.size = mem_size;
 	mkey_attr.pd = priv->pdn;
@@ -297,13 +286,8 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
 	priv->gpa_mkey_index = entry->mkey->id;
 	return 0;
 error:
-	if (entry) {
-		if (entry->mkey)
-			mlx5_devx_cmd_destroy(entry->mkey);
-		if (entry->umem)
-			mlx5_glue->devx_umem_dereg(entry->umem);
+	if (entry)
 		rte_free(entry);
-	}
 	mlx5_vdpa_mem_dereg(priv);
 	rte_errno = -ret;
 	return ret;
-- 
2.34.0

---
  Diff of the applied patch vs upstream commit (please double-check if non-empty:
---
--- -	2021-11-28 22:41:03.769042316 +0800
+++ 0004-vdpa-mlx5-workaround-guest-MR-registrations.patch	2021-11-28 22:41:03.186876727 +0800
@@ -1 +1 @@
-From 04b4e4cbc0352c6323f459d501d7bc0d82b2f1aa Mon Sep 17 00:00:00 2001
+From 2e7f6f9336fe91e94e06274f6cf00410c1a8f55f Mon Sep 17 00:00:00 2001
@@ -4,0 +5,3 @@
+Cc: Xueming Li <xuemingl at nvidia.com>
+
+[ upstream commit 04b4e4cbc0352c6323f459d501d7bc0d82b2f1aa ]
@@ -10 +12,0 @@
-Cc: stable at dpdk.org
@@ -15,3 +17,3 @@
- drivers/vdpa/mlx5/mlx5_vdpa.h     |  8 +++---
- drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 41 +++++++++----------------------
- 2 files changed, 16 insertions(+), 33 deletions(-)
+ drivers/vdpa/mlx5/mlx5_vdpa.h     |  8 +++----
+ drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 40 ++++++++++---------------------
+ 2 files changed, 16 insertions(+), 32 deletions(-)
@@ -20 +22 @@
-index 15212a2b30..22617924ea 100644
+index 954fd1e86a..0ffe16e295 100644
@@ -23 +25 @@
-@@ -59,10 +59,10 @@ struct mlx5_vdpa_event_qp {
+@@ -67,10 +67,10 @@ struct mlx5_vdpa_event_qp {
@@ -39 +41 @@
-index d7707bbd91..b1b9053bff 100644
+index f36ceabaec..97a5f1c1ea 100644
@@ -56,9 +58 @@
-@@ -202,7 +203,6 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
- 		goto error;
- 	}
- 	DRV_LOG(DEBUG, "Dump fill Mkey = %u.", priv->null_mr->lkey);
--	memset(&mkey_attr, 0, sizeof(mkey_attr));
- 	for (i = 0; i < mem->nregions; i++) {
- 		reg = &mem->regions[i];
- 		entry = rte_zmalloc(__func__, sizeof(*entry), 0);
-@@ -211,28 +211,15 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
+@@ -210,31 +211,18 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
@@ -68 +62 @@
--		entry->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
+-		entry->umem = mlx5_glue->devx_umem_reg(priv->ctx,
@@ -79 +73 @@
--		mkey_attr.pd = priv->cdev->pdn;
+-		mkey_attr.pd = priv->pdn;
@@ -81,4 +75,3 @@
--		entry->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx,
--							&mkey_attr);
--		if (!entry->mkey) {
-+		entry->mr = mlx5_glue->reg_mr_iova(priv->cdev->pd,
+-		mkey_attr.klm_array = NULL;
+-		mkey_attr.klm_num = 0;
++		entry->mr = mlx5_glue->reg_mr_iova(priv->pd,
@@ -88,0 +82,4 @@
+ 		mkey_attr.relaxed_ordering_read = 0;
+ 		mkey_attr.relaxed_ordering_write = 0;
+ 		entry->mkey = mlx5_devx_cmd_mkey_create(priv->ctx, &mkey_attr);
+-		if (!entry->mkey) {
@@ -98 +95 @@
-@@ -262,12 +249,13 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
+@@ -264,12 +252,13 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
@@ -112,2 +109,2 @@
- 	mkey_attr.pd = priv->cdev->pdn;
-@@ -295,13 +283,8 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
+ 	mkey_attr.pd = priv->pdn;
+@@ -297,13 +286,8 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)