[dpdk-dev] [PATCH v2] pci_vfio: Support 64KB kernel page_size with vfio-pci driver

[tone.zhang]

With a larger PAGE_SIZE it is possible for the MSI table to very
close to the end of the BAR s.t. when we align the start and end
of the MSI table to the PAGE_SIZE, the end offset of the MSI
table is out of the PCI BAR boundary.

This patch addresses the issue by comparing both the start and the
end offset of the MSI table with the BAR size, and skip the mapping
if it is out of Bar scope.

The patch fixes the debug log as below:
EAL: Skipping BAR0

Signed-off-by: tone.zhang <tone.zhang at arm.com>
Reviewed-by: Gavin Hu <Gavin.Hu at arm.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli at arm.com>
Reviewed-by: Steve Capper <Steve.Capper at arm.com>
Reviewed-by: Burakov Anatoly <anatoly.burakov at intel.com>
---
 drivers/bus/pci/linux/pci_vfio.c | 36 +++++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 305cc06..9a0affe 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -445,9 +445,11 @@ pci_vfio_mmap_bar(int vfio_dev_fd, struct mapped_pci_resource *vfio_res,
 	struct pci_msix_table *msix_table = &vfio_res->msix_table;
 	struct pci_map *bar = &vfio_res->maps[bar_index];
 
-	if (bar->size == 0)
+	if (bar->size == 0) {
 		/* Skip this BAR */
+		RTE_LOG(INFO, EAL, "Skipping BAR%d\n", bar_index);
 		return 0;
+	}
 
 	if (msix_table->bar_index == bar_index) {
 		/*
@@ -456,8 +458,22 @@ pci_vfio_mmap_bar(int vfio_dev_fd, struct mapped_pci_resource *vfio_res,
 		 */
 		uint32_t table_start = msix_table->offset;
 		uint32_t table_end = table_start + msix_table->size;
-		table_end = (table_end + ~PAGE_MASK) & PAGE_MASK;
-		table_start &= PAGE_MASK;
+		table_end = RTE_ALIGN(table_end, PAGE_SIZE);
+		table_start = RTE_ALIGN(table_start, PAGE_SIZE);
+		/* after rounding to PAGE_SIZE, it is over the bar->size,
+		 * fall back to the MSI-X table offset in the bar and
+		 * align with PAGE_SIZE.
+		 */
+		if (table_start >= bar->size) {
+			table_start = RTE_ALIGN_FLOOR(msix_table->offset,
+							PAGE_SIZE);
+			/* after aligning with PAGE_SIZE, if it is less than
+			 * the MSI-X table offset, continue falling back to
+			 * the actual MSI-X table offset in the bar. 
+			 */
+			if (table_start < msix_table->offset)
+				table_start = msix_table->offset;
+		}
 
 		if (table_start == 0 && table_end >= bar->size) {
 			/* Cannot map this BAR */
@@ -469,8 +485,18 @@ pci_vfio_mmap_bar(int vfio_dev_fd, struct mapped_pci_resource *vfio_res,
 
 		memreg[0].offset = bar->offset;
 		memreg[0].size = table_start;
-		memreg[1].offset = bar->offset + table_end;
-		memreg[1].size = bar->size - table_end;
+		if (bar->size < table_end) {
+			/*
+			 * after rounding to PAGE_SIZE we don't have any space
+			 * left after the MSI table, so don't try and map it.
+			 */
+			memreg[1].offset = 0;
+			memreg[1].size = 0;
+		}
+		else {
+			memreg[1].offset = bar->offset + table_end;
+			memreg[1].size = bar->size - table_end;
+		}
 
 		RTE_LOG(DEBUG, EAL,
 			"Trying to map BAR%d that contains the MSI-X "
-- 
2.7.4