[dpdk-stable] patch 'eal: fix control thread affinity with --lcores' has been queued to LTS release 18.11.3

[Kevin Traynor]

Hi,

FYI, your patch has been queued to LTS release 18.11.3

Note it hasn't been pushed to http://dpdk.org/browse/dpdk-stable yet.
It will be pushed if I get no objections before 09/12/19. So please
shout if anyone has objections.

Also note that after the patch there's a diff of the upstream commit vs the
patch applied to the branch. This will indicate if there was any rebasing
needed to apply to the stable branch. If there were code changes for rebasing
(ie: not only metadata diffs), please double check that the rebase was
correctly done.

Queued patches are on a temporary branch at:
https://github.com/kevintraynor/dpdk-stable-queue

This queued commit can be viewed at:
https://github.com/kevintraynor/dpdk-stable-queue/commit/c0238120c05529f0f26505b6196edd53ee37e050

Thanks.

Kevin Traynor

---
>From c0238120c05529f0f26505b6196edd53ee37e050 Mon Sep 17 00:00:00 2001
From: David Marchand <david.marchand at redhat.com>
Date: Tue, 30 Jul 2019 17:05:22 +0200
Subject: [PATCH] eal: fix control thread affinity with --lcores
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

[ upstream commit b6be16acfeb1cc58c7f37638f007e302527657cd ]

The ctrl thread cpu affinity setting has been broken when using --lcores.

Using -l/-c options makes each lcore associated to a physical cpu in a 1:1
fashion.
On the contrary, when using --lcores, each lcore cpu affinity can be set
to a list of any online cpu on the system.

To handle both cases, each lcore cpu affinity is considered and removed
from the process startup cpu affinity.

Introduced macros to manipulate dpdk cpu sets in both Linux and FreeBSD.

Examples on a 8 cores Linux system:

$ cd /sys/fs/cgroup/cpuset/
$ mkdir dpdk
$ cd dpdk
$ echo 4-7 > cpuset.cpus
$ echo 0 > cpuset.mems
$ echo $$ > tasks

Before the fix:
$ ./master/app/testpmd --master-lcore 0 --lcores '(0,7)@(7,4,5)' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

8427	cpu_list=4-5,7	testpmd
8428	cpu_list=4-6	eal-intr-thread
8429	cpu_list=4-6	rte_mp_handle
8430	cpu_list=4-5,7	lcore-slave-7

$ taskset -c 7 \
 ./master/app/testpmd --master-lcore 0 --lcores '(0,7)@(7,4,5)' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

EAL: Detected 8 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Failed to create thread for interrupt handling
EAL: FATAL: Cannot init interrupt-handling thread
EAL: Cannot init interrupt-handling thread
PANIC in main():
Cannot init EAL

After the fix:
$ ./master/app/testpmd --master-lcore 0 --lcores '(0,7)@(7,4,5)' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

15214	cpu_list=4-5,7	testpmd
15215	cpu_list=6	eal-intr-thread
15216	cpu_list=6	rte_mp_handle
15217	cpu_list=4-5,7	lcore-slave-7

$ taskset -c 7 \
 ./master/app/testpmd --master-lcore 0 --lcores '(0,7)@(7,4,5)' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

15297	cpu_list=4-5,7	testpmd
15298	cpu_list=4-5,7	eal-intr-thread
15299	cpu_list=4-5,7	rte_mp_handle
15300	cpu_list=4-5,7	lcore-slave-7

Bugzilla ID: 322
Fixes: c3568ea37670 ("eal: restrict control threads to startup CPU affinity")

Reported-by: Johan Källström <johan.kallstrom at ericsson.com>
Signed-off-by: David Marchand <david.marchand at redhat.com>
---
 lib/librte_eal/common/eal_common_options.c | 16 ++++++------
 lib/librte_eal/common/include/rte_lcore.h  | 29 ++++++++++++++++++++++
 2 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 79efb15d0..f742d4d38 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -1451,9 +1451,9 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
 
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
-		if (eal_cpu_detected(lcore_id) &&
-				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
-			CPU_SET(lcore_id, cpuset);
-		}
+		if (rte_lcore_has_role(lcore_id, ROLE_OFF))
+			continue;
+		RTE_CPU_OR(cpuset, cpuset, &lcore_config[lcore_id].cpuset);
 	}
+	RTE_CPU_NOT(cpuset, cpuset);
 
 	if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
@@ -1463,7 +1463,9 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
 	RTE_CPU_AND(cpuset, cpuset, &default_set);
 
-	/* if no detected CPU is off, use master core */
-	if (!CPU_COUNT(cpuset))
-		CPU_SET(rte_get_master_lcore(), cpuset);
+	/* if no remaining cpu, use master lcore cpu affinity */
+	if (!CPU_COUNT(cpuset)) {
+		memcpy(cpuset, &lcore_config[rte_get_master_lcore()].cpuset,
+			sizeof(*cpuset));
+	}
 }
 
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index dea17f500..31af0e501 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -26,4 +26,18 @@ extern "C" {
 typedef	cpu_set_t rte_cpuset_t;
 #define RTE_CPU_AND(dst, src1, src2) CPU_AND(dst, src1, src2)
+#define RTE_CPU_OR(dst, src1, src2) CPU_OR(dst, src1, src2)
+#define RTE_CPU_FILL(set) do \
+{ \
+	unsigned int i; \
+	CPU_ZERO(set); \
+	for (i = 0; i < CPU_SETSIZE; i++) \
+		CPU_SET(i, set); \
+} while (0)
+#define RTE_CPU_NOT(dst, src) do \
+{ \
+	cpu_set_t tmp; \
+	RTE_CPU_FILL(&tmp); \
+	CPU_XOR(dst, &tmp, src); \
+} while (0)
 #elif defined(__FreeBSD__)
 #include <pthread_np.h>
@@ -36,4 +50,19 @@ typedef cpuset_t rte_cpuset_t;
 	CPU_COPY(&tmp, dst); \
 } while (0)
+#define RTE_CPU_OR(dst, src1, src2) do \
+{ \
+	cpuset_t tmp; \
+	CPU_COPY(src1, &tmp); \
+	CPU_OR(&tmp, src2); \
+	CPU_COPY(&tmp, dst); \
+} while (0)
+#define RTE_CPU_FILL(set) CPU_FILL(set)
+#define RTE_CPU_NOT(dst, src) do \
+{ \
+	cpuset_t tmp; \
+	CPU_FILL(&tmp); \
+	CPU_NAND(&tmp, src); \
+	CPU_COPY(&tmp, dst); \
+} while (0)
 #endif
 
-- 
2.20.1

---
  Diff of the applied patch vs upstream commit (please double-check if non-empty:
---
--- -	2019-09-05 10:36:49.105515044 +0100
+++ 0026-eal-fix-control-thread-affinity-with-lcores.patch	2019-09-05 10:36:47.522700235 +0100
@@ -1 +1 @@
-From b6be16acfeb1cc58c7f37638f007e302527657cd Mon Sep 17 00:00:00 2001
+From c0238120c05529f0f26505b6196edd53ee37e050 Mon Sep 17 00:00:00 2001
@@ -8,0 +9,2 @@
+[ upstream commit b6be16acfeb1cc58c7f37638f007e302527657cd ]
+
@@ -71 +72,0 @@
-Cc: stable at dpdk.org
@@ -81 +82 @@
-index 512d5088e..7b182b82d 100644
+index 79efb15d0..f742d4d38 100644
@@ -84 +85 @@
-@@ -1453,9 +1453,9 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+@@ -1451,9 +1451,9 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
@@ -98 +99 @@
-@@ -1465,7 +1465,9 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+@@ -1463,7 +1463,9 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
@@ -112 +113 @@
-index 411df30d5..c86f72eb1 100644
+index dea17f500..31af0e501 100644