[dpdk-dev] [PATCH] Patch introducing API to read/write Intel Architecture Model Specific Registers (MSR), rte_msr_read and rte_msr_write functions.

[Wojciech Andralojc]

There is work in progress to implement Intel Cache Allocation Technology (CAT) support in DPDK, this technology is programmed through MSRs.
In the future it will be possible to program CAT through Linux cgroups and DPDK CAT implementation will take advantage of it.

MSR R/W's are privileged ring 0 operations and they must be done in kernel space. For this reason implementation utilizes Linux MSR driver.

Signed-off-by: Wojciech Andralojc <wojciechx.andralojc at intel.com>
---
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/include/arch/arm/rte_msr.h   |  65 ++++++++++
 .../common/include/arch/ppc_64/rte_msr.h           |  65 ++++++++++
 lib/librte_eal/common/include/arch/tile/rte_msr.h  |  65 ++++++++++
 lib/librte_eal/common/include/arch/x86/rte_msr.h   | 143 +++++++++++++++++++++
 lib/librte_eal/common/include/generic/rte_msr.h    |  78 +++++++++++
 lib/librte_eal/common/include/rte_lcore.h          |  18 +++
 7 files changed, 435 insertions(+)
 create mode 100644 lib/librte_eal/common/include/arch/arm/rte_msr.h
 create mode 100644 lib/librte_eal/common/include/arch/ppc_64/rte_msr.h
 create mode 100644 lib/librte_eal/common/include/arch/tile/rte_msr.h
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_msr.h
 create mode 100644 lib/librte_eal/common/include/generic/rte_msr.h

diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index f5ea0ee..567c206 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -48,6 +48,7 @@ endif
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
 GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
+GENERIC_INC += rte_msr.h
 # defined in mk/arch/$(RTE_ARCH)/rte.vars.mk
 ARCH_DIR ?= $(RTE_ARCH)
 ARCH_INC := $(notdir $(wildcard $(RTE_SDK)/lib/librte_eal/common/include/arch/$(ARCH_DIR)/*.h))
diff --git a/lib/librte_eal/common/include/arch/arm/rte_msr.h b/lib/librte_eal/common/include/arch/arm/rte_msr.h
new file mode 100644
index 0000000..85c009c
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/arm/rte_msr.h
@@ -0,0 +1,65 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MSR_ARM_H_
+#define _RTE_MSR_ARM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_msr.h"
+
+/* Function to read CPU's MSR */
+static inline int
+rte_msr_read(__attribute__((unused)) const unsigned lcore,
+		__attribute__((unused)) const uint32_t reg,
+		__attribute__((unused)) uint64_t *value)
+{
+	return -1;
+}
+
+/* Function to write CPU's MSR */
+static inline int
+rte_msr_write(__attribute__((unused)) const unsigned lcore,
+		__attribute__((unused)) const uint32_t reg,
+		__attribute__((unused)) const uint64_t value)
+{
+	return -1;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_MSR_ARM_H_ */
diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_msr.h b/lib/librte_eal/common/include/arch/ppc_64/rte_msr.h
new file mode 100644
index 0000000..44f3de2
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/ppc_64/rte_msr.h
@@ -0,0 +1,65 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MSR_PPC_64_H_
+#define _RTE_MSR_PPC_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_msr.h"
+
+/* Function to read CPU's MSR */
+static inline int
+rte_msr_read(__attribute__((unused)) const unsigned lcore,
+		__attribute__((unused)) const uint32_t reg,
+		__attribute__((unused)) uint64_t *value)
+{
+	return -1;
+}
+
+/* Function to write CPU's MSR */
+static inline int
+rte_msr_write(__attribute__((unused)) const unsigned lcore,
+		__attribute__((unused)) const uint32_t reg,
+		__attribute__((unused)) const uint64_t value)
+{
+	return -1;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_MSR_PPC_64_H_ */
diff --git a/lib/librte_eal/common/include/arch/tile/rte_msr.h b/lib/librte_eal/common/include/arch/tile/rte_msr.h
new file mode 100644
index 0000000..8a446f7
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/tile/rte_msr.h
@@ -0,0 +1,65 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MSR_TILE_H_
+#define _RTE_MSR_TILE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_msr.h"
+
+/* Function to read CPU's MSR */
+static inline int
+rte_msr_read(__attribute__((unused)) const unsigned lcore,
+		__attribute__((unused)) const uint32_t reg,
+		__attribute__((unused)) uint64_t *value)
+{
+	return -1;
+}
+
+/* Function to write CPU's MSR */
+static inline int
+rte_msr_write(__attribute__((unused)) const unsigned lcore,
+		__attribute__((unused)) const uint32_t reg,
+		__attribute__((unused)) const uint64_t value)
+{
+	return -1;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_MSR_TILE_H_ */
diff --git a/lib/librte_eal/common/include/arch/x86/rte_msr.h b/lib/librte_eal/common/include/arch/x86/rte_msr.h
new file mode 100644
index 0000000..7632208
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/x86/rte_msr.h
@@ -0,0 +1,143 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MSR_X86_64_H_
+#define _RTE_MSR_X86_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <fcntl.h> /* O_RDONLY */
+#include <unistd.h> /* pread */
+
+#include <rte_debug.h>
+#include <rte_lcore.h>
+#include <rte_log.h>
+
+#include "generic/rte_msr.h"
+
+#define CPU_MSR_PATH "/dev/cpu/%u/msr"
+#define CPU_MSR_PATH_MAX_LEN 32
+
+/**
+ * This function should not be called directly.
+ * Function to open CPU's MSR file
+ */
+static int
+__msr_open_file(const unsigned lcore, int flags)
+{
+	RTE_VERIFY(rte_lcore_is_detected(lcore) == 1);
+	if (rte_lcore_is_detected(lcore) != 1)
+		return -1;
+
+	int fd = -1;
+
+	char fname[CPU_MSR_PATH_MAX_LEN];
+
+	memset(fname, 0, sizeof(fname));
+	snprintf(fname, sizeof(fname)-1, CPU_MSR_PATH, lcore);
+
+	fd = open(fname, flags);
+
+	if (fd < 0)
+		RTE_LOG(ERR, PQOS, "Error opening file '%s'!\n", fname);
+
+	return fd;
+}
+
+/* Function to read CPU's MSR */
+static inline int
+rte_msr_read(const unsigned lcore, const uint32_t reg, uint64_t *value)
+{
+	RTE_VERIFY(value != NULL);
+	RTE_VERIFY(rte_lcore_is_detected(lcore) == 1);
+	if (value == NULL || rte_lcore_is_detected(lcore) != 1)
+		return -1;
+
+	int ret = -1;
+	int fd = -1;
+
+	fd = __msr_open_file(lcore, O_RDONLY);
+
+	if (fd >= 0) {
+		ssize_t read_ret = 0;
+
+		read_ret = pread(fd, value, sizeof(value[0]), (off_t)reg);
+
+		if (read_ret != sizeof(value[0])) {
+			RTE_LOG(ERR, PQOS, "RDMSR failed for reg[0x%x] on lcore %u\n",
+				(unsigned)reg, lcore);
+		} else
+			ret = 0;
+
+		close(fd);
+	}
+
+	return ret;
+}
+
+/* Function to write CPU's MSR */
+static inline int
+rte_msr_write(const unsigned lcore, const uint32_t reg, const uint64_t value)
+{
+	RTE_VERIFY(rte_lcore_is_detected(lcore) == 1);
+	if (rte_lcore_is_detected(lcore) != 1)
+		return -1;
+
+	int ret = -1;
+	int fd = -1;
+
+	fd = __msr_open_file(lcore, O_WRONLY);
+
+	if (fd >= 0) {
+		ssize_t write_ret = 0;
+
+		write_ret = pwrite(fd, &value, sizeof(value), (off_t)reg);
+		if (write_ret != sizeof(value)) {
+			RTE_LOG(ERR, PQOS, "WRMSR failed for reg[0x%x] <- value[0x%llx] on "
+					"lcore %u\n", (unsigned)reg, (unsigned long long)value, lcore);
+		} else
+			ret = 0;
+
+		close(fd);
+	}
+
+	return ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_MSR_X86_64_H_ */
diff --git a/lib/librte_eal/common/include/generic/rte_msr.h b/lib/librte_eal/common/include/generic/rte_msr.h
new file mode 100644
index 0000000..62f0530
--- /dev/null
+++ b/lib/librte_eal/common/include/generic/rte_msr.h
@@ -0,0 +1,78 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MSR_H_
+#define _RTE_MSR_H_
+
+/**
+ * @file
+ * Architecture specific API to read/write CPU's MSR.
+ */
+
+#include <stdint.h>
+
+/**
+ * Function to read CPU's MSR
+ *
+ * @param [in] lcore
+ *  CPU logical core id
+ *
+ * @param [in] reg
+ *  MSR reg to read
+ *
+ * @param [out] value
+ *  Read value of MSR reg
+ *
+ * @return
+ *  Operations status
+ */
+static inline int rte_msr_read(const unsigned lcore, const uint32_t reg, uint64_t *value);
+
+/**
+ * Function to write CPU's MSR
+ *
+ * @param [in] lcore
+ *  CPU logical core id
+ *
+ * @param [in] reg
+ *  MSR reg to write
+ *
+ * @param [out] value
+ *  Value to be written to MSR reg
+ *
+ * @return
+ *  Operations status
+ */
+static inline int rte_msr_write(const unsigned lcore, const uint32_t reg, const uint64_t value);
+
+#endif /* _RTE_MSR_H_ */
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index 25460b9..1b72d36 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -268,6 +268,24 @@ void rte_thread_get_affinity(rte_cpuset_t *cpusetp);
 #endif
 #endif
 
+/**
+ * Test if lcore is detected lcore in system.
+ *
+ * @param lcore_id
+ *   The identifier of the lcore, which MUST be between 0 and
+ *   RTE_MAX_LCORE-1.
+ * @return
+ *   True if the given lcore is detected; false otherwise.
+ */
+static inline unsigned
+rte_lcore_is_detected(const unsigned lcore_id)
+{
+    if (lcore_id >= RTE_MAX_LCORE)
+	return 0;
+
+    return lcore_config[lcore_id].detected;
+}
+
 #ifdef __cplusplus
 }
 #endif
-- 
1.9.3

--------------------------------------------------------------
Intel Research and Development Ireland Limited
Registered in Ireland
Registered Office: Collinstown Industrial Park, Leixlip, County Kildare
Registered Number: 308263


This e-mail and any attachments may contain confidential material for the sole
use of the intended recipient(s). Any review or distribution by others is
strictly prohibited. If you are not the intended recipient, please contact the
sender and delete all copies.