@@ -26,6 +26,7 @@ flags_common_default = [
['RTE_LIBRTE_AVP_PMD', false],
['RTE_SCHED_VECTOR', false],
+ ['RTE_ARM_USE_WFE', false],
]
flags_generic = [
@@ -110,6 +110,11 @@ CONFIG_RTE_MAX_VFIO_CONTAINERS=64
CONFIG_RTE_MALLOC_DEBUG=n
CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
CONFIG_RTE_USE_LIBBSD=n
+# Use WFE instructions to implement the rte_wait_for_equal_xxx APIs,
+# calling these APIs put the cores in low power state while waiting
+# for the memory address to become equal to the expected value.
+# This is supported only by aarch64.
+CONFIG_RTE_ARM_USE_WFE=n
#
# Recognize/ignore the AVX/AVX512 CPU flags for performance/power testing.
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2017 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
*/
#ifndef _RTE_PAUSE_ARM64_H_
@@ -10,6 +11,11 @@ extern "C" {
#endif
#include <rte_common.h>
+
+#ifdef RTE_ARM_USE_WFE
+#define RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
+#endif
+
#include "generic/rte_pause.h"
static inline void rte_pause(void)
@@ -17,6 +23,165 @@ static inline void rte_pause(void)
asm volatile("yield" ::: "memory");
}
+#ifdef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
+/**
+ * Send an event to quit WFE.
+ */
+static inline void _sevl(void)
+{
+ asm volatile("sevl" : : : "memory");
+}
+
+/**
+ * Put processor into low power WFE(Wait For Event) state
+ */
+static inline void _wfe(void)
+{
+ asm volatile("wfe" : : : "memory");
+}
+
+/**
+ * Atomic exclusive load from addr, it returns the 16-bit content of *addr
+ * while making it 'monitored',when it is written by someone else, the
+ * 'monitored' state is cleared and a event is generated implicitly to exit
+ * WFE.
+ *
+ * @param addr
+ * A pointer to the memory location.
+ * @param memorder
+ * The valid memory order variants are __ATOMIC_ACQUIRE and __ATOMIC_RELAXED.
+ * These map to C++11 memory orders with the same names, see the C++11 standard
+ * the GCC wiki on atomic synchronization for detailed definitions.
+ */
+static __rte_always_inline uint16_t
+__atomic_load_ex_16(volatile uint16_t *addr, int memorder);
+
+/**
+ * Atomic exclusive load from addr, it returns the 32-bit content of *addr
+ * while making it 'monitored',when it is written by someone else, the
+ * 'monitored' state is cleared and a event is generated implicitly to exit
+ * WFE.
+ *
+ * @param addr
+ * A pointer to the memory location.
+ * @param memorder
+ * The valid memory order variants are __ATOMIC_ACQUIRE and __ATOMIC_RELAXED.
+ * These map to C++11 memory orders with the same names, see the C++11 standard
+ * the GCC wiki on atomic synchronization for detailed definitions.
+ */
+static __rte_always_inline uint32_t
+__atomic_load_ex_32(volatile uint32_t *addr, int memorder);
+
+/**
+ * Atomic exclusive load from addr, it returns the 64-bit content of *addr
+ * while making it 'monitored',when it is written by someone else, the
+ * 'monitored' state is cleared and a event is generated implicitly to exit
+ * WFE.
+ *
+ * @param addr
+ * A pointer to the memory location.
+ * @param memorder
+ * The valid memory order variants are __ATOMIC_ACQUIRE and __ATOMIC_RELAXED.
+ * These map to C++11 memory orders with the same names, see the C++11 standard
+ * the GCC wiki on atomic synchronization for detailed definitions.
+ */
+static __rte_always_inline uint64_t
+__atomic_load_ex_64(volatile uint64_t *addr, int memorder);
+
+static __rte_always_inline uint16_t
+__atomic_load_ex_16(volatile uint16_t *addr, int memorder)
+{
+ uint16_t tmp;
+ assert((memorder == __ATOMIC_ACQUIRE)
+ || (memorder == __ATOMIC_RELAXED));
+ if (memorder == __ATOMIC_ACQUIRE)
+ asm volatile("ldaxrh %w[tmp], [%x[addr]]"
+ : [tmp] "=&r" (tmp)
+ : [addr] "r"(addr)
+ : "memory");
+ else if (memorder == __ATOMIC_RELAXED)
+ asm volatile("ldxrh %w[tmp], [%x[addr]]"
+ : [tmp] "=&r" (tmp)
+ : [addr] "r"(addr)
+ : "memory");
+ return tmp;
+}
+
+static __rte_always_inline uint32_t
+__atomic_load_ex_32(volatile uint32_t *addr, int memorder)
+{
+ uint32_t tmp;
+ assert((memorder == __ATOMIC_ACQUIRE)
+ || (memorder == __ATOMIC_RELAXED));
+ if (memorder == __ATOMIC_ACQUIRE)
+ asm volatile("ldaxr %w[tmp], [%x[addr]]"
+ : [tmp] "=&r" (tmp)
+ : [addr] "r"(addr)
+ : "memory");
+ else if (memorder == __ATOMIC_RELAXED)
+ asm volatile("ldxr %w[tmp], [%x[addr]]"
+ : [tmp] "=&r" (tmp)
+ : [addr] "r"(addr)
+ : "memory");
+ return tmp;
+}
+
+static __rte_always_inline uint64_t
+__atomic_load_ex_64(volatile uint64_t *addr, int memorder)
+{
+ uint64_t tmp;
+ assert((memorder == __ATOMIC_ACQUIRE)
+ || (memorder == __ATOMIC_RELAXED));
+ if (memorder == __ATOMIC_ACQUIRE)
+ asm volatile("ldaxr %x[tmp], [%x[addr]]"
+ : [tmp] "=&r" (tmp)
+ : [addr] "r"(addr)
+ : "memory");
+ else if (memorder == __ATOMIC_RELAXED)
+ asm volatile("ldxr %x[tmp], [%x[addr]]"
+ : [tmp] "=&r" (tmp)
+ : [addr] "r"(addr)
+ : "memory");
+ return tmp;
+}
+
+static __rte_always_inline void
+rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
+int memorder)
+{
+ if (__atomic_load_ex_16(addr, memorder) != expected) {
+ _sevl();
+ do {
+ _wfe();
+ } while (__atomic_load_ex_16(addr, memorder) != expected);
+ }
+}
+
+static __rte_always_inline void
+rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
+int memorder)
+{
+ if (__atomic_load_ex_32(addr, memorder) != expected) {
+ _sevl();
+ do {
+ _wfe();
+ } while (__atomic_load_ex_32(addr, memorder) != expected);
+ }
+}
+
+static __rte_always_inline void
+rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
+int memorder)
+{
+ if (__atomic_load_ex_64(addr, memorder) != expected) {
+ _sevl();
+ do {
+ _wfe();
+ } while (__atomic_load_ex_64(addr, memorder) != expected);
+ }
+}
+#endif
+
#ifdef __cplusplus
}
#endif
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2017 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
*/
#ifndef _RTE_PAUSE_H_
@@ -12,6 +13,12 @@
*
*/
+#include <stdint.h>
+#include <assert.h>
+#include <rte_common.h>
+#include <rte_atomic.h>
+#include <rte_compat.h>
+
/**
* Pause CPU execution for a short while
*
@@ -20,4 +27,96 @@
*/
static inline void rte_pause(void);
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Wait for *addr to be updated with a 16-bit expected value, with a relaxed
+ * memory ordering model meaning the loads around this API can be reordered.
+ *
+ * @param addr
+ * A pointer to the memory location.
+ * @param expected
+ * A 16-bit expected value to be in the memory location.
+ * @param memorder
+ * Two different memory orders that can be specified:
+ * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ * C++11 memory orders with the same names, see the C++11 standard or
+ * the GCC wiki on atomic synchronization for detailed definition.
+ */
+__rte_experimental
+static __rte_always_inline void
+rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
+int memorder);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Wait for *addr to be updated with a 32-bit expected value, with a relaxed
+ * memory ordering model meaning the loads around this API can be reordered.
+ *
+ * @param addr
+ * A pointer to the memory location.
+ * @param expected
+ * A 32-bit expected value to be in the memory location.
+ * @param memorder
+ * Two different memory orders that can be specified:
+ * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ * C++11 memory orders with the same names, see the C++11 standard or
+ * the GCC wiki on atomic synchronization for detailed definition.
+ */
+__rte_experimental
+static __rte_always_inline void
+rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
+int memorder);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Wait for *addr to be updated with a 64-bit expected value, with a relaxed
+ * memory ordering model meaning the loads around this API can be reordered.
+ *
+ * @param addr
+ * A pointer to the memory location.
+ * @param expected
+ * A 64-bit expected value to be in the memory location.
+ * @param memorder
+ * Two different memory orders that can be specified:
+ * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ * C++11 memory orders with the same names, see the C++11 standard or
+ * the GCC wiki on atomic synchronization for detailed definition.
+ */
+__rte_experimental
+static __rte_always_inline void
+rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
+int memorder);
+
+#ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
+static __rte_always_inline void
+rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
+int memorder)
+{
+ while (__atomic_load_n(addr, memorder) != expected)
+ rte_pause();
+}
+
+static __rte_always_inline void
+rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
+int memorder)
+{
+ while (__atomic_load_n(addr, memorder) != expected)
+ rte_pause();
+}
+
+static __rte_always_inline void
+rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
+int memorder)
+{
+ while (__atomic_load_n(addr, memorder) != expected)
+ rte_pause();
+}
+#endif
+
#endif /* _RTE_PAUSE_H_ */