@@ -, +, @@ --- drivers/mempool/bucket/rte_mempool_bucket.c | 1 + drivers/mempool/cnxk/cn10k_mempool_ops.c | 1 + drivers/mempool/cnxk/cn9k_mempool_ops.c | 1 + drivers/mempool/dpaa/dpaa_mempool.c | 1 + drivers/mempool/dpaa2/dpaa2_hw_mempool.c | 1 + drivers/mempool/meson.build | 2 +- .../mempool/octeontx/rte_mempool_octeontx.c | 1 + drivers/mempool/ring/rte_mempool_ring.c | 1 + drivers/mempool/stack/rte_mempool_stack.c | 1 + lib/mempool/meson.build | 4 + lib/mempool/rte_mempool.h | 16 ++- lib/mempool/rte_mempool_driver.h | 32 +++++ lib/mempool/rte_mempool_ops.c | 113 ++++++++++++++++-- lib/mempool/version.map | 3 + 14 files changed, 162 insertions(+), 16 deletions(-) create mode 100644 lib/mempool/rte_mempool_driver.h --- a/drivers/mempool/bucket/rte_mempool_bucket.c +++ a/drivers/mempool/bucket/rte_mempool_bucket.c @@ -14,6 +14,7 @@ #include #include #include +#include #include /* --- a/drivers/mempool/cnxk/cn10k_mempool_ops.c +++ a/drivers/mempool/cnxk/cn10k_mempool_ops.c @@ -3,6 +3,7 @@ */ #include +#include #include "roc_api.h" #include "cnxk_mempool.h" --- a/drivers/mempool/cnxk/cn9k_mempool_ops.c +++ a/drivers/mempool/cnxk/cn9k_mempool_ops.c @@ -3,6 +3,7 @@ */ #include +#include #include "roc_api.h" #include "cnxk_mempool.h" --- a/drivers/mempool/dpaa/dpaa_mempool.c +++ a/drivers/mempool/dpaa/dpaa_mempool.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include --- a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c +++ a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c @@ -22,6 +22,7 @@ #include #include #include "rte_dpaa2_mempool.h" +#include #include "fslmc_vfio.h" #include --- a/drivers/mempool/meson.build +++ a/drivers/mempool/meson.build @@ -11,6 +11,6 @@ drivers = [ 'stack', ] -std_deps = ['mempool'] +std_deps = ['mempool', 'bus_vdev'] log_prefix = 'mempool' --- a/drivers/mempool/octeontx/rte_mempool_octeontx.c +++ a/drivers/mempool/octeontx/rte_mempool_octeontx.c @@ -4,6 +4,7 @@ #include #include +#include #include #include --- a/drivers/mempool/ring/rte_mempool_ring.c +++ a/drivers/mempool/ring/rte_mempool_ring.c @@ -8,6 +8,7 @@ #include #include #include +#include static int common_ring_mp_enqueue(struct rte_mempool *mp, void * const *obj_table, --- a/drivers/mempool/stack/rte_mempool_stack.c +++ a/drivers/mempool/stack/rte_mempool_stack.c @@ -4,6 +4,7 @@ #include #include +#include #include static int --- a/lib/mempool/meson.build +++ a/lib/mempool/meson.build @@ -20,4 +20,8 @@ headers = files( 'rte_mempool_trace.h', 'rte_mempool_trace_fp.h', ) +driver_sdk_headers += files( + 'rte_mempool_driver.h', +) + deps += ['ring', 'telemetry'] --- a/lib/mempool/rte_mempool.h +++ a/lib/mempool/rte_mempool.h @@ -684,7 +684,11 @@ struct rte_mempool_ops { */ struct rte_mempool_ops_table { rte_spinlock_t sl; /**< Spinlock for add/delete. */ - uint32_t num_ops; /**< Number of used ops structs in the table. */ + /** + * Number of used ops structs in the table. Note: in a secondary + * process, some ops_index can be higher than num_ops. + */ + uint32_t num_ops; /** * Storage for all possible ops structs. */ @@ -918,8 +922,14 @@ rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, * Pointer to an ops structure to register. * @return * - >=0: Success; return the index of the ops struct in the table. - * - -EINVAL - some missing callbacks while registering ops struct. - * - -ENOSPC - the maximum number of ops structs has been reached. + * - -EINVAL: some missing callbacks while registering ops struct. + * - -ENOSPC: the maximum number of ops structs has been reached, or + * the maximum number of memzones has already been allocated + * - -ENOMEM: no appropriate memory area found in which to create memzone + * - -EXISTS: the mempool ops are already registered (primary process only) + * - -ENOENT: the mempool ops are not registered on primary process + * (secondary process only) + * - -ENAMETOOLONG: the name of the mempool ops is too long */ int rte_mempool_register_ops(const struct rte_mempool_ops *ops); --- a/lib/mempool/rte_mempool_driver.h +++ a/lib/mempool/rte_mempool_driver.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2022 6WIND S.A. + */ + +#ifndef _RTE_MEMPOOL_DRIVER_H_ +#define _RTE_MEMPOOL_DRIVER_H_ + +#include +#include +#include + +/** + * Macro to statically register the ops of a mempool handler. + * Note that the rte_mempool_register_ops fails silently here when + * more than RTE_MEMPOOL_MAX_OPS_IDX is registered. + */ +#define RTE_MEMPOOL_REGISTER_OPS(name) \ + static int \ + mempool_ops_probe_##name(struct rte_vdev_device *vdev) \ + { \ + RTE_SET_USED(vdev); \ + rte_mempool_register_ops(&name); \ + return 0; \ + } \ + static struct rte_vdev_driver drv_##name = { \ + .probe = mempool_ops_probe_##name, \ + .remove = NULL, \ + }; \ + RTE_PMD_REGISTER_VDEV(mempool_##name, drv_##name); \ + RTE_PMD_AUTOLOAD_VDEV(mempool_##name##0) + +#endif /* _RTE_MEMPOOL_DRIVER_H_ */ --- a/lib/mempool/rte_mempool_ops.c +++ a/lib/mempool/rte_mempool_ops.c @@ -9,31 +9,77 @@ #include #include #include -#include #include "rte_mempool_trace.h" +#define RTE_MEMPOOL_OPS_MZNAME "rte_mempool_ops" + /* indirect jump table to support external memory pools. */ struct rte_mempool_ops_table rte_mempool_ops_table = { .sl = RTE_SPINLOCK_INITIALIZER, .num_ops = 0 }; +/* shared mempool ops table, for multiprocess support */ +struct rte_mempool_shared_ops_table { + size_t count; + struct { + char name[RTE_MEMPOOL_OPS_NAMESIZE]; + } ops[RTE_MEMPOOL_MAX_OPS_IDX]; +}; +static struct rte_mempool_shared_ops_table *shared_ops_table; + +/* Allocate and initialize the shared memory. */ +static int +rte_mempool_ops_shm_init(void) +{ + const struct rte_memzone *mz; + static int saved_ret = -EAGAIN; + + if (saved_ret != -EAGAIN) + return saved_ret; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + mz = rte_memzone_reserve(RTE_MEMPOOL_OPS_MZNAME, + sizeof(*shared_ops_table), + SOCKET_ID_ANY, 0); + if (mz == NULL) { + saved_ret = -rte_errno; + return saved_ret; + } + shared_ops_table = mz->addr; + memset(shared_ops_table, 0, sizeof(*shared_ops_table)); + } else { + mz = rte_memzone_lookup(RTE_MEMPOOL_OPS_MZNAME); + if (mz == NULL) { + saved_ret = -ENOENT; + return saved_ret; + } + shared_ops_table = mz->addr; + } + + saved_ret = 0; + return saved_ret; +} + /* add a new ops struct in rte_mempool_ops_table, return its index. */ int rte_mempool_register_ops(const struct rte_mempool_ops *h) { struct rte_mempool_ops *ops; int16_t ops_index; + unsigned int i; + int ret; rte_spinlock_lock(&rte_mempool_ops_table.sl); - if (rte_mempool_ops_table.num_ops >= - RTE_MEMPOOL_MAX_OPS_IDX) { + ret = rte_mempool_ops_shm_init(); + if (ret < 0) { rte_spinlock_unlock(&rte_mempool_ops_table.sl); RTE_LOG(ERR, MEMPOOL, - "Maximum number of mempool ops structs exceeded\n"); - return -ENOSPC; + "Failed to register shared memzone for mempool ops: %s\n", + strerror(-ret)); + return ret; } if (h->alloc == NULL || h->enqueue == NULL || @@ -46,13 +92,50 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h) if (strlen(h->name) >= sizeof(ops->name) - 1) { rte_spinlock_unlock(&rte_mempool_ops_table.sl); - RTE_LOG(DEBUG, EAL, "%s(): mempool_ops <%s>: name too long\n", + RTE_LOG(ERR, MEMPOOL, "%s(): mempool_ops <%s>: name too long\n", __func__, h->name); - rte_errno = EEXIST; - return -EEXIST; + rte_errno = ENAMETOOLONG; + return -ENAMETOOLONG; + } + + /* lookup for ops in shared memory */ + for (i = 0; i < RTE_MEMPOOL_MAX_OPS_IDX; i++) { + if (!strcmp(h->name, shared_ops_table->ops[i].name)) + break; + } + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + if (i < RTE_MEMPOOL_MAX_OPS_IDX) { + rte_spinlock_unlock(&rte_mempool_ops_table.sl); + RTE_LOG(ERR, MEMPOOL, + "%s(): mempool_ops <%s>: already registered\n", + __func__, h->name); + rte_errno = EEXIST; + return -EEXIST; + } + ops_index = shared_ops_table->count; + } else { + if (i == RTE_MEMPOOL_MAX_OPS_IDX) { + rte_spinlock_unlock(&rte_mempool_ops_table.sl); + RTE_LOG(ERR, MEMPOOL, + "%s(): mempool_ops <%s>: not registered in primary process\n", + __func__, h->name); + rte_errno = ENOENT; + return -ENOENT; + } + ops_index = i; } - ops_index = rte_mempool_ops_table.num_ops++; + if (ops_index >= RTE_MEMPOOL_MAX_OPS_IDX) { + rte_spinlock_unlock(&rte_mempool_ops_table.sl); + RTE_LOG(ERR, MEMPOOL, + "Maximum number of mempool ops structs exceeded\n"); + rte_errno = ENOSPC; + return -ENOSPC; + } + + /* update local table */ + rte_mempool_ops_table.num_ops++; ops = &rte_mempool_ops_table.ops[ops_index]; strlcpy(ops->name, h->name, sizeof(ops->name)); ops->alloc = h->alloc; @@ -65,6 +148,13 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h) ops->get_info = h->get_info; ops->dequeue_contig_blocks = h->dequeue_contig_blocks; + /* update shared memory on primary process */ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + strlcpy(shared_ops_table->ops[ops_index].name, h->name, + sizeof(ops->name)); + shared_ops_table->count++; + } + rte_spinlock_unlock(&rte_mempool_ops_table.sl); return ops_index; @@ -171,9 +261,8 @@ rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, if (mp->flags & RTE_MEMPOOL_F_POOL_CREATED) return -EEXIST; - for (i = 0; i < rte_mempool_ops_table.num_ops; i++) { - if (!strcmp(name, - rte_mempool_ops_table.ops[i].name)) { + for (i = 0; i < RTE_MEMPOOL_MAX_OPS_IDX; i++) { + if (!strcmp(name, rte_mempool_ops_table.ops[i].name)) { ops = &rte_mempool_ops_table.ops[i]; break; } --- a/lib/mempool/version.map +++ a/lib/mempool/version.map @@ -71,4 +71,7 @@ INTERNAL { # added in 21.11 rte_mempool_event_callback_register; rte_mempool_event_callback_unregister; + + # added in 22.03 + rte_mempool_ops_shm_init; }; --