@@ -147,6 +147,7 @@ struct mlx5_dev_spawn_data {
struct mlx5_switch_info info; /**< Switch information. */
struct ibv_device *ibv_dev; /**< Associated IB device. */
struct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */
+ struct rte_pci_device *pci_dev; /**< Backend PCI device. */
};
static LIST_HEAD(, mlx5_ibv_shared) mlx5_ibv_list = LIST_HEAD_INITIALIZER();
@@ -225,6 +226,7 @@ struct mlx5_dev_spawn_data {
sizeof(sh->ibdev_name));
strncpy(sh->ibdev_path, sh->ctx->device->ibdev_path,
sizeof(sh->ibdev_path));
+ sh->pci_dev = spawn->pci_dev;
pthread_mutex_init(&sh->intr_mutex, NULL);
/*
* Setting port_id to max unallowed value means
@@ -239,6 +241,22 @@ struct mlx5_dev_spawn_data {
err = ENOMEM;
goto error;
}
+ /*
+ * Once the device is added to the list of memory event
+ * callback, its global MR cache table cannot be expanded
+ * on the fly because of deadlock. If it overflows, lookup
+ * should be done by searching MR list linearly, which is slow.
+ *
+ * At this point the device is not added to the memory
+ * event list yet, context is just being created.
+ */
+ err = mlx5_mr_btree_init(&sh->mr.cache,
+ MLX5_MR_BTREE_CACHE_N * 2,
+ sh->pci_dev->device.numa_node);
+ if (err) {
+ err = rte_errno;
+ goto error;
+ }
LIST_INSERT_HEAD(&mlx5_ibv_list, sh, next);
exit:
pthread_mutex_unlock(&mlx5_ibv_list_mutex);
@@ -286,6 +304,8 @@ struct mlx5_dev_spawn_data {
assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
if (--sh->refcnt)
goto exit;
+ /* Release created Memory Regions. */
+ mlx5_mr_release(sh);
LIST_REMOVE(sh, next);
/*
* Ensure there is no async event handler installed.
@@ -651,7 +671,10 @@ struct mlx5_dev_spawn_data {
}
mlx5_proc_priv_uninit(dev);
mlx5_mprq_free_mp(dev);
- mlx5_mr_release(dev);
+ /* Remove from memory callback device list. */
+ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+ LIST_REMOVE(priv, mem_event_cb);
+ rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
assert(priv->sh);
mlx5_free_shared_dr(priv);
if (priv->rss_conf.rss_key != NULL)
@@ -1548,19 +1571,6 @@ struct mlx5_dev_spawn_data {
goto error;
}
priv->config.flow_prio = err;
- /*
- * Once the device is added to the list of memory event
- * callback, its global MR cache table cannot be expanded
- * on the fly because of deadlock. If it overflows, lookup
- * should be done by searching MR list linearly, which is slow.
- */
- err = mlx5_mr_btree_init(&priv->mr.cache,
- MLX5_MR_BTREE_CACHE_N * 2,
- eth_dev->device->numa_node);
- if (err) {
- err = rte_errno;
- goto error;
- }
/* Add device to memory callback list. */
rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
@@ -1757,6 +1767,7 @@ struct mlx5_dev_spawn_data {
list[ns].ibv_port = i;
list[ns].ibv_dev = ibv_match[0];
list[ns].eth_dev = NULL;
+ list[ns].pci_dev = pci_dev;
list[ns].ifindex = mlx5_nl_ifindex
(nl_rdma, list[ns].ibv_dev->name, i);
if (!list[ns].ifindex) {
@@ -1823,6 +1834,7 @@ struct mlx5_dev_spawn_data {
list[ns].ibv_port = 1;
list[ns].ibv_dev = ibv_match[i];
list[ns].eth_dev = NULL;
+ list[ns].pci_dev = pci_dev;
list[ns].ifindex = 0;
if (nl_rdma >= 0)
list[ns].ifindex = mlx5_nl_ifindex
@@ -275,6 +275,14 @@ struct mlx5_ibv_shared {
char ibdev_name[IBV_SYSFS_NAME_MAX]; /* IB device name. */
char ibdev_path[IBV_SYSFS_PATH_MAX]; /* IB device path for secondary */
struct ibv_device_attr_ex device_attr; /* Device properties. */
+ struct rte_pci_device *pci_dev; /* Backend PCI device. */
+ struct {
+ uint32_t dev_gen; /* Generation number to flush local caches. */
+ rte_rwlock_t rwlock; /* MR Lock. */
+ struct mlx5_mr_btree cache; /* Global MR cache table. */
+ struct mlx5_mr_list mr_list; /* Registered MR list. */
+ struct mlx5_mr_list mr_free_list; /* Freed MR list. */
+ } mr;
/* Shared DV/DR flow data section. */
pthread_mutex_t dv_mutex; /* DV context mutex. */
uint32_t dv_refcnt; /* DV/DR data reference counter. */
@@ -347,13 +355,6 @@ struct mlx5_priv {
struct mlx5_flows ctrl_flows; /* Control flow rules. */
LIST_HEAD(counters, mlx5_flow_counter) flow_counters;
/* Flow counters. */
- struct {
- uint32_t dev_gen; /* Generation number to flush local caches. */
- rte_rwlock_t rwlock; /* MR Lock. */
- struct mlx5_mr_btree cache; /* Global MR cache table. */
- struct mlx5_mr_list mr_list; /* Registered MR list. */
- struct mlx5_mr_list mr_free_list; /* Freed MR list. */
- } mr;
LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */
LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
@@ -36,7 +36,7 @@ struct mr_update_mp_data {
/**
* Expand B-tree table to a given size. Can't be called with holding
- * memory_hotplug_lock or priv->mr.rwlock due to rte_realloc().
+ * memory_hotplug_lock or sh->mr.rwlock due to rte_realloc().
*
* @param bt
* Pointer to B-tree structure.
@@ -350,7 +350,7 @@ struct mr_update_mp_data {
n = mr_find_next_chunk(mr, &entry, n);
if (!entry.end)
break;
- if (mr_btree_insert(&priv->mr.cache, &entry) < 0) {
+ if (mr_btree_insert(&priv->sh->mr.cache, &entry) < 0) {
/*
* Overflowed, but the global table cannot be expanded
* because of deadlock.
@@ -382,7 +382,7 @@ struct mr_update_mp_data {
struct mlx5_mr *mr;
/* Iterate all the existing MRs. */
- LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ LIST_FOREACH(mr, &priv->sh->mr.mr_list, mr) {
unsigned int n;
if (mr->ms_n == 0)
@@ -420,6 +420,7 @@ struct mr_update_mp_data {
uintptr_t addr)
{
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
uint16_t idx;
uint32_t lkey = UINT32_MAX;
struct mlx5_mr *mr;
@@ -430,10 +431,10 @@ struct mr_update_mp_data {
* has to be searched by traversing the original MR list instead, which
* is very slow path. Otherwise, the global cache is all inclusive.
*/
- if (!unlikely(priv->mr.cache.overflow)) {
- lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ if (!unlikely(sh->mr.cache.overflow)) {
+ lkey = mr_btree_lookup(&sh->mr.cache, &idx, addr);
if (lkey != UINT32_MAX)
- *entry = (*priv->mr.cache.table)[idx];
+ *entry = (*sh->mr.cache.table)[idx];
} else {
/* Falling back to the slowest path. */
mr = mr_lookup_dev_list(dev, entry, addr);
@@ -468,13 +469,12 @@ struct mr_update_mp_data {
/**
* Release resources of detached MR having no online entry.
*
- * @param dev
- * Pointer to Ethernet device.
+ * @param sh
+ * Pointer to Ethernet device shared context.
*/
static void
-mlx5_mr_garbage_collect(struct rte_eth_dev *dev)
+mlx5_mr_garbage_collect(struct mlx5_ibv_shared *sh)
{
- struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_mr *mr_next;
struct mlx5_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);
@@ -484,11 +484,11 @@ struct mr_update_mp_data {
* MR can't be freed with holding the lock because rte_free() could call
* memory free callback function. This will be a deadlock situation.
*/
- rte_rwlock_write_lock(&priv->mr.rwlock);
+ rte_rwlock_write_lock(&sh->mr.rwlock);
/* Detach the whole free list and release it after unlocking. */
- free_list = priv->mr.mr_free_list;
- LIST_INIT(&priv->mr.mr_free_list);
- rte_rwlock_write_unlock(&priv->mr.rwlock);
+ free_list = sh->mr.mr_free_list;
+ LIST_INIT(&sh->mr.mr_free_list);
+ rte_rwlock_write_unlock(&sh->mr.rwlock);
/* Release resources. */
mr_next = LIST_FIRST(&free_list);
while (mr_next != NULL) {
@@ -548,12 +548,12 @@ struct mr_update_mp_data {
dev->data->port_id, (void *)addr);
return UINT32_MAX;
}
- rte_rwlock_read_lock(&priv->mr.rwlock);
+ rte_rwlock_read_lock(&priv->sh->mr.rwlock);
/* Fill in output data. */
mr_lookup_dev(dev, entry, addr);
/* Lookup can't fail. */
assert(entry->lkey != UINT32_MAX);
- rte_rwlock_read_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&priv->sh->mr.rwlock);
DEBUG("port %u MR CREATED by primary process for %p:\n"
" [0x%" PRIxPTR ", 0x%" PRIxPTR "), lkey=0x%x",
dev->data->port_id, (void *)addr,
@@ -582,6 +582,7 @@ struct mr_update_mp_data {
uintptr_t addr)
{
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
struct mlx5_dev_config *config = &priv->config;
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
const struct rte_memseg_list *msl;
@@ -602,12 +603,12 @@ struct mr_update_mp_data {
dev->data->port_id, (void *)addr);
/*
* Release detached MRs if any. This can't be called with holding either
- * memory_hotplug_lock or priv->mr.rwlock. MRs on the free list have
+ * memory_hotplug_lock or sh->mr.rwlock. MRs on the free list have
* been detached by the memory free event but it couldn't be released
* inside the callback due to deadlock. As a result, releasing resources
* is quite opportunistic.
*/
- mlx5_mr_garbage_collect(dev);
+ mlx5_mr_garbage_collect(sh);
/*
* If enabled, find out a contiguous virtual address chunk in use, to
* which the given address belongs, in order to register maximum range.
@@ -710,7 +711,7 @@ struct mr_update_mp_data {
goto alloc_resources;
}
assert(data.msl == data_re.msl);
- rte_rwlock_write_lock(&priv->mr.rwlock);
+ rte_rwlock_write_lock(&sh->mr.rwlock);
/*
* Check the address is really missing. If other thread already created
* one or it is not found due to overflow, abort and return.
@@ -721,10 +722,10 @@ struct mr_update_mp_data {
* low-on-memory. Then, this entry will have to be searched
* here again.
*/
- mr_btree_insert(&priv->mr.cache, entry);
+ mr_btree_insert(&sh->mr.cache, entry);
DEBUG("port %u found MR for %p on final lookup, abort",
dev->data->port_id, (void *)addr);
- rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_write_unlock(&sh->mr.rwlock);
rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
/*
* Must be unlocked before calling rte_free() because
@@ -769,7 +770,7 @@ struct mr_update_mp_data {
* mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
* through mlx5_alloc_verbs_buf().
*/
- mr->ibv_mr = mlx5_glue->reg_mr(priv->sh->pd, (void *)data.start, len,
+ mr->ibv_mr = mlx5_glue->reg_mr(sh->pd, (void *)data.start, len,
IBV_ACCESS_LOCAL_WRITE);
if (mr->ibv_mr == NULL) {
DEBUG("port %u fail to create a verbs MR for address (%p)",
@@ -779,7 +780,7 @@ struct mr_update_mp_data {
}
assert((uintptr_t)mr->ibv_mr->addr == data.start);
assert(mr->ibv_mr->length == len);
- LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+ LIST_INSERT_HEAD(&sh->mr.mr_list, mr, mr);
DEBUG("port %u MR CREATED (%p) for %p:\n"
" [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
" lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
@@ -792,11 +793,11 @@ struct mr_update_mp_data {
mr_lookup_dev(dev, entry, addr);
/* Lookup can't fail. */
assert(entry->lkey != UINT32_MAX);
- rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_write_unlock(&sh->mr.rwlock);
rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
return entry->lkey;
err_mrlock:
- rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_write_unlock(&sh->mr.rwlock);
err_memlock:
rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
err_nolock:
@@ -854,14 +855,15 @@ struct mr_update_mp_data {
mr_rebuild_dev_cache(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
struct mlx5_mr *mr;
DRV_LOG(DEBUG, "port %u rebuild dev cache[]", dev->data->port_id);
/* Flush cache to rebuild. */
- priv->mr.cache.len = 1;
- priv->mr.cache.overflow = 0;
+ sh->mr.cache.len = 1;
+ sh->mr.cache.overflow = 0;
/* Iterate all the existing MRs. */
- LIST_FOREACH(mr, &priv->mr.mr_list, mr)
+ LIST_FOREACH(mr, &sh->mr.mr_list, mr)
if (mr_insert_dev_cache(dev, mr) < 0)
return;
}
@@ -888,6 +890,7 @@ struct mr_update_mp_data {
mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
{
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
const struct rte_memseg_list *msl;
struct mlx5_mr *mr;
int ms_n;
@@ -901,7 +904,7 @@ struct mr_update_mp_data {
assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl->page_sz));
assert(len == RTE_ALIGN(len, msl->page_sz));
ms_n = len / msl->page_sz;
- rte_rwlock_write_lock(&priv->mr.rwlock);
+ rte_rwlock_write_lock(&sh->mr.rwlock);
/* Clear bits of freed memsegs from MR. */
for (i = 0; i < ms_n; ++i) {
const struct rte_memseg *ms;
@@ -928,7 +931,7 @@ struct mr_update_mp_data {
rte_bitmap_clear(mr->ms_bmp, pos);
if (--mr->ms_n == 0) {
LIST_REMOVE(mr, mr);
- LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ LIST_INSERT_HEAD(&sh->mr.mr_free_list, mr, mr);
DEBUG("port %u remove MR(%p) from list",
dev->data->port_id, (void *)mr);
}
@@ -949,12 +952,12 @@ struct mr_update_mp_data {
* generation below) will be guaranteed to be seen by other core
* before the core sees the newly allocated memory.
*/
- ++priv->mr.dev_gen;
+ ++sh->mr.dev_gen;
DEBUG("broadcasting local cache flush, gen=%d",
- priv->mr.dev_gen);
+ sh->mr.dev_gen);
rte_smp_wmb();
}
- rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_write_unlock(&sh->mr.rwlock);
}
/**
@@ -1013,6 +1016,7 @@ struct mr_update_mp_data {
struct mlx5_mr_cache *entry, uintptr_t addr)
{
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
uint16_t idx;
uint32_t lkey;
@@ -1021,12 +1025,12 @@ struct mr_update_mp_data {
if (unlikely(bt->len == bt->size))
mr_btree_expand(bt, bt->size << 1);
/* Look up in the global cache. */
- rte_rwlock_read_lock(&priv->mr.rwlock);
- lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ rte_rwlock_read_lock(&sh->mr.rwlock);
+ lkey = mr_btree_lookup(&sh->mr.cache, &idx, addr);
if (lkey != UINT32_MAX) {
/* Found. */
- *entry = (*priv->mr.cache.table)[idx];
- rte_rwlock_read_unlock(&priv->mr.rwlock);
+ *entry = (*sh->mr.cache.table)[idx];
+ rte_rwlock_read_unlock(&sh->mr.rwlock);
/*
* Update local cache. Even if it fails, return the found entry
* to update top-half cache. Next time, this entry will be found
@@ -1035,7 +1039,7 @@ struct mr_update_mp_data {
mr_btree_insert(bt, entry);
return lkey;
}
- rte_rwlock_read_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&sh->mr.rwlock);
/* First time to see the address? Create a new MR. */
lkey = mlx5_mr_create(dev, entry, addr);
/*
@@ -1261,6 +1265,7 @@ struct mr_update_mp_data {
struct mr_update_mp_data *data = opaque;
struct rte_eth_dev *dev = data->dev;
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
struct mlx5_mr_ctrl *mr_ctrl = data->mr_ctrl;
struct mlx5_mr *mr = NULL;
uintptr_t addr = (uintptr_t)memhdr->addr;
@@ -1270,9 +1275,9 @@ struct mr_update_mp_data {
assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
/* If already registered, it should return. */
- rte_rwlock_read_lock(&priv->mr.rwlock);
+ rte_rwlock_read_lock(&sh->mr.rwlock);
lkey = mr_lookup_dev(dev, &entry, addr);
- rte_rwlock_read_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&sh->mr.rwlock);
if (lkey != UINT32_MAX)
return;
DRV_LOG(DEBUG, "port %u register MR for chunk #%d of mempool (%s)",
@@ -1286,11 +1291,11 @@ struct mr_update_mp_data {
data->ret = -1;
return;
}
- rte_rwlock_write_lock(&priv->mr.rwlock);
- LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+ rte_rwlock_write_lock(&sh->mr.rwlock);
+ LIST_INSERT_HEAD(&sh->mr.mr_list, mr, mr);
/* Insert to the global cache table. */
mr_insert_dev_cache(dev, mr);
- rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_write_unlock(&sh->mr.rwlock);
/* Insert to the local cache table */
mlx5_mr_addr2mr_bh(dev, mr_ctrl, addr);
}
@@ -1339,6 +1344,7 @@ struct mr_update_mp_data {
struct rte_eth_dev *dev;
struct mlx5_mr *mr;
struct mlx5_priv *priv;
+ struct mlx5_ibv_shared *sh;
dev = pci_dev_to_eth_dev(pdev);
if (!dev) {
@@ -1355,11 +1361,12 @@ struct mr_update_mp_data {
rte_errno = EINVAL;
return -1;
}
- rte_rwlock_write_lock(&priv->mr.rwlock);
- LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+ sh = priv->sh;
+ rte_rwlock_write_lock(&sh->mr.rwlock);
+ LIST_INSERT_HEAD(&sh->mr.mr_list, mr, mr);
/* Insert to the global cache table. */
mr_insert_dev_cache(dev, mr);
- rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_write_unlock(&sh->mr.rwlock);
return 0;
}
@@ -1384,6 +1391,7 @@ struct mr_update_mp_data {
{
struct rte_eth_dev *dev;
struct mlx5_priv *priv;
+ struct mlx5_ibv_shared *sh;
struct mlx5_mr *mr;
struct mlx5_mr_cache entry;
@@ -1395,10 +1403,11 @@ struct mr_update_mp_data {
return -1;
}
priv = dev->data->dev_private;
- rte_rwlock_read_lock(&priv->mr.rwlock);
+ sh = priv->sh;
+ rte_rwlock_read_lock(&sh->mr.rwlock);
mr = mr_lookup_dev_list(dev, &entry, (uintptr_t)addr);
if (!mr) {
- rte_rwlock_read_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&sh->mr.rwlock);
DRV_LOG(WARNING, "address 0x%" PRIxPTR " wasn't registered "
"to PCI device %p", (uintptr_t)addr,
(void *)pdev);
@@ -1406,7 +1415,7 @@ struct mr_update_mp_data {
return -1;
}
LIST_REMOVE(mr, mr);
- LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ LIST_INSERT_HEAD(&sh->mr.mr_free_list, mr, mr);
DEBUG("port %u remove MR(%p) from list", dev->data->port_id,
(void *)mr);
mr_rebuild_dev_cache(dev);
@@ -1419,11 +1428,10 @@ struct mr_update_mp_data {
* generation below) will be guaranteed to be seen by other core
* before the core sees the newly allocated memory.
*/
- ++priv->mr.dev_gen;
- DEBUG("broadcasting local cache flush, gen=%d",
- priv->mr.dev_gen);
+ ++sh->mr.dev_gen;
+ DEBUG("broadcasting local cache flush, gen=%d", sh->mr.dev_gen);
rte_smp_wmb();
- rte_rwlock_read_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&sh->mr.rwlock);
return 0;
}
@@ -1544,25 +1552,24 @@ struct mr_update_mp_data {
/**
* Dump all the created MRs and the global cache entries.
*
- * @param dev
- * Pointer to Ethernet device.
+ * @param sh
+ * Pointer to Ethernet device shared context.
*/
void
-mlx5_mr_dump_dev(struct rte_eth_dev *dev __rte_unused)
+mlx5_mr_dump_dev(struct mlx5_ibv_shared *sh __rte_unused)
{
#ifndef NDEBUG
- struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_mr *mr;
int mr_n = 0;
int chunk_n = 0;
- rte_rwlock_read_lock(&priv->mr.rwlock);
+ rte_rwlock_read_lock(&sh->mr.rwlock);
/* Iterate all the existing MRs. */
- LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ LIST_FOREACH(mr, &sh->mr.mr_list, mr) {
unsigned int n;
- DEBUG("port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
- dev->data->port_id, mr_n++,
+ DEBUG("device %s MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
+ sh->ibdev_name, mr_n++,
rte_cpu_to_be_32(mr->ibv_mr->lkey),
mr->ms_n, mr->ms_bmp_n);
if (mr->ms_n == 0)
@@ -1577,45 +1584,40 @@ struct mr_update_mp_data {
chunk_n++, ret.start, ret.end);
}
}
- DEBUG("port %u dumping global cache", dev->data->port_id);
- mlx5_mr_btree_dump(&priv->mr.cache);
- rte_rwlock_read_unlock(&priv->mr.rwlock);
+ DEBUG("device %s dumping global cache", sh->ibdev_name);
+ mlx5_mr_btree_dump(&sh->mr.cache);
+ rte_rwlock_read_unlock(&sh->mr.rwlock);
#endif
}
/**
- * Release all the created MRs and resources. Remove device from memory callback
+ * Release all the created MRs and resources for shared device context.
* list.
*
- * @param dev
- * Pointer to Ethernet device.
+ * @param sh
+ * Pointer to Ethernet device shared context.
*/
void
-mlx5_mr_release(struct rte_eth_dev *dev)
+mlx5_mr_release(struct mlx5_ibv_shared *sh)
{
- struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_mr *mr_next;
- /* Remove from memory callback device list. */
- rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
- LIST_REMOVE(priv, mem_event_cb);
- rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
- mlx5_mr_dump_dev(dev);
- rte_rwlock_write_lock(&priv->mr.rwlock);
+ mlx5_mr_dump_dev(sh);
+ rte_rwlock_write_lock(&sh->mr.rwlock);
/* Detach from MR list and move to free list. */
- mr_next = LIST_FIRST(&priv->mr.mr_list);
+ mr_next = LIST_FIRST(&sh->mr.mr_list);
while (mr_next != NULL) {
struct mlx5_mr *mr = mr_next;
mr_next = LIST_NEXT(mr, mr);
LIST_REMOVE(mr, mr);
- LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ LIST_INSERT_HEAD(&sh->mr.mr_free_list, mr, mr);
}
- LIST_INIT(&priv->mr.mr_list);
+ LIST_INIT(&sh->mr.mr_list);
/* Free global cache. */
- mlx5_mr_btree_free(&priv->mr.cache);
- rte_rwlock_write_unlock(&priv->mr.rwlock);
+ mlx5_mr_btree_free(&sh->mr.cache);
+ rte_rwlock_write_unlock(&sh->mr.rwlock);
/* Free all remaining MRs. */
- mlx5_mr_garbage_collect(dev);
+ mlx5_mr_garbage_collect(sh);
}
@@ -62,6 +62,7 @@ struct mlx5_mr_ctrl {
struct mlx5_mr_btree cache_bh; /* Cache for bottom-half. */
} __rte_packed;
+struct mlx5_ibv_shared;
extern struct mlx5_dev_list mlx5_mem_event_cb_list;
extern rte_rwlock_t mlx5_mem_event_rwlock;
@@ -76,11 +77,11 @@ void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
size_t len, void *arg);
int mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
struct rte_mempool *mp);
-void mlx5_mr_release(struct rte_eth_dev *dev);
+void mlx5_mr_release(struct mlx5_ibv_shared *sh);
/* Debug purpose functions. */
void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt);
-void mlx5_mr_dump_dev(struct rte_eth_dev *dev);
+void mlx5_mr_dump_dev(struct mlx5_ibv_shared *sh);
/**
* Look up LKey from given lookup table by linear search. Firstly look up the
@@ -814,7 +814,7 @@ struct mlx5_txq_ctrl *
goto error;
}
/* Save pointer of global generation number to check memory event. */
- tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
+ tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->sh->mr.dev_gen;
assert(desc > MLX5_TX_COMP_THRESH);
tmpl->txq.offloads = conf->offloads |
dev->data->dev_conf.txmode.offloads;