[dpdk-dev,6/7] vmxnet3: introduce command to register memory region
Checks
Commit Message
In vmxnet3 version 3, the emulation added support for the vmxnet3 driver
to communicate information about the memory regions the driver will use
for rx/tx buffers. The driver can also indicate which rx/tx queue the
memory region is applicable for. If this information is communicated
to the emulation, the emulation will always keep these memory regions
mapped, thereby avoiding the mapping/unmapping overhead for every packet.
Signed-off-by: Shrikrishna Khare <skhare@vmware.com>
Signed-off-by: Guolin Yang <gyang@vmware.com>
Acked-by: Yong Wang <yongwang@vmware.com>
Acked-by: Jin Heo <heoj@vmware.com>
---
drivers/net/vmxnet3/base/vmxnet3_defs.h | 25 ++++++++
drivers/net/vmxnet3/vmxnet3_ethdev.c | 102 ++++++++++++++++++++++++++++++++
drivers/net/vmxnet3/vmxnet3_ethdev.h | 2 +
3 files changed, 129 insertions(+)
Comments
On 2/25/2017 9:59 PM, Shrikrishna Khare wrote:
> In vmxnet3 version 3, the emulation added support for the vmxnet3 driver
> to communicate information about the memory regions the driver will use
> for rx/tx buffers. The driver can also indicate which rx/tx queue the
> memory region is applicable for. If this information is communicated
> to the emulation, the emulation will always keep these memory regions
> mapped, thereby avoiding the mapping/unmapping overhead for every packet.
>
> Signed-off-by: Shrikrishna Khare <skhare@vmware.com>
> Signed-off-by: Guolin Yang <gyang@vmware.com>
> Acked-by: Yong Wang <yongwang@vmware.com>
> Acked-by: Jin Heo <heoj@vmware.com>
<...>
> + PMD_INIT_LOG(INFO,
> + "index: %u startPA: %lu length: %u, rxBits: %x",
> + j, mr->startPA, mr->length, mr->rxQueueBits);
> + j++;
> + }
> + hw->memRegs->numRegs = j;
> + PMD_INIT_LOG("numRegs: %u", j);
Macro argument is not correct, causing following build error:
.../drivers/net/vmxnet3/vmxnet3_ethdev.c:568:2: error: pasting formed
'RTE_LOG_"numRegs: %u"', an invalid preprocessing token
PMD_INIT_LOG("numRegs: %u", j);
^
On 2/25/2017 9:59 PM, Shrikrishna Khare wrote:
> In vmxnet3 version 3, the emulation added support for the vmxnet3 driver
> to communicate information about the memory regions the driver will use
> for rx/tx buffers. The driver can also indicate which rx/tx queue the
> memory region is applicable for. If this information is communicated
> to the emulation, the emulation will always keep these memory regions
> mapped, thereby avoiding the mapping/unmapping overhead for every packet.
>
> Signed-off-by: Shrikrishna Khare <skhare@vmware.com>
> Signed-off-by: Guolin Yang <gyang@vmware.com>
> Acked-by: Yong Wang <yongwang@vmware.com>
> Acked-by: Jin Heo <heoj@vmware.com>
<...>
> + PMD_INIT_LOG(INFO,
> + "index: %u startPA: %lu length: %u, rxBits: %x",
> + j, mr->startPA, mr->length, mr->rxQueueBits);
Also this gives following build error for 32bit (i686) build:
.../drivers/net/vmxnet3/vmxnet3_ethdev.c: In function
‘vmxnet3_dev_setup_memreg’:
.../drivers/net/vmxnet3/vmxnet3_ethdev.c:564:52: error: format ‘%lu’
expects argument of type ‘long unsigned int’, but argument 6 has type
‘uint64 {aka long long unsigned int}’ [-Werror=format=]
j, mr->startPA, mr->length, mr->rxQueueBits);
^
@@ -111,6 +111,7 @@ typedef enum {
VMXNET3_CMD_ACTIVATE_VF,
VMXNET3_CMD_RESERVED3,
VMXNET3_CMD_RESERVED4,
+ VMXNET3_CMD_REGISTER_MEMREGS,
VMXNET3_CMD_FIRST_GET = 0xF00D0000,
VMXNET3_CMD_GET_QUEUE_STATUS = VMXNET3_CMD_FIRST_GET,
@@ -722,6 +723,30 @@ struct Vmxnet3_SetPolling {
#include "vmware_pack_end.h"
Vmxnet3_SetPolling;
+typedef
+#include "vmware_pack_begin.h"
+struct Vmxnet3_MemoryRegion {
+ __le64 startPA;
+ __le32 length;
+ __le16 txQueueBits; /* bit n corresponding to tx queue n */
+ __le16 rxQueueBits; /* bit n corresponding to rx queue n */
+}
+#include "vmware_pack_end.h"
+Vmxnet3_MemoryRegion;
+
+#define MAX_MEMORY_REGION_PER_QUEUE 16
+#define MAX_MEMORY_REGION_PER_DEVICE 256
+
+typedef
+#include "vmware_pack_begin.h"
+struct Vmxnet3_MemRegs {
+ __le16 numRegs;
+ __le16 pad[3];
+ Vmxnet3_MemoryRegion memRegs[1];
+}
+#include "vmware_pack_end.h"
+Vmxnet3_MemRegs;
+
/*
* If the command data <= 16 bytes, use the shared memory direcly.
* Otherwise, use the variable length configuration descriptor.
@@ -489,6 +489,92 @@ vmxnet3_write_mac(struct vmxnet3_hw *hw, const uint8_t *addr)
}
static int
+vmxnet3_dev_setup_memreg(struct rte_eth_dev *dev)
+{
+ struct vmxnet3_hw *hw = dev->data->dev_private;
+ Vmxnet3_DriverShared *shared = hw->shared;
+ Vmxnet3_CmdInfo *cmdInfo;
+ struct rte_mempool *mp[VMXNET3_MAX_RX_QUEUES];
+ uint8_t index[VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES];
+ uint32_t num, i, j, size;
+
+ if (hw->memRegsPA == 0) {
+ const struct rte_memzone *mz;
+
+ size = sizeof(Vmxnet3_MemRegs) +
+ (VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES) *
+ sizeof(Vmxnet3_MemoryRegion);
+
+ mz = gpa_zone_reserve(dev, size, "memRegs", rte_socket_id(), 8,
+ 1);
+ if (mz == NULL) {
+ PMD_INIT_LOG(ERR, "ERROR: Creating memRegs zone");
+ return -ENOMEM;
+ }
+ memset(mz->addr, 0, mz->len);
+ hw->memRegs = mz->addr;
+ hw->memRegsPA = mz->phys_addr;
+ }
+
+ num = hw->num_rx_queues;
+
+ for (i = 0; i < num; i++) {
+ vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
+
+ mp[i] = rxq->mp;
+ index[i] = 1 << i;
+ }
+
+ /*
+ * The same mempool could be used by multiple queues. In such a case,
+ * remove duplicate mempool entries. Only one entry is kept with
+ * bitmask indicating queues that are using this mempool.
+ */
+ for (i = 1; i < num; i++) {
+ for (j = 0; j < i; j++) {
+ if (mp[i] == mp[j]) {
+ mp[i] = NULL;
+ index[j] |= 1 << i;
+ break;
+ }
+ }
+ }
+
+ j = 0;
+ for (i = 0; i < num; i++) {
+ if (mp[i] == NULL) {
+ continue;
+ }
+
+ Vmxnet3_MemoryRegion *mr = &hw->memRegs->memRegs[j];
+
+ mr->startPA =
+ (uintptr_t)STAILQ_FIRST(&mp[i]->mem_list)->phys_addr;
+ mr->length = STAILQ_FIRST(&mp[i]->mem_list)->len <= INT32_MAX ?
+ STAILQ_FIRST(&mp[i]->mem_list)->len : INT32_MAX;
+ mr->txQueueBits = index[i];
+ mr->rxQueueBits = index[i];
+
+ PMD_INIT_LOG(INFO,
+ "index: %u startPA: %lu length: %u, rxBits: %x",
+ j, mr->startPA, mr->length, mr->rxQueueBits);
+ j++;
+ }
+ hw->memRegs->numRegs = j;
+ PMD_INIT_LOG("numRegs: %u", j);
+
+ size = sizeof(Vmxnet3_MemRegs) +
+ (j - 1) * sizeof(Vmxnet3_MemoryRegion);
+
+ cmdInfo = &shared->cu.cmdInfo;
+ cmdInfo->varConf.confVer = 1;
+ cmdInfo->varConf.confLen = size;
+ cmdInfo->varConf.confPA = hw->memRegsPA;
+
+ return 0;
+}
+
+static int
vmxnet3_setup_driver_shared(struct rte_eth_dev *dev)
{
struct rte_eth_conf port_conf = dev->data->dev_conf;
@@ -628,6 +714,20 @@ vmxnet3_dev_start(struct rte_eth_dev *dev)
return -EINVAL;
}
+ /* Setup memory region for rx buffers */
+ ret = vmxnet3_dev_setup_memreg(dev);
+ if (ret == 0) {
+ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
+ VMXNET3_CMD_REGISTER_MEMREGS);
+ ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
+ if (ret != 0) {
+ PMD_INIT_LOG(DEBUG, "Failed in setup memory region cmd\n");
+ }
+ ret = 0;
+ } else {
+ PMD_INIT_LOG(DEBUG, "Failed to setup memory region\n");
+ }
+
/* Disable interrupts */
vmxnet3_disable_intr(hw);
@@ -641,6 +741,8 @@ vmxnet3_dev_start(struct rte_eth_dev *dev)
return ret;
}
+ hw->adapter_stopped = FALSE;
+
/* Setting proper Rx Mode and issue Rx Mode Update command */
vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1);
@@ -119,6 +119,8 @@ struct vmxnet3_hw {
uint64_t rss_confPA;
vmxnet3_mf_table_t *mf_table;
uint32_t shadow_vfta[VMXNET3_VFT_SIZE];
+ Vmxnet3_MemRegs *memRegs;
+ uint64_t memRegsPA;
#define VMXNET3_VFT_TABLE_SIZE (VMXNET3_VFT_SIZE * sizeof(uint32_t))
};