[dpdk-dev] [PATCH] virtio: split virtio rx/tx queue
Huawei Xie
huawei.xie at intel.com
Wed May 4 02:50:27 CEST 2016
Currently virtio RX/TX paths use common vq structure.
The initial idea is to split virtio RX and TX queues completely as they
have different memory requirement and we could arrange data friendly for
optimization for different paths in future.
With this patch, we keep a common vq structure, as we have too
many common vq operations. Split fields into virtnet_rx
and virtnet_tx respectively.
Signed-off-by: Huawei Xie <huawei.xie at intel.com>
---
drivers/net/virtio/virtio_ethdev.c | 333 +++++++++-----------
drivers/net/virtio/virtio_pci.c | 4 +-
drivers/net/virtio/virtio_pci.h | 3 +-
drivers/net/virtio/virtio_rxtx.c | 531 +++++++++++++++++++++++---------
drivers/net/virtio/virtio_rxtx.h | 54 +++-
drivers/net/virtio/virtio_rxtx_simple.c | 85 ++---
drivers/net/virtio/virtqueue.h | 67 ++--
7 files changed, 655 insertions(+), 422 deletions(-)
diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
index 63a368a..4d4e59e 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -115,40 +115,62 @@ struct rte_virtio_xstats_name_off {
};
/* [rt]x_qX_ is prepended to the name string here */
-static const struct rte_virtio_xstats_name_off rte_virtio_q_stat_strings[] = {
- {"good_packets", offsetof(struct virtqueue, packets)},
- {"good_bytes", offsetof(struct virtqueue, bytes)},
- {"errors", offsetof(struct virtqueue, errors)},
- {"multicast_packets", offsetof(struct virtqueue, multicast)},
- {"broadcast_packets", offsetof(struct virtqueue, broadcast)},
- {"undersize_packets", offsetof(struct virtqueue, size_bins[0])},
- {"size_64_packets", offsetof(struct virtqueue, size_bins[1])},
- {"size_65_127_packets", offsetof(struct virtqueue, size_bins[2])},
- {"size_128_255_packets", offsetof(struct virtqueue, size_bins[3])},
- {"size_256_511_packets", offsetof(struct virtqueue, size_bins[4])},
- {"size_512_1023_packets", offsetof(struct virtqueue, size_bins[5])},
- {"size_1024_1517_packets", offsetof(struct virtqueue, size_bins[6])},
- {"size_1518_max_packets", offsetof(struct virtqueue, size_bins[7])},
+static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = {
+ {"good_packets", offsetof(struct virtnet_rx, stats.packets)},
+ {"good_bytes", offsetof(struct virtnet_rx, stats.bytes)},
+ {"errors", offsetof(struct virtnet_rx, stats.errors)},
+ {"multicast_packets", offsetof(struct virtnet_rx, stats.multicast)},
+ {"broadcast_packets", offsetof(struct virtnet_rx, stats.broadcast)},
+ {"undersize_packets", offsetof(struct virtnet_rx, stats.size_bins[0])},
+ {"size_64_packets", offsetof(struct virtnet_rx, stats.size_bins[1])},
+ {"size_65_127_packets", offsetof(struct virtnet_rx, stats.size_bins[2])},
+ {"size_128_255_packets", offsetof(struct virtnet_rx, stats.size_bins[3])},
+ {"size_256_511_packets", offsetof(struct virtnet_rx, stats.size_bins[4])},
+ {"size_512_1023_packets", offsetof(struct virtnet_rx, stats.size_bins[5])},
+ {"size_1024_1517_packets", offsetof(struct virtnet_rx, stats.size_bins[6])},
+ {"size_1518_max_packets", offsetof(struct virtnet_rx, stats.size_bins[7])},
};
-#define VIRTIO_NB_Q_XSTATS (sizeof(rte_virtio_q_stat_strings) / \
- sizeof(rte_virtio_q_stat_strings[0]))
+/* [rt]x_qX_ is prepended to the name string here */
+static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = {
+ {"good_packets", offsetof(struct virtnet_tx, stats.packets)},
+ {"good_bytes", offsetof(struct virtnet_tx, stats.bytes)},
+ {"errors", offsetof(struct virtnet_tx, stats.errors)},
+ {"multicast_packets", offsetof(struct virtnet_tx, stats.multicast)},
+ {"broadcast_packets", offsetof(struct virtnet_tx, stats.broadcast)},
+ {"undersize_packets", offsetof(struct virtnet_tx, stats.size_bins[0])},
+ {"size_64_packets", offsetof(struct virtnet_tx, stats.size_bins[1])},
+ {"size_65_127_packets", offsetof(struct virtnet_tx, stats.size_bins[2])},
+ {"size_128_255_packets", offsetof(struct virtnet_tx, stats.size_bins[3])},
+ {"size_256_511_packets", offsetof(struct virtnet_tx, stats.size_bins[4])},
+ {"size_512_1023_packets", offsetof(struct virtnet_tx, stats.size_bins[5])},
+ {"size_1024_1517_packets", offsetof(struct virtnet_tx, stats.size_bins[6])},
+ {"size_1518_max_packets", offsetof(struct virtnet_tx, stats.size_bins[7])},
+};
+
+#define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
+ sizeof(rte_virtio_rxq_stat_strings[0]))
+#define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
+ sizeof(rte_virtio_txq_stat_strings[0]))
static int
-virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl,
+virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl,
int *dlen, int pkt_num)
{
uint32_t head, i;
int k, sum = 0;
virtio_net_ctrl_ack status = ~0;
struct virtio_pmd_ctrl result;
+ struct virtqueue *vq;
ctrl->status = status;
- if (!(vq && vq->hw->cvq)) {
+ if (!cvq || !cvq->vq) {
PMD_INIT_LOG(ERR, "Control queue is not supported.");
return -1;
}
+ vq = cvq->vq;
+
head = vq->vq_desc_head_idx;
PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
@@ -158,7 +180,7 @@ virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl,
if ((vq->vq_free_cnt < ((uint32_t)pkt_num + 2)) || (pkt_num < 1))
return -1;
- memcpy(vq->virtio_net_hdr_mz->addr, ctrl,
+ memcpy(cvq->virtio_net_hdr_mz->addr, ctrl,
sizeof(struct virtio_pmd_ctrl));
/*
@@ -168,14 +190,14 @@ virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl,
* One RX packet for ACK.
*/
vq->vq_ring.desc[head].flags = VRING_DESC_F_NEXT;
- vq->vq_ring.desc[head].addr = vq->virtio_net_hdr_mz->phys_addr;
+ vq->vq_ring.desc[head].addr = cvq->virtio_net_hdr_mz->phys_addr;
vq->vq_ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
vq->vq_free_cnt--;
i = vq->vq_ring.desc[head].next;
for (k = 0; k < pkt_num; k++) {
vq->vq_ring.desc[i].flags = VRING_DESC_F_NEXT;
- vq->vq_ring.desc[i].addr = vq->virtio_net_hdr_mz->phys_addr
+ vq->vq_ring.desc[i].addr = cvq->virtio_net_hdr_mz->phys_addr
+ sizeof(struct virtio_net_ctrl_hdr)
+ sizeof(ctrl->status) + sizeof(uint8_t)*sum;
vq->vq_ring.desc[i].len = dlen[k];
@@ -185,7 +207,7 @@ virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl,
}
vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
- vq->vq_ring.desc[i].addr = vq->virtio_net_hdr_mz->phys_addr
+ vq->vq_ring.desc[i].addr = cvq->virtio_net_hdr_mz->phys_addr
+ sizeof(struct virtio_net_ctrl_hdr);
vq->vq_ring.desc[i].len = sizeof(ctrl->status);
vq->vq_free_cnt--;
@@ -230,7 +252,7 @@ virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl,
PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
vq->vq_free_cnt, vq->vq_desc_head_idx);
- memcpy(&result, vq->virtio_net_hdr_mz->addr,
+ memcpy(&result, cvq->virtio_net_hdr_mz->addr,
sizeof(struct virtio_pmd_ctrl));
return result.status;
@@ -273,28 +295,25 @@ virtio_dev_queue_release(struct virtqueue *vq) {
}
}
-int virtio_dev_queue_setup(struct rte_eth_dev *dev,
- int queue_type,
- uint16_t queue_idx,
+static int
+virtio_dev_cq_queue_setup(struct rte_eth_dev *dev,
uint16_t vtpci_queue_idx,
- uint16_t nb_desc,
- unsigned int socket_id,
- struct virtqueue **pvq)
+ unsigned int socket_id)
{
char vq_name[VIRTQUEUE_MAX_NAME_SZ];
const struct rte_memzone *mz;
unsigned int vq_size, size;
struct virtio_hw *hw = dev->data->dev_private;
+ struct virtnet_ctl *cvq;
struct virtqueue *vq = NULL;
PMD_INIT_LOG(DEBUG, "setting up queue: %u", vtpci_queue_idx);
-
/*
* Read the virtqueue size from the Queue Size field
* Always power of 2 and if 0 virtqueue does not exist
*/
vq_size = hw->vtpci_ops->get_queue_num(hw, vtpci_queue_idx);
- PMD_INIT_LOG(DEBUG, "vq_size: %u nb_desc:%u", vq_size, nb_desc);
+ PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size);
if (vq_size == 0) {
PMD_INIT_LOG(ERR, "virtqueue does not exist");
return -EINVAL;
@@ -305,52 +324,32 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev,
return -EINVAL;
}
- if (queue_type == VTNET_RQ) {
- snprintf(vq_name, sizeof(vq_name), "port%d_rvq%d",
- dev->data->port_id, queue_idx);
- vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) +
- vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE);
- vq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
- (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
- sizeof(vq->sw_ring[0]), RTE_CACHE_LINE_SIZE, socket_id);
- } else if (queue_type == VTNET_TQ) {
- snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d",
- dev->data->port_id, queue_idx);
- vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) +
- vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE);
- } else if (queue_type == VTNET_CQ) {
- snprintf(vq_name, sizeof(vq_name), "port%d_cvq",
- dev->data->port_id);
- vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) +
- vq_size * sizeof(struct vq_desc_extra),
- RTE_CACHE_LINE_SIZE);
- }
- if (vq == NULL) {
+ snprintf(vq_name, sizeof(vq_name), "port%d_cvq",
+ dev->data->port_id);
+ cvq = rte_zmalloc(vq_name, sizeof(*cvq), RTE_CACHE_LINE_SIZE);
+ vq = rte_zmalloc_socket("cvq->vq", sizeof(*vq) +
+ vq_size * sizeof(struct vq_desc_extra),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (cvq == NULL || vq == NULL) {
PMD_INIT_LOG(ERR, "Can not allocate virtqueue");
- return -ENOMEM;
- }
- if (queue_type == VTNET_RQ && vq->sw_ring == NULL) {
- PMD_INIT_LOG(ERR, "Can not allocate RX soft ring");
rte_free(vq);
+ rte_free(cvq);
return -ENOMEM;
}
+ cvq->vq = vq;
vq->hw = hw;
- vq->port_id = dev->data->port_id;
- vq->queue_id = queue_idx;
- vq->vq_queue_index = vtpci_queue_idx;
+ cvq->port_id = dev->data->port_id;
vq->vq_nentries = vq_size;
-
- if (nb_desc == 0 || nb_desc > vq_size)
- nb_desc = vq_size;
- vq->vq_free_cnt = nb_desc;
+ vq->vq_free_cnt = vq_size;
/*
* Reserve a memzone for vring elements
*/
size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
- PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", size, vq->vq_ring_size);
+ PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
+ size, vq->vq_ring_size);
mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
@@ -359,6 +358,7 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev,
mz = rte_memzone_lookup(vq_name);
if (mz == NULL) {
rte_free(vq);
+ rte_free(cvq);
return -ENOMEM;
}
}
@@ -371,103 +371,39 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev,
if ((mz->phys_addr + vq->vq_ring_size - 1) >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!");
rte_free(vq);
+ rte_free(cvq);
return -ENOMEM;
}
memset(mz->addr, 0, sizeof(mz->len));
- vq->mz = mz;
+ cvq->mz = mz;
vq->vq_ring_mem = mz->phys_addr;
vq->vq_ring_virt_mem = mz->addr;
- PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%"PRIx64, (uint64_t)mz->phys_addr);
- PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%"PRIx64, (uint64_t)(uintptr_t)mz->addr);
- vq->virtio_net_hdr_mz = NULL;
- vq->virtio_net_hdr_mem = 0;
-
- if (queue_type == VTNET_TQ) {
- const struct rte_memzone *hdr_mz;
- struct virtio_tx_region *txr;
- unsigned int i;
-
- /*
- * For each xmit packet, allocate a virtio_net_hdr
- * and indirect ring elements
- */
- snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d_hdrzone",
- dev->data->port_id, queue_idx);
- hdr_mz = rte_memzone_reserve_aligned(vq_name,
- vq_size * sizeof(*txr),
- socket_id, 0,
- RTE_CACHE_LINE_SIZE);
- if (hdr_mz == NULL) {
- if (rte_errno == EEXIST)
- hdr_mz = rte_memzone_lookup(vq_name);
- if (hdr_mz == NULL) {
- rte_free(vq);
- return -ENOMEM;
- }
- }
- vq->virtio_net_hdr_mz = hdr_mz;
- vq->virtio_net_hdr_mem = hdr_mz->phys_addr;
-
- txr = hdr_mz->addr;
- memset(txr, 0, vq_size * sizeof(*txr));
- for (i = 0; i < vq_size; i++) {
- struct vring_desc *start_dp = txr[i].tx_indir;
-
- vring_desc_init(start_dp, RTE_DIM(txr[i].tx_indir));
-
- /* first indirect descriptor is always the tx header */
- start_dp->addr = vq->virtio_net_hdr_mem
- + i * sizeof(*txr)
- + offsetof(struct virtio_tx_region, tx_hdr);
-
- start_dp->len = vq->hw->vtnet_hdr_size;
- start_dp->flags = VRING_DESC_F_NEXT;
- }
-
- } else if (queue_type == VTNET_CQ) {
- /* Allocate a page for control vq command, data and status */
- snprintf(vq_name, sizeof(vq_name), "port%d_cvq_hdrzone",
- dev->data->port_id);
- vq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name,
- PAGE_SIZE, socket_id, 0, RTE_CACHE_LINE_SIZE);
- if (vq->virtio_net_hdr_mz == NULL) {
- if (rte_errno == EEXIST)
- vq->virtio_net_hdr_mz =
- rte_memzone_lookup(vq_name);
- if (vq->virtio_net_hdr_mz == NULL) {
- rte_free(vq);
- return -ENOMEM;
- }
+ PMD_INIT_LOG(DEBUG, "cvq->vq_ring_mem: 0x%"PRIx64,
+ (uint64_t)mz->phys_addr);
+ PMD_INIT_LOG(DEBUG, "cvq->vq_ring_virt_mem: 0x%"PRIx64,
+ (uint64_t)(uintptr_t)mz->addr);
+ /* Allocate a page for control vq command, data and status */
+ snprintf(vq_name, sizeof(vq_name), "port%d_cvq_hdrzone",
+ dev->data->port_id);
+ cvq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name,
+ PAGE_SIZE, socket_id, 0, RTE_CACHE_LINE_SIZE);
+ if (cvq->virtio_net_hdr_mz == NULL) {
+ if (rte_errno == EEXIST)
+ cvq->virtio_net_hdr_mz =
+ rte_memzone_lookup(vq_name);
+ if (cvq->virtio_net_hdr_mz == NULL) {
+ rte_free(vq);
+ rte_free(cvq);
+ return -ENOMEM;
}
- vq->virtio_net_hdr_mem =
- vq->virtio_net_hdr_mz->phys_addr;
- memset(vq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE);
}
+ cvq->virtio_net_hdr_mem =
+ cvq->virtio_net_hdr_mz->phys_addr;
+ memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE);
hw->vtpci_ops->setup_queue(hw, vq);
- *pvq = vq;
- return 0;
-}
-
-static int
-virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx,
- uint32_t socket_id)
-{
- struct virtqueue *vq;
- int ret;
- struct virtio_hw *hw = dev->data->dev_private;
-
- PMD_INIT_FUNC_TRACE();
- ret = virtio_dev_queue_setup(dev, VTNET_CQ, VTNET_SQ_CQ_QUEUE_IDX,
- vtpci_queue_idx, 0, socket_id, &vq);
- if (ret < 0) {
- PMD_INIT_LOG(ERR, "control vq initialization failed");
- return ret;
- }
-
- hw->cvq = vq;
return 0;
}
@@ -675,32 +611,32 @@ virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
unsigned i;
for (i = 0; i < dev->data->nb_tx_queues; i++) {
- const struct virtqueue *txvq = dev->data->tx_queues[i];
+ const struct virtnet_tx *txvq = dev->data->tx_queues[i];
if (txvq == NULL)
continue;
- stats->opackets += txvq->packets;
- stats->obytes += txvq->bytes;
- stats->oerrors += txvq->errors;
+ stats->opackets += txvq->stats.packets;
+ stats->obytes += txvq->stats.bytes;
+ stats->oerrors += txvq->stats.errors;
if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
- stats->q_opackets[i] = txvq->packets;
- stats->q_obytes[i] = txvq->bytes;
+ stats->q_opackets[i] = txvq->stats.packets;
+ stats->q_obytes[i] = txvq->stats.bytes;
}
}
for (i = 0; i < dev->data->nb_rx_queues; i++) {
- const struct virtqueue *rxvq = dev->data->rx_queues[i];
+ const struct virtnet_rx *rxvq = dev->data->rx_queues[i];
if (rxvq == NULL)
continue;
- stats->ipackets += rxvq->packets;
- stats->ibytes += rxvq->bytes;
- stats->ierrors += rxvq->errors;
+ stats->ipackets += rxvq->stats.packets;
+ stats->ibytes += rxvq->stats.bytes;
+ stats->ierrors += rxvq->stats.errors;
if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
- stats->q_ipackets[i] = rxvq->packets;
- stats->q_ibytes[i] = rxvq->bytes;
+ stats->q_ipackets[i] = rxvq->stats.packets;
+ stats->q_ibytes[i] = rxvq->stats.bytes;
}
}
@@ -714,44 +650,44 @@ virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
unsigned i;
unsigned count = 0;
- unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_Q_XSTATS +
- dev->data->nb_rx_queues * VIRTIO_NB_Q_XSTATS;
+ unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
+ dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
if (n < nstats)
return nstats;
for (i = 0; i < dev->data->nb_rx_queues; i++) {
- struct virtqueue *rxvq = dev->data->rx_queues[i];
+ struct virtnet_rx *rxvq = dev->data->rx_queues[i];
if (rxvq == NULL)
continue;
unsigned t;
- for (t = 0; t < VIRTIO_NB_Q_XSTATS; t++) {
+ for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
snprintf(xstats[count].name, sizeof(xstats[count].name),
"rx_q%u_%s", i,
- rte_virtio_q_stat_strings[t].name);
+ rte_virtio_rxq_stat_strings[t].name);
xstats[count].value = *(uint64_t *)(((char *)rxvq) +
- rte_virtio_q_stat_strings[t].offset);
+ rte_virtio_rxq_stat_strings[t].offset);
count++;
}
}
for (i = 0; i < dev->data->nb_tx_queues; i++) {
- struct virtqueue *txvq = dev->data->tx_queues[i];
+ struct virtnet_tx *txvq = dev->data->tx_queues[i];
if (txvq == NULL)
continue;
unsigned t;
- for (t = 0; t < VIRTIO_NB_Q_XSTATS; t++) {
+ for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
snprintf(xstats[count].name, sizeof(xstats[count].name),
"tx_q%u_%s", i,
- rte_virtio_q_stat_strings[t].name);
+ rte_virtio_txq_stat_strings[t].name);
xstats[count].value = *(uint64_t *)(((char *)txvq) +
- rte_virtio_q_stat_strings[t].offset);
+ rte_virtio_txq_stat_strings[t].offset);
count++;
}
}
@@ -771,29 +707,31 @@ virtio_dev_stats_reset(struct rte_eth_dev *dev)
unsigned int i;
for (i = 0; i < dev->data->nb_tx_queues; i++) {
- struct virtqueue *txvq = dev->data->tx_queues[i];
+ struct virtnet_tx *txvq = dev->data->tx_queues[i];
if (txvq == NULL)
continue;
- txvq->packets = 0;
- txvq->bytes = 0;
- txvq->errors = 0;
- txvq->multicast = 0;
- txvq->broadcast = 0;
- memset(txvq->size_bins, 0, sizeof(txvq->size_bins[0]) * 8);
+ txvq->stats.packets = 0;
+ txvq->stats.bytes = 0;
+ txvq->stats.errors = 0;
+ txvq->stats.multicast = 0;
+ txvq->stats.broadcast = 0;
+ memset(txvq->stats.size_bins, 0,
+ sizeof(txvq->stats.size_bins[0]) * 8);
}
for (i = 0; i < dev->data->nb_rx_queues; i++) {
- struct virtqueue *rxvq = dev->data->rx_queues[i];
+ struct virtnet_rx *rxvq = dev->data->rx_queues[i];
if (rxvq == NULL)
continue;
- rxvq->packets = 0;
- rxvq->bytes = 0;
- rxvq->errors = 0;
- rxvq->multicast = 0;
- rxvq->broadcast = 0;
- memset(rxvq->size_bins, 0, sizeof(rxvq->size_bins[0]) * 8);
+ rxvq->stats.packets = 0;
+ rxvq->stats.bytes = 0;
+ rxvq->stats.errors = 0;
+ rxvq->stats.multicast = 0;
+ rxvq->stats.broadcast = 0;
+ memset(rxvq->stats.size_bins, 0,
+ sizeof(rxvq->stats.size_bins[0]) * 8);
}
}
@@ -1184,7 +1122,8 @@ eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev)
eth_dev->tx_pkt_burst = NULL;
eth_dev->rx_pkt_burst = NULL;
- virtio_dev_queue_release(hw->cvq);
+ virtio_dev_queue_release(hw->cvq->vq);
+ rte_free(hw->cvq);
rte_free(eth_dev->data->mac_addrs);
eth_dev->data->mac_addrs = NULL;
@@ -1274,6 +1213,8 @@ virtio_dev_start(struct rte_eth_dev *dev)
uint16_t nb_queues, i;
struct virtio_hw *hw = dev->data->dev_private;
struct rte_pci_device *pci_dev = dev->pci_dev;
+ struct virtnet_rx *rxvq;
+ struct virtnet_tx *txvq __rte_unused;
/* check if lsc interrupt feature is enabled */
if (dev->data->dev_conf.intr_conf.lsc) {
@@ -1313,16 +1254,22 @@ virtio_dev_start(struct rte_eth_dev *dev)
PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues);
- for (i = 0; i < nb_queues; i++)
- virtqueue_notify(dev->data->rx_queues[i]);
+ for (i = 0; i < nb_queues; i++) {
+ rxvq = dev->data->rx_queues[i];
+ virtqueue_notify(rxvq->vq);
+ }
PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
- for (i = 0; i < dev->data->nb_rx_queues; i++)
- VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ rxvq = dev->data->rx_queues[i];
+ VIRTQUEUE_DUMP(rxvq->vq);
+ }
- for (i = 0; i < dev->data->nb_tx_queues; i++)
- VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
+ for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ txvq = dev->data->tx_queues[i];
+ VIRTQUEUE_DUMP(txvq->vq);
+ }
return 0;
}
diff --git a/drivers/net/virtio/virtio_pci.c b/drivers/net/virtio/virtio_pci.c
index c007959..45edecc 100644
--- a/drivers/net/virtio/virtio_pci.c
+++ b/drivers/net/virtio/virtio_pci.c
@@ -150,7 +150,7 @@ legacy_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
rte_eal_pci_ioport_write(&hw->io, &vq->vq_queue_index, 2,
VIRTIO_PCI_QUEUE_SEL);
- src = vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
+ src = vq->vq_ring_mem >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
rte_eal_pci_ioport_write(&hw->io, &src, 4, VIRTIO_PCI_QUEUE_PFN);
}
@@ -373,7 +373,7 @@ modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
uint64_t desc_addr, avail_addr, used_addr;
uint16_t notify_off;
- desc_addr = vq->mz->phys_addr;
+ desc_addr = vq->vq_ring_mem;
avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
ring[vq->vq_nentries]),
diff --git a/drivers/net/virtio/virtio_pci.h b/drivers/net/virtio/virtio_pci.h
index b69785e..052a417 100644
--- a/drivers/net/virtio/virtio_pci.h
+++ b/drivers/net/virtio/virtio_pci.h
@@ -40,6 +40,7 @@
#include <rte_ethdev.h>
struct virtqueue;
+struct virtnet_ctl;
/* VirtIO PCI vendor/device ID. */
#define VIRTIO_PCI_VENDORID 0x1AF4
@@ -242,7 +243,7 @@ struct virtio_pci_ops {
struct virtio_net_config;
struct virtio_hw {
- struct virtqueue *cvq;
+ struct virtnet_ctl *cvq;
struct rte_pci_ioport io;
uint64_t guest_features;
uint32_t max_tx_queues;
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index ef21d8e..541d124 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -209,23 +209,24 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
}
static inline void
-virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie,
+virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
uint16_t needed, int use_indirect, int can_push)
{
struct vq_desc_extra *dxp;
+ struct virtqueue *vq = txvq->vq;
struct vring_desc *start_dp;
uint16_t seg_num = cookie->nb_segs;
uint16_t head_idx, idx;
- uint16_t head_size = txvq->hw->vtnet_hdr_size;
+ uint16_t head_size = vq->hw->vtnet_hdr_size;
unsigned long offs;
- head_idx = txvq->vq_desc_head_idx;
+ head_idx = vq->vq_desc_head_idx;
idx = head_idx;
- dxp = &txvq->vq_descx[idx];
+ dxp = &vq->vq_descx[idx];
dxp->cookie = (void *)cookie;
dxp->ndescs = needed;
- start_dp = txvq->vq_ring.desc;
+ start_dp = vq->vq_ring.desc;
if (can_push) {
/* put on zero'd transmit header (no offloads) */
@@ -259,7 +260,7 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie,
+ offsetof(struct virtio_tx_region, tx_hdr);
start_dp[idx].addr = txvq->virtio_net_hdr_mem + offs;
- start_dp[idx].len = txvq->hw->vtnet_hdr_size;
+ start_dp[idx].len = vq->hw->vtnet_hdr_size;
start_dp[idx].flags = VRING_DESC_F_NEXT;
idx = start_dp[idx].next;
}
@@ -274,13 +275,13 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie,
start_dp[idx].flags &= ~VRING_DESC_F_NEXT;
if (use_indirect)
- idx = txvq->vq_ring.desc[head_idx].next;
+ idx = vq->vq_ring.desc[head_idx].next;
- txvq->vq_desc_head_idx = idx;
- if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
- txvq->vq_desc_tail_idx = idx;
- txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed);
- vq_update_avail_ring(txvq, head_idx);
+ vq->vq_desc_head_idx = idx;
+ if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
+ vq->vq_desc_tail_idx = idx;
+ vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
+ vq_update_avail_ring(vq, head_idx);
}
static inline struct rte_mbuf *
@@ -295,10 +296,9 @@ rte_rxmbuf_alloc(struct rte_mempool *mp)
}
static void
-virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
+virtio_dev_vring_start(struct virtqueue *vq)
{
- struct rte_mbuf *m;
- int i, nbufs, error, size = vq->vq_nentries;
+ int size = vq->vq_nentries;
struct vring *vr = &vq->vq_ring;
uint8_t *ring_mem = vq->vq_ring_virt_mem;
@@ -322,13 +322,44 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
* Disable device(host) interrupting guest
*/
virtqueue_disable_intr(vq);
+}
+
+void
+virtio_dev_cq_start(struct rte_eth_dev *dev)
+{
+ struct virtio_hw *hw = dev->data->dev_private;
+
+ if (hw->cvq && hw->cvq->vq) {
+ virtio_dev_vring_start(hw->cvq->vq);
+ VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq->vq);
+ }
+}
+
+void
+virtio_dev_rxtx_start(struct rte_eth_dev *dev)
+{
+ /*
+ * Start receive and transmit vrings
+ * - Setup vring structure for all queues
+ * - Initialize descriptor for the rx vring
+ * - Allocate blank mbufs for the each rx descriptor
+ *
+ */
+ int i;
+
+ PMD_INIT_FUNC_TRACE();
- /* Only rx virtqueue needs mbufs to be allocated at initialization */
- if (queue_type == VTNET_RQ) {
- if (vq->mpool == NULL)
+ /* Start rx vring. */
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ struct virtnet_rx *rxvq = dev->data->rx_queues[i];
+ struct virtqueue *vq = rxvq->vq;
+ int error, nbufs;
+ struct rte_mbuf *m;
+
+ virtio_dev_vring_start(vq);
+ if (rxvq->mpool == NULL)
rte_exit(EXIT_FAILURE,
"Cannot allocate initial mbufs for rx virtqueue");
-
/* Allocate blank mbufs for the each rx descriptor */
nbufs = 0;
error = ENOSPC;
@@ -340,12 +371,12 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
}
#endif
- memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf));
+ memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf));
for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++)
- vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf;
+ vq->sw_ring[vq->vq_nentries + i] = &rxvq->fake_mbuf;
while (!virtqueue_full(vq)) {
- m = rte_rxmbuf_alloc(vq->mpool);
+ m = rte_rxmbuf_alloc(rxvq->mpool);
if (m == NULL)
break;
@@ -368,7 +399,16 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
vq_update_avail_idx(vq);
PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
- } else if (queue_type == VTNET_TQ) {
+
+ VIRTQUEUE_DUMP(vq);
+ }
+
+ /* Start tx vring. */
+ for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ struct virtnet_tx *txvq = dev->data->tx_queues[i];
+ struct virtqueue *vq = txvq->vq;
+
+ virtio_dev_vring_start(vq);
#ifdef RTE_MACHINE_CPUFLAG_SSSE3
if (use_simple_rxtx) {
int mid_idx = vq->vq_nentries >> 1;
@@ -376,7 +416,7 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
vq->vq_ring.avail->ring[i] = i + mid_idx;
vq->vq_ring.desc[i + mid_idx].next = i;
vq->vq_ring.desc[i + mid_idx].addr =
- vq->virtio_net_hdr_mem +
+ txvq->virtio_net_hdr_mem +
offsetof(struct virtio_tx_region, tx_hdr);
vq->vq_ring.desc[i + mid_idx].len =
vq->hw->vtnet_hdr_size;
@@ -388,74 +428,125 @@ virtio_dev_vring_start(struct virtqueue *vq, int queue_type)
vq->vq_ring.avail->ring[i] = i;
}
#endif
+ VIRTQUEUE_DUMP(vq);
}
}
-void
-virtio_dev_cq_start(struct rte_eth_dev *dev)
+int virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
+ uint16_t queue_idx,
+ uint16_t nb_desc,
+ unsigned int socket_id,
+ __rte_unused const struct rte_eth_rxconf *rx_conf,
+ struct rte_mempool *mp)
{
+ char vq_name[VIRTQUEUE_MAX_NAME_SZ];
+ const struct rte_memzone *mz;
+ unsigned int vq_size, size;
struct virtio_hw *hw = dev->data->dev_private;
+ struct virtnet_rx *rxvq;
+ struct virtqueue *vq = NULL;
+ void *sw_ring = NULL;
+ uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
- if (hw->cvq) {
- virtio_dev_vring_start(hw->cvq, VTNET_CQ);
- VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq);
- }
-}
+ PMD_INIT_FUNC_TRACE();
+ PMD_INIT_LOG(DEBUG, "setting up rx queue: %u", vtpci_queue_idx);
-void
-virtio_dev_rxtx_start(struct rte_eth_dev *dev)
-{
/*
- * Start receive and transmit vrings
- * - Setup vring structure for all queues
- * - Initialize descriptor for the rx vring
- * - Allocate blank mbufs for the each rx descriptor
- *
+ * Read the virtqueue size from the Queue Size field
+ * Alaways power of 2 and if 0 virtqueue does not exist
*/
- int i;
-
- PMD_INIT_FUNC_TRACE();
+ vq_size = hw->vtpci_ops->get_queue_num(hw, vtpci_queue_idx);
+ PMD_INIT_LOG(DEBUG, "vq_size: %u nb_desc: %u", vq_size, nb_desc);
+ if (vq_size == 0) {
+ PMD_INIT_LOG(ERR, "virtqueue does not exist");
+ return -EINVAL;
+ }
- /* Start rx vring. */
- for (i = 0; i < dev->data->nb_rx_queues; i++) {
- virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ);
- VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
+ if (!rte_is_power_of_2(vq_size)) {
+ PMD_INIT_LOG(ERR, "virtqueue size is not powerof 2");
+ return -EINVAL;
}
- /* Start tx vring. */
- for (i = 0; i < dev->data->nb_tx_queues; i++) {
- virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ);
- VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
+ snprintf(vq_name, sizeof(vq_name), "port%d_rvq%d",
+ dev->data->port_id, queue_idx);
+ vq = rte_zmalloc_socket(vq_name,
+ sizeof(*vq) + vq_size * sizeof(struct vq_desc_extra) + \
+ sizeof(*rxvq),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ sw_ring = rte_zmalloc_socket("vq->sw_ring",
+ (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
+ sizeof(vq->sw_ring[0]),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (vq == NULL || sw_ring == NULL) {
+ PMD_INIT_LOG(ERR, "Can not allocate rxq");
+ rte_free(sw_ring);
+ rte_free(vq);
+ return -ENOMEM;
}
-}
+ rxvq = (struct virtnet_rx *)RTE_PTR_ADD(vq,
+ sizeof(*vq) + vq_size * sizeof(struct vq_desc_extra));
+ rxvq->vq = vq;
+ vq->sw_ring = sw_ring;
-int
-virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
- uint16_t queue_idx,
- uint16_t nb_desc,
- unsigned int socket_id,
- __rte_unused const struct rte_eth_rxconf *rx_conf,
- struct rte_mempool *mp)
-{
- uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
- struct virtqueue *vq;
- int ret;
+ vq->hw = hw;
+ rxvq->port_id = dev->data->port_id;
+ rxvq->queue_id = queue_idx;
+ vq->vq_queue_index = vtpci_queue_idx;
+ vq->vq_nentries = vq_size;
- PMD_INIT_FUNC_TRACE();
- ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx,
- nb_desc, socket_id, &vq);
- if (ret < 0) {
- PMD_INIT_LOG(ERR, "rvq initialization failed");
- return ret;
+ if (nb_desc == 0 || nb_desc > vq_size)
+ nb_desc = vq_size;
+ vq->vq_free_cnt = nb_desc;
+
+ /*
+ * Reserve a memzone for vring elements
+ */
+ size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
+ vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
+ PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", size,
+ vq->vq_ring_size);
+
+ mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
+ socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
+ if (mz == NULL) {
+ if (rte_errno == EEXIST)
+ mz = rte_memzone_lookup(vq_name);
+ if (mz == NULL) {
+ rte_free(sw_ring);
+ rte_free(vq);
+ return -ENOMEM;
+ }
}
- /* Create mempool for rx mbuf allocation */
- vq->mpool = mp;
+ /*
+ * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
+ * and only accepts 32 bit page frame number.
+ * Check if the allocated physical memory exceeds 16TB.
+ */
+ if ((mz->phys_addr + vq->vq_ring_size - 1) >>
+ (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
+ PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!");
+ rte_free(sw_ring);
+ rte_free(vq);
+ return -ENOMEM;
+ }
+
+ memset(mz->addr, 0, sizeof(mz->len));
+ rxvq->mz = mz;
+ vq->vq_ring_mem = mz->phys_addr;
+ vq->vq_ring_virt_mem = mz->addr;
+ PMD_INIT_LOG(DEBUG, "vq_ring_mem: 0x%"PRIx64,
+ (uint64_t)mz->phys_addr);
+ PMD_INIT_LOG(DEBUG, "vq_ring_virt_mem: 0x%"PRIx64,
+ (uint64_t)(uintptr_t)mz->addr);
+
+ hw->vtpci_ops->setup_queue(hw, vq);
- dev->data->rx_queues[queue_idx] = vq;
+ rxvq->mpool = mp;
+ dev->data->rx_queues[queue_idx] = rxvq;
#ifdef RTE_MACHINE_CPUFLAG_SSSE3
- virtio_rxq_vec_setup(vq);
+ virtio_rxq_vec_setup(rxvq);
#endif
return 0;
@@ -464,7 +555,11 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
void
virtio_dev_rx_queue_release(void *rxq)
{
- virtio_dev_queue_release(rxq);
+ struct virtqueue *vq = ((struct virtnet_rx *)rxq)->vq;
+
+ /* no need to free rxq as vq and rxq are allocated together */
+ virtio_dev_queue_release(vq);
+
}
/*
@@ -474,23 +569,28 @@ virtio_dev_rx_queue_release(void *rxq)
* const struct rte_eth_txconf *tx_conf: Used to setup tx engine
* uint16_t queue_idx: Just used as an index in dev txq list
*/
-int
-virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
+
+int virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
uint16_t queue_idx,
uint16_t nb_desc,
unsigned int socket_id,
const struct rte_eth_txconf *tx_conf)
{
- uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
-
+ char vq_name[VIRTQUEUE_MAX_NAME_SZ];
+ const struct rte_memzone *mz;
+ unsigned int vq_size, size;
+ uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
#ifdef RTE_MACHINE_CPUFLAG_SSSE3
struct virtio_hw *hw = dev->data->dev_private;
#endif
- struct virtqueue *vq;
+ struct virtqueue *vq = NULL;
+ struct virtnet_tx *txvq;
uint16_t tx_free_thresh;
- int ret;
+ struct virtio_tx_region *txr;
+ unsigned int i;
PMD_INIT_FUNC_TRACE();
+ PMD_INIT_LOG(DEBUG, "setting up tx queue: %u", vtpci_queue_idx);
if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMS)
!= ETH_TXQ_FLAGS_NOXSUMS) {
@@ -509,12 +609,121 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
}
#endif
- ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx,
- nb_desc, socket_id, &vq);
- if (ret < 0) {
- PMD_INIT_LOG(ERR, "rvq initialization failed");
- return ret;
+ /*
+ * Read the virtqueue size from the Queue Size field
+ * Always power of 2 and if 0 virtqueue does not exist
+ */
+ vq_size = hw->vtpci_ops->get_queue_num(hw, vtpci_queue_idx);
+ PMD_INIT_LOG(DEBUG, "vq_size: %u nb_desc:%u", vq_size, nb_desc);
+ if (vq_size == 0) {
+ PMD_INIT_LOG(ERR, "virtqueue does not exist");
+ return -EINVAL;
+ }
+
+ if (!rte_is_power_of_2(vq_size)) {
+ PMD_INIT_LOG(ERR, "virtqueue size is not powerof 2");
+ return -EINVAL;
+ }
+
+ snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d",
+ dev->data->port_id, queue_idx);
+ vq = rte_zmalloc_socket(vq_name,
+ sizeof(*vq) + vq_size * sizeof(struct vq_desc_extra) + \
+ sizeof(*txvq),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (vq == NULL) {
+ PMD_INIT_LOG(ERR, "Cannot allocate txvq");
+ return -ENOMEM;
}
+ txvq = (struct virtnet_tx *)RTE_PTR_ADD(vq,
+ sizeof(*vq) + vq_size * sizeof(struct vq_desc_extra));
+ txvq->vq = vq;
+
+ vq->hw = hw;
+ txvq->port_id = dev->data->port_id;
+ txvq->queue_id = queue_idx;
+ vq->vq_queue_index = vtpci_queue_idx;
+ vq->vq_nentries = vq_size;
+
+ if (nb_desc == 0 || nb_desc > vq_size)
+ nb_desc = vq_size;
+ vq->vq_free_cnt = nb_desc;
+
+ /*
+ * Reserve a memzone for vring elements
+ */
+ size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
+ vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
+ PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
+ size, vq->vq_ring_size);
+
+ mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
+ socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
+ if (mz == NULL) {
+ if (rte_errno == EEXIST)
+ mz = rte_memzone_lookup(vq_name);
+ if (mz == NULL) {
+ rte_free(vq);
+ return -ENOMEM;
+ }
+ }
+
+ /*
+ * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
+ * and only accepts 32 bit page frame number.
+ * Check if the allocated physical memory exceeds 16TB.
+ */
+ if ((mz->phys_addr + vq->vq_ring_size - 1) >>
+ (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
+ PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!");
+ rte_free(vq);
+ return -ENOMEM;
+ }
+
+ memset(mz->addr, 0, sizeof(mz->len));
+ txvq->mz = mz;
+ vq->vq_ring_mem = mz->phys_addr;
+ vq->vq_ring_virt_mem = mz->addr;
+ PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%"PRIx64,
+ (uint64_t)mz->phys_addr);
+ PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%"PRIx64,
+ (uint64_t)(uintptr_t)mz->addr);
+
+ /*
+ * For each xmit packet, allocate a virtio_net_hdr
+ */
+ snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d_hdrzone",
+ dev->data->port_id, queue_idx);
+ txvq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name,
+ vq_size * sizeof(*txr), socket_id, 0, RTE_CACHE_LINE_SIZE);
+ if (txvq->virtio_net_hdr_mz == NULL) {
+ if (rte_errno == EEXIST)
+ txvq->virtio_net_hdr_mz =
+ rte_memzone_lookup(vq_name);
+ if (txvq->virtio_net_hdr_mz == NULL) {
+ rte_free(vq);
+ return -ENOMEM;
+ }
+ }
+ txvq->virtio_net_hdr_mem =
+ txvq->virtio_net_hdr_mz->phys_addr;
+ txr = txvq->virtio_net_hdr_mz->addr;
+ memset(txr, 0, vq_size * sizeof(*txr));
+ for (i = 0; i < vq_size; i++) {
+ struct vring_desc *start_dp = txr[i].tx_indir;
+
+ vring_desc_init(start_dp, RTE_DIM(txr[i].tx_indir));
+
+ /* first indirect descriptor is always the tx header */
+ start_dp->addr = txvq->virtio_net_hdr_mem
+ + i * sizeof(*txr)
+ + offsetof(struct virtio_tx_region, tx_hdr);
+
+ start_dp->len = vq->hw->vtnet_hdr_size;
+ start_dp->flags = VRING_DESC_F_NEXT;
+ }
+
+ hw->vtpci_ops->setup_queue(hw, vq);
tx_free_thresh = tx_conf->tx_free_thresh;
if (tx_free_thresh == 0)
@@ -532,14 +741,17 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
vq->vq_free_thresh = tx_free_thresh;
- dev->data->tx_queues[queue_idx] = vq;
+ dev->data->tx_queues[queue_idx] = txvq;
return 0;
}
void
virtio_dev_tx_queue_release(void *txq)
{
- virtio_dev_queue_release(txq);
+ struct virtqueue *vq = ((struct virtnet_tx *)txq)->vq;
+
+ /* no need to free txq as vq and txq are allocated together */
+ virtio_dev_queue_release(vq);
}
static void
@@ -558,34 +770,66 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
}
static void
-virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf)
+virtio_update_rxq_stats(struct virtnet_rx *rxvq, struct rte_mbuf *mbuf)
{
uint32_t s = mbuf->pkt_len;
struct ether_addr *ea;
if (s == 64) {
- vq->size_bins[1]++;
+ rxvq->stats.size_bins[1]++;
} else if (s > 64 && s < 1024) {
uint32_t bin;
/* count zeros, and offset into correct bin */
bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
- vq->size_bins[bin]++;
+ rxvq->stats.size_bins[bin]++;
} else {
if (s < 64)
- vq->size_bins[0]++;
+ rxvq->stats.size_bins[0]++;
else if (s < 1519)
- vq->size_bins[6]++;
+ rxvq->stats.size_bins[6]++;
else if (s >= 1519)
- vq->size_bins[7]++;
+ rxvq->stats.size_bins[7]++;
}
ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *);
if (is_multicast_ether_addr(ea)) {
if (is_broadcast_ether_addr(ea))
- vq->broadcast++;
+ rxvq->stats.broadcast++;
else
- vq->multicast++;
+ rxvq->stats.multicast++;
+ }
+}
+
+static void
+virtio_update_txq_stats(struct virtnet_tx *txvq, struct rte_mbuf *mbuf)
+{
+ uint32_t s = mbuf->pkt_len;
+ struct ether_addr *ea;
+
+ if (s == 64) {
+ txvq->stats.size_bins[1]++;
+ } else if (s > 64 && s < 1024) {
+ uint32_t bin;
+
+ /* count zeros, and offset into correct bin */
+ bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
+ txvq->stats.size_bins[bin]++;
+ } else {
+ if (s < 64)
+ txvq->stats.size_bins[0]++;
+ else if (s < 1519)
+ txvq->stats.size_bins[6]++;
+ else if (s >= 1519)
+ txvq->stats.size_bins[7]++;
+ }
+
+ ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *);
+ if (is_multicast_ether_addr(ea)) {
+ if (is_broadcast_ether_addr(ea))
+ txvq->stats.broadcast++;
+ else
+ txvq->stats.multicast++;
}
}
@@ -594,7 +838,8 @@ virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf)
uint16_t
virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
- struct virtqueue *rxvq = rx_queue;
+ struct virtnet_rx *rxvq = rx_queue;
+ struct virtqueue *vq = rxvq->vq;
struct virtio_hw *hw;
struct rte_mbuf *rxm, *new_mbuf;
uint16_t nb_used, num, nb_rx;
@@ -604,19 +849,19 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
uint32_t i, nb_enqueued;
uint32_t hdr_size;
- nb_used = VIRTQUEUE_NUSED(rxvq);
+ nb_used = VIRTQUEUE_NUSED(vq);
virtio_rmb();
num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ);
if (likely(num > DESC_PER_CACHELINE))
- num = num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
+ num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
- num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num);
+ num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
- hw = rxvq->hw;
+ hw = vq->hw;
nb_rx = 0;
nb_enqueued = 0;
hdr_size = hw->vtnet_hdr_size;
@@ -629,8 +874,8 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
PMD_RX_LOG(ERR, "Packet drop");
nb_enqueued++;
- virtio_discard_rxbuf(rxvq, rxm);
- rxvq->errors++;
+ virtio_discard_rxbuf(vq, rxm);
+ rxvq->stats.errors++;
continue;
}
@@ -651,15 +896,15 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
rx_pkts[nb_rx++] = rxm;
- rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len;
- virtio_update_packet_stats(rxvq, rxm);
+ rxvq->stats.bytes += rx_pkts[nb_rx - 1]->pkt_len;
+ virtio_update_rxq_stats(rxvq, rxm);
}
- rxvq->packets += nb_rx;
+ rxvq->stats.packets += nb_rx;
/* Allocate new mbuf for the used descriptor */
error = ENOSPC;
- while (likely(!virtqueue_full(rxvq))) {
+ while (likely(!virtqueue_full(vq))) {
new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
if (unlikely(new_mbuf == NULL)) {
struct rte_eth_dev *dev
@@ -667,7 +912,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
dev->data->rx_mbuf_alloc_failed++;
break;
}
- error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
+ error = virtqueue_enqueue_recv_refill(vq, new_mbuf);
if (unlikely(error)) {
rte_pktmbuf_free(new_mbuf);
break;
@@ -676,10 +921,10 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
}
if (likely(nb_enqueued)) {
- vq_update_avail_idx(rxvq);
+ vq_update_avail_idx(vq);
- if (unlikely(virtqueue_kick_prepare(rxvq))) {
- virtqueue_notify(rxvq);
+ if (unlikely(virtqueue_kick_prepare(vq))) {
+ virtqueue_notify(vq);
PMD_RX_LOG(DEBUG, "Notified\n");
}
}
@@ -692,7 +937,8 @@ virtio_recv_mergeable_pkts(void *rx_queue,
struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
- struct virtqueue *rxvq = rx_queue;
+ struct virtnet_rx *rxvq = rx_queue;
+ struct virtqueue *vq = rxvq->vq;
struct virtio_hw *hw;
struct rte_mbuf *rxm, *new_mbuf;
uint16_t nb_used, num, nb_rx;
@@ -706,13 +952,13 @@ virtio_recv_mergeable_pkts(void *rx_queue,
uint32_t seg_res;
uint32_t hdr_size;
- nb_used = VIRTQUEUE_NUSED(rxvq);
+ nb_used = VIRTQUEUE_NUSED(vq);
virtio_rmb();
PMD_RX_LOG(DEBUG, "used:%d\n", nb_used);
- hw = rxvq->hw;
+ hw = vq->hw;
nb_rx = 0;
i = 0;
nb_enqueued = 0;
@@ -727,7 +973,7 @@ virtio_recv_mergeable_pkts(void *rx_queue,
if (nb_rx == nb_pkts)
break;
- num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1);
+ num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1);
if (num != 1)
continue;
@@ -741,8 +987,8 @@ virtio_recv_mergeable_pkts(void *rx_queue,
if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) {
PMD_RX_LOG(ERR, "Packet drop\n");
nb_enqueued++;
- virtio_discard_rxbuf(rxvq, rxm);
- rxvq->errors++;
+ virtio_discard_rxbuf(vq, rxm);
+ rxvq->stats.errors++;
continue;
}
@@ -773,9 +1019,9 @@ virtio_recv_mergeable_pkts(void *rx_queue,
*/
uint16_t rcv_cnt =
RTE_MIN(seg_res, RTE_DIM(rcv_pkts));
- if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) {
+ if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) {
uint32_t rx_num =
- virtqueue_dequeue_burst_rx(rxvq,
+ virtqueue_dequeue_burst_rx(vq,
rcv_pkts, len, rcv_cnt);
i += rx_num;
rcv_cnt = rx_num;
@@ -783,8 +1029,8 @@ virtio_recv_mergeable_pkts(void *rx_queue,
PMD_RX_LOG(ERR,
"No enough segments for packet.\n");
nb_enqueued++;
- virtio_discard_rxbuf(rxvq, rxm);
- rxvq->errors++;
+ virtio_discard_rxbuf(vq, rxm);
+ rxvq->stats.errors++;
break;
}
@@ -814,16 +1060,16 @@ virtio_recv_mergeable_pkts(void *rx_queue,
VIRTIO_DUMP_PACKET(rx_pkts[nb_rx],
rx_pkts[nb_rx]->data_len);
- rxvq->bytes += rx_pkts[nb_rx]->pkt_len;
- virtio_update_packet_stats(rxvq, rx_pkts[nb_rx]);
+ rxvq->stats.bytes += rx_pkts[nb_rx]->pkt_len;
+ virtio_update_rxq_stats(rxvq, rx_pkts[nb_rx]);
nb_rx++;
}
- rxvq->packets += nb_rx;
+ rxvq->stats.packets += nb_rx;
/* Allocate new mbuf for the used descriptor */
error = ENOSPC;
- while (likely(!virtqueue_full(rxvq))) {
+ while (likely(!virtqueue_full(vq))) {
new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
if (unlikely(new_mbuf == NULL)) {
struct rte_eth_dev *dev
@@ -831,7 +1077,7 @@ virtio_recv_mergeable_pkts(void *rx_queue,
dev->data->rx_mbuf_alloc_failed++;
break;
}
- error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf);
+ error = virtqueue_enqueue_recv_refill(vq, new_mbuf);
if (unlikely(error)) {
rte_pktmbuf_free(new_mbuf);
break;
@@ -840,10 +1086,10 @@ virtio_recv_mergeable_pkts(void *rx_queue,
}
if (likely(nb_enqueued)) {
- vq_update_avail_idx(rxvq);
+ vq_update_avail_idx(vq);
- if (unlikely(virtqueue_kick_prepare(rxvq))) {
- virtqueue_notify(rxvq);
+ if (unlikely(virtqueue_kick_prepare(vq))) {
+ virtqueue_notify(vq);
PMD_RX_LOG(DEBUG, "Notified");
}
}
@@ -854,8 +1100,9 @@ virtio_recv_mergeable_pkts(void *rx_queue,
uint16_t
virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
- struct virtqueue *txvq = tx_queue;
- struct virtio_hw *hw = txvq->hw;
+ struct virtnet_tx *txvq = tx_queue;
+ struct virtqueue *vq = txvq->vq;
+ struct virtio_hw *hw = vq->hw;
uint16_t hdr_size = hw->vtnet_hdr_size;
uint16_t nb_used, nb_tx;
int error;
@@ -864,11 +1111,11 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
return nb_pkts;
PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
- nb_used = VIRTQUEUE_NUSED(txvq);
+ nb_used = VIRTQUEUE_NUSED(vq);
virtio_rmb();
- if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh))
- virtio_xmit_cleanup(txvq, nb_used);
+ if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
+ virtio_xmit_cleanup(vq, nb_used);
for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
struct rte_mbuf *txm = tx_pkts[nb_tx];
@@ -901,16 +1148,16 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
* default => number of segments + 1
*/
slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
- need = slots - txvq->vq_free_cnt;
+ need = slots - vq->vq_free_cnt;
/* Positive value indicates it need free vring descriptors */
if (unlikely(need > 0)) {
- nb_used = VIRTQUEUE_NUSED(txvq);
+ nb_used = VIRTQUEUE_NUSED(vq);
virtio_rmb();
need = RTE_MIN(need, (int)nb_used);
- virtio_xmit_cleanup(txvq, need);
- need = slots - txvq->vq_free_cnt;
+ virtio_xmit_cleanup(vq, need);
+ need = slots - vq->vq_free_cnt;
if (unlikely(need > 0)) {
PMD_TX_LOG(ERR,
"No free tx descriptors to transmit");
@@ -921,17 +1168,17 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
/* Enqueue Packet buffers */
virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, can_push);
- txvq->bytes += txm->pkt_len;
- virtio_update_packet_stats(txvq, txm);
+ txvq->stats.bytes += txm->pkt_len;
+ virtio_update_txq_stats(txvq, txm);
}
- txvq->packets += nb_tx;
+ txvq->stats.packets += nb_tx;
if (likely(nb_tx)) {
- vq_update_avail_idx(txvq);
+ vq_update_avail_idx(vq);
- if (unlikely(virtqueue_kick_prepare(txvq))) {
- virtqueue_notify(txvq);
+ if (unlikely(virtqueue_kick_prepare(vq))) {
+ virtqueue_notify(vq);
PMD_TX_LOG(DEBUG, "Notified backend after xmit");
}
}
diff --git a/drivers/net/virtio/virtio_rxtx.h b/drivers/net/virtio/virtio_rxtx.h
index a76c3e5..ced55a3 100644
--- a/drivers/net/virtio/virtio_rxtx.h
+++ b/drivers/net/virtio/virtio_rxtx.h
@@ -34,7 +34,59 @@
#define RTE_PMD_VIRTIO_RX_MAX_BURST 64
#ifdef RTE_MACHINE_CPUFLAG_SSSE3
-int virtio_rxq_vec_setup(struct virtqueue *rxq);
+
+struct virtnet_stats {
+ uint64_t packets;
+ uint64_t bytes;
+ uint64_t errors;
+ uint64_t multicast;
+ uint64_t broadcast;
+ /* Size bins in array as RFC 2819, undersized [0], 64 [1], etc */
+ uint64_t size_bins[8];
+};
+
+struct virtnet_rx {
+ struct virtqueue *vq;
+ /* dummy mbuf, for wraparound when processing RX ring. */
+ struct rte_mbuf fake_mbuf;
+ uint64_t mbuf_initializer; /**< value to init mbufs. */
+ struct rte_mempool *mpool; /**< mempool for mbuf allocation */
+
+ uint16_t queue_id; /**< DPDK queue index. */
+ uint8_t port_id; /**< Device port identifier. */
+
+ /* Statistics */
+ struct virtnet_stats stats;
+
+ const struct rte_memzone *mz; /**< mem zone to populate RX ring. */
+};
+
+struct virtnet_tx {
+ struct virtqueue *vq;
+ /**< memzone to populate hdr. */
+ const struct rte_memzone *virtio_net_hdr_mz;
+ phys_addr_t virtio_net_hdr_mem; /**< hdr for each xmit packet */
+
+ uint16_t queue_id; /**< DPDK queue index. */
+ uint8_t port_id; /**< Device port identifier. */
+
+ /* Statistics */
+ struct virtnet_stats stats;
+
+ const struct rte_memzone *mz; /**< mem zone to populate TX ring. */
+};
+
+struct virtnet_ctl {
+ struct virtqueue *vq;
+ /**< memzone to populate hdr. */
+ const struct rte_memzone *virtio_net_hdr_mz;
+ phys_addr_t virtio_net_hdr_mem; /**< hdr for each xmit packet */
+ uint8_t port_id; /**< Device port identifier. */
+ const struct rte_memzone *mz; /**< mem zone to populate RX ring. */
+};
+
+
+int virtio_rxq_vec_setup(struct virtnet_rx *rxvq);
int virtqueue_enqueue_recv_refill_simple(struct virtqueue *vq,
struct rte_mbuf *m);
diff --git a/drivers/net/virtio/virtio_rxtx_simple.c b/drivers/net/virtio/virtio_rxtx_simple.c
index 8f5293d..c6ec882 100644
--- a/drivers/net/virtio/virtio_rxtx_simple.c
+++ b/drivers/net/virtio/virtio_rxtx_simple.c
@@ -92,17 +92,18 @@ virtqueue_enqueue_recv_refill_simple(struct virtqueue *vq,
}
static inline void
-virtio_rxq_rearm_vec(struct virtqueue *rxvq)
+virtio_rxq_rearm_vec(struct virtnet_rx *rxvq)
{
int i;
uint16_t desc_idx;
struct rte_mbuf **sw_ring;
struct vring_desc *start_dp;
int ret;
+ struct virtqueue *vq = rxvq->vq;
- desc_idx = rxvq->vq_avail_idx & (rxvq->vq_nentries - 1);
- sw_ring = &rxvq->sw_ring[desc_idx];
- start_dp = &rxvq->vq_ring.desc[desc_idx];
+ desc_idx = vq->vq_avail_idx & (vq->vq_nentries - 1);
+ sw_ring = &vq->sw_ring[desc_idx];
+ start_dp = &vq->vq_ring.desc[desc_idx];
ret = rte_mempool_get_bulk(rxvq->mpool, (void **)sw_ring,
RTE_VIRTIO_VPMD_RX_REARM_THRESH);
@@ -120,14 +121,14 @@ virtio_rxq_rearm_vec(struct virtqueue *rxvq)
start_dp[i].addr =
(uint64_t)((uintptr_t)sw_ring[i]->buf_physaddr +
- RTE_PKTMBUF_HEADROOM - rxvq->hw->vtnet_hdr_size);
+ RTE_PKTMBUF_HEADROOM - vq->hw->vtnet_hdr_size);
start_dp[i].len = sw_ring[i]->buf_len -
- RTE_PKTMBUF_HEADROOM + rxvq->hw->vtnet_hdr_size;
+ RTE_PKTMBUF_HEADROOM + vq->hw->vtnet_hdr_size;
}
- rxvq->vq_avail_idx += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
- rxvq->vq_free_cnt -= RTE_VIRTIO_VPMD_RX_REARM_THRESH;
- vq_update_avail_idx(rxvq);
+ vq->vq_avail_idx += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
+ vq->vq_free_cnt -= RTE_VIRTIO_VPMD_RX_REARM_THRESH;
+ vq_update_avail_idx(vq);
}
/* virtio vPMD receive routine, only accept(nb_pkts >= RTE_VIRTIO_DESC_PER_LOOP)
@@ -143,7 +144,8 @@ uint16_t
virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
- struct virtqueue *rxvq = rx_queue;
+ struct virtnet_rx *rxvq = rx_queue;
+ struct virtqueue *vq = rxvq->vq;
uint16_t nb_used;
uint16_t desc_idx;
struct vring_used_elem *rused;
@@ -175,15 +177,15 @@ virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
len_adjust = _mm_set_epi16(
0, 0,
0,
- (uint16_t)-rxvq->hw->vtnet_hdr_size,
- 0, (uint16_t)-rxvq->hw->vtnet_hdr_size,
+ (uint16_t)-vq->hw->vtnet_hdr_size,
+ 0, (uint16_t)-vq->hw->vtnet_hdr_size,
0, 0);
if (unlikely(nb_pkts < RTE_VIRTIO_DESC_PER_LOOP))
return 0;
- nb_used = *(volatile uint16_t *)&rxvq->vq_ring.used->idx -
- rxvq->vq_used_cons_idx;
+ nb_used = *(volatile uint16_t *)&vq->vq_ring.used->idx -
+ vq->vq_used_cons_idx;
rte_compiler_barrier();
@@ -193,17 +195,17 @@ virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_VIRTIO_DESC_PER_LOOP);
nb_used = RTE_MIN(nb_used, nb_pkts);
- desc_idx = (uint16_t)(rxvq->vq_used_cons_idx & (rxvq->vq_nentries - 1));
- rused = &rxvq->vq_ring.used->ring[desc_idx];
- sw_ring = &rxvq->sw_ring[desc_idx];
- sw_ring_end = &rxvq->sw_ring[rxvq->vq_nentries];
+ desc_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
+ rused = &vq->vq_ring.used->ring[desc_idx];
+ sw_ring = &vq->sw_ring[desc_idx];
+ sw_ring_end = &vq->sw_ring[vq->vq_nentries];
_mm_prefetch((const void *)rused, _MM_HINT_T0);
- if (rxvq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
+ if (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
virtio_rxq_rearm_vec(rxvq);
- if (unlikely(virtqueue_kick_prepare(rxvq)))
- virtqueue_notify(rxvq);
+ if (unlikely(virtqueue_kick_prepare(vq)))
+ virtqueue_notify(vq);
}
for (nb_pkts_received = 0;
@@ -286,9 +288,9 @@ virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
}
}
- rxvq->vq_used_cons_idx += nb_pkts_received;
- rxvq->vq_free_cnt += nb_pkts_received;
- rxvq->packets += nb_pkts_received;
+ vq->vq_used_cons_idx += nb_pkts_received;
+ vq->vq_free_cnt += nb_pkts_received;
+ rxvq->stats.packets += nb_pkts_received;
return nb_pkts_received;
}
@@ -342,28 +344,29 @@ uint16_t
virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
- struct virtqueue *txvq = tx_queue;
+ struct virtnet_tx *txvq = tx_queue;
+ struct virtqueue *vq = txvq->vq;
uint16_t nb_used;
uint16_t desc_idx;
struct vring_desc *start_dp;
uint16_t nb_tail, nb_commit;
int i;
- uint16_t desc_idx_max = (txvq->vq_nentries >> 1) - 1;
+ uint16_t desc_idx_max = (vq->vq_nentries >> 1) - 1;
- nb_used = VIRTQUEUE_NUSED(txvq);
+ nb_used = VIRTQUEUE_NUSED(vq);
rte_compiler_barrier();
if (nb_used >= VIRTIO_TX_FREE_THRESH)
virtio_xmit_cleanup(tx_queue);
- nb_commit = nb_pkts = RTE_MIN((txvq->vq_free_cnt >> 1), nb_pkts);
- desc_idx = (uint16_t) (txvq->vq_avail_idx & desc_idx_max);
- start_dp = txvq->vq_ring.desc;
+ nb_commit = nb_pkts = RTE_MIN((vq->vq_free_cnt >> 1), nb_pkts);
+ desc_idx = (uint16_t) (vq->vq_avail_idx & desc_idx_max);
+ start_dp = vq->vq_ring.desc;
nb_tail = (uint16_t) (desc_idx_max + 1 - desc_idx);
if (nb_commit >= nb_tail) {
for (i = 0; i < nb_tail; i++)
- txvq->vq_descx[desc_idx + i].cookie = tx_pkts[i];
+ vq->vq_descx[desc_idx + i].cookie = tx_pkts[i];
for (i = 0; i < nb_tail; i++) {
start_dp[desc_idx].addr =
rte_mbuf_data_dma_addr(*tx_pkts);
@@ -375,7 +378,7 @@ virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
desc_idx = 0;
}
for (i = 0; i < nb_commit; i++)
- txvq->vq_descx[desc_idx + i].cookie = tx_pkts[i];
+ vq->vq_descx[desc_idx + i].cookie = tx_pkts[i];
for (i = 0; i < nb_commit; i++) {
start_dp[desc_idx].addr = rte_mbuf_data_dma_addr(*tx_pkts);
start_dp[desc_idx].len = (*tx_pkts)->pkt_len;
@@ -385,34 +388,34 @@ virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
rte_compiler_barrier();
- txvq->vq_free_cnt -= (uint16_t)(nb_pkts << 1);
- txvq->vq_avail_idx += nb_pkts;
- txvq->vq_ring.avail->idx = txvq->vq_avail_idx;
- txvq->packets += nb_pkts;
+ vq->vq_free_cnt -= (uint16_t)(nb_pkts << 1);
+ vq->vq_avail_idx += nb_pkts;
+ vq->vq_ring.avail->idx = vq->vq_avail_idx;
+ txvq->stats.packets += nb_pkts;
if (likely(nb_pkts)) {
- if (unlikely(virtqueue_kick_prepare(txvq)))
- virtqueue_notify(txvq);
+ if (unlikely(virtqueue_kick_prepare(vq)))
+ virtqueue_notify(vq);
}
return nb_pkts;
}
int __attribute__((cold))
-virtio_rxq_vec_setup(struct virtqueue *rxq)
+virtio_rxq_vec_setup(struct virtnet_rx *rxvq)
{
uintptr_t p;
struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
mb_def.nb_segs = 1;
mb_def.data_off = RTE_PKTMBUF_HEADROOM;
- mb_def.port = rxq->port_id;
+ mb_def.port = rxvq->port_id;
rte_mbuf_refcnt_set(&mb_def, 1);
/* prevent compiler reordering: rearm_data covers previous fields */
rte_compiler_barrier();
p = (uintptr_t)&mb_def.rearm_data;
- rxq->mbuf_initializer = *(uint64_t *)p;
+ rxvq->mbuf_initializer = *(uint64_t *)p;
return 0;
}
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 4e9239e..b56a99a 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -153,23 +153,29 @@ struct virtio_pmd_ctrl {
uint8_t data[VIRTIO_MAX_CTRL_DATA];
};
+struct vq_desc_extra {
+ void *cookie;
+ uint16_t ndescs;
+};
+
struct virtqueue {
- struct virtio_hw *hw; /**< virtio_hw structure pointer. */
- const struct rte_memzone *mz; /**< mem zone to populate RX ring. */
- const struct rte_memzone *virtio_net_hdr_mz; /**< memzone to populate hdr. */
- struct rte_mempool *mpool; /**< mempool for mbuf allocation */
- uint16_t queue_id; /**< DPDK queue index. */
- uint8_t port_id; /**< Device port identifier. */
- uint16_t vq_queue_index; /**< PCI queue index */
-
- void *vq_ring_virt_mem; /**< linear address of vring*/
+ struct virtio_hw *hw; /**< virtio_hw structure pointer. */
+ struct vring vq_ring; /**< vring keeping desc, used and avail */
+ /**
+ * Last consumed descriptor in the used table,
+ * trails vq_ring.used->idx.
+ */
+ uint16_t vq_used_cons_idx;
+ uint16_t vq_nentries; /**< vring desc numbers */
+ uint16_t vq_free_cnt; /**< num of desc available */
+ uint16_t vq_avail_idx; /**< sync until needed */
+ uint16_t vq_free_thresh; /**< free threshold */
+
+ void *vq_ring_virt_mem; /**< linear address of vring*/
unsigned int vq_ring_size;
- phys_addr_t vq_ring_mem; /**< physical address of vring */
- struct vring vq_ring; /**< vring keeping desc, used and avail */
- uint16_t vq_free_cnt; /**< num of desc available */
- uint16_t vq_nentries; /**< vring desc numbers */
- uint16_t vq_free_thresh; /**< free threshold */
+ phys_addr_t vq_ring_mem; /**< physical address of vring */
+
/**
* Head of the free chain in the descriptor table. If
* there are no free descriptors, this will be set to
@@ -177,36 +183,13 @@ struct virtqueue {
*/
uint16_t vq_desc_head_idx;
uint16_t vq_desc_tail_idx;
- /**
- * Last consumed descriptor in the used table,
- * trails vq_ring.used->idx.
- */
- uint16_t vq_used_cons_idx;
- uint16_t vq_avail_idx;
- uint64_t mbuf_initializer; /**< value to init mbufs. */
- phys_addr_t virtio_net_hdr_mem; /**< hdr for each xmit packet */
-
- struct rte_mbuf **sw_ring; /**< RX software ring. */
- /* dummy mbuf, for wraparound when processing RX ring. */
- struct rte_mbuf fake_mbuf;
-
- /* Statistics */
- uint64_t packets;
- uint64_t bytes;
- uint64_t errors;
- uint64_t multicast;
- uint64_t broadcast;
- /* Size bins in array as RFC 2819, undersized [0], 64 [1], etc */
- uint64_t size_bins[8];
-
- uint16_t *notify_addr;
-
- struct vq_desc_extra {
- void *cookie;
- uint16_t ndescs;
- } vq_descx[0];
+ uint16_t vq_queue_index; /**< PCI queue index */
+ uint16_t *notify_addr;
+ struct rte_mbuf **sw_ring; /**< RX software ring. */
+ struct vq_desc_extra vq_descx[0];
};
+
/* If multiqueue is provided by host, then we suppport it. */
#define VIRTIO_NET_CTRL_MQ 4
#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
--
1.8.1.4
More information about the dev
mailing list