[v12,06/10] net/virtio: implement receive path for packed queues

  Implement the receive part.

Signed-off-by: Jens Freimann <jfreimann@redhat.com>
Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
 drivers/net/virtio/virtio_ethdev.c |  61 +++--
 drivers/net/virtio/virtio_ethdev.h |   5 +
 drivers/net/virtio/virtio_rxtx.c   | 385 ++++++++++++++++++++++++++++-
 drivers/net/virtio/virtqueue.c     |  22 ++
 drivers/net/virtio/virtqueue.h     |   2 +-
 5 files changed, 444 insertions(+), 31 deletions(-)
  

On 12/13/18 1:34 PM, Jens Freimann wrote:
> Implement the receive part.
> 
> Signed-off-by: Jens Freimann <jfreimann@redhat.com>
> Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
> ---
>   drivers/net/virtio/virtio_ethdev.c |  61 +++--
>   drivers/net/virtio/virtio_ethdev.h |   5 +
>   drivers/net/virtio/virtio_rxtx.c   | 385 ++++++++++++++++++++++++++++-
>   drivers/net/virtio/virtqueue.c     |  22 ++
>   drivers/net/virtio/virtqueue.h     |   2 +-
>   5 files changed, 444 insertions(+), 31 deletions(-)
> 
> diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
> index 6fdc18245..381816adf 100644
> --- a/drivers/net/virtio/virtio_ethdev.c
> +++ b/drivers/net/virtio/virtio_ethdev.c
> @@ -1361,27 +1361,39 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
>   		}
>   	}
>   
> -	if (hw->use_simple_rx) {
> -		PMD_INIT_LOG(INFO, "virtio: using simple Rx path on port %u",
> -			eth_dev->data->port_id);
> -		eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
> -	} else if (hw->use_inorder_rx) {
> -		PMD_INIT_LOG(INFO,
> -			"virtio: using inorder mergeable buffer Rx path on port %u",
> -			eth_dev->data->port_id);
> -		eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts_inorder;
> -	} else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
> -		PMD_INIT_LOG(INFO,
> -			"virtio: using mergeable buffer Rx path on port %u",
> -			eth_dev->data->port_id);
> -		eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
> -	} else {
> -		PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
> -			eth_dev->data->port_id);
> -		eth_dev->rx_pkt_burst = &virtio_recv_pkts;
> +	if (vtpci_packed_queue(hw)) {
> +		if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
> +			PMD_INIT_LOG(INFO,
> +				"virtio: using packed ring mergeable buffer Rx path on port %u",
> +				eth_dev->data->port_id);
> +			eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts_packed;
> +		} else {
> +			PMD_INIT_LOG(INFO,
> +				"virtio: using packed ring standard Rx path on port %u",
> +				eth_dev->data->port_id);
> +			eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
> +		}
> +	} else {
> +		if (hw->use_simple_rx) {
> +			PMD_INIT_LOG(INFO, "virtio: using simple Rx path on port %u",
> +				eth_dev->data->port_id);
> +			eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
> +		} else if (hw->use_inorder_rx) {
> +			PMD_INIT_LOG(INFO,
> +				"virtio: using inorder mergeable buffer Rx path on port %u",
> +				eth_dev->data->port_id);
> +			eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts_inorder;
> +		} else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
> +			PMD_INIT_LOG(INFO,
> +				"virtio: using mergeable buffer Rx path on port %u",
> +				eth_dev->data->port_id);
> +			eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
> +		} else {
> +			PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
> +				eth_dev->data->port_id);
> +			eth_dev->rx_pkt_burst = &virtio_recv_pkts;
> +		}
>   	}
> -
> -
>   }
>   
>   /* Only support 1:1 queue/interrupt mapping so far.
> @@ -1513,7 +1525,8 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
>   
>   	/* Setting up rx_header size for the device */
>   	if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) ||
> -	    vtpci_with_feature(hw, VIRTIO_F_VERSION_1))
> +	    vtpci_with_feature(hw, VIRTIO_F_VERSION_1) ||
> +	    vtpci_with_feature(hw, VIRTIO_F_RING_PACKED))
>   		hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
>   	else
>   		hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);

Shouldn't this change be in patch 1?

> @@ -1942,6 +1955,12 @@ virtio_dev_configure(struct rte_eth_dev *dev)
>   		}
>   	}
>   
> +	if (vtpci_packed_queue(hw)) {
> +		hw->use_simple_rx = 0;
> +		hw->use_inorder_rx = 0;
> +		hw->use_inorder_tx = 0;
> +	}
> +
>   #if defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM
>   	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
>   		hw->use_simple_rx = 0;
> diff --git a/drivers/net/virtio/virtio_ethdev.h b/drivers/net/virtio/virtio_ethdev.h
> index 05d355180..5cf295418 100644
> --- a/drivers/net/virtio/virtio_ethdev.h
> +++ b/drivers/net/virtio/virtio_ethdev.h
> @@ -73,10 +73,15 @@ int virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
>   
>   uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
>   		uint16_t nb_pkts);
> +uint16_t virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts,
> +		uint16_t nb_pkts);
>   
>   uint16_t virtio_recv_mergeable_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
>   		uint16_t nb_pkts);
>   
> +uint16_t virtio_recv_mergeable_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts,
> +		uint16_t nb_pkts);
> +
>   uint16_t virtio_recv_mergeable_pkts_inorder(void *rx_queue,
>   		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
>   
> diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
> index d1c294e1b..db61c4197 100644
> --- a/drivers/net/virtio/virtio_rxtx.c
> +++ b/drivers/net/virtio/virtio_rxtx.c
> @@ -31,6 +31,7 @@
>   #include "virtqueue.h"
>   #include "virtio_rxtx.h"
>   #include "virtio_rxtx_simple.h"
> +#include "virtio_ring.h"
>   
>   #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
>   #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
> @@ -105,6 +106,47 @@ vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id)
>   	dxp->next = VQ_RING_DESC_CHAIN_END;
>   }
>   
> +static uint16_t
> +virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq,
> +				  struct rte_mbuf **rx_pkts,
> +				  uint32_t *len,
> +				  uint16_t num)
> +{
> +	struct rte_mbuf *cookie;
> +	uint16_t used_idx;
> +	uint16_t id;
> +	struct vring_packed_desc *desc;
> +	uint16_t i;
> +
> +	desc = vq->ring_packed.desc_packed;
> +
> +	for (i = 0; i < num; i++) {
> +		used_idx = vq->vq_used_cons_idx;
> +		if (!desc_is_used(&desc[used_idx], vq))
> +			return i;
> +		len[i] = desc[used_idx].len;
> +		id = desc[used_idx].id;
> +		cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
> +		if (unlikely(cookie == NULL)) {
> +			PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
> +				vq->vq_used_cons_idx);
> +			break;
> +		}
> +		rte_prefetch0(cookie);
> +		rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
> +		rx_pkts[i] = cookie;
> +
> +		vq->vq_free_cnt++;
> +		vq->vq_used_cons_idx++;
> +		if (vq->vq_used_cons_idx >= vq->vq_nentries) {
> +			vq->vq_used_cons_idx -= vq->vq_nentries;
> +			vq->used_wrap_counter ^= 1;
> +		}
> +	}
> +
> +	return i;
> +}
> +
>   static uint16_t
>   virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
>   			   uint32_t *len, uint16_t num)
> @@ -350,6 +392,50 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
>   	return 0;
>   }
>   
> +static inline int
> +virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, struct rte_mbuf **cookie,
> +				     uint16_t num)
> +{
> +	struct vring_packed_desc *start_dp = vq->ring_packed.desc_packed;
> +	uint16_t flags = VRING_DESC_F_WRITE | vq->avail_used_flags;
> +	struct virtio_hw *hw = vq->hw;
> +	struct vq_desc_extra *dxp;
> +	uint16_t idx;
> +	int i;
> +
> +	if (unlikely(vq->vq_free_cnt == 0))
> +		return -ENOSPC;
> +	if (unlikely(vq->vq_free_cnt < num))
> +		return -EMSGSIZE;
> +
> +	for (i = 0; i < num; i++) {
> +		idx = vq->vq_avail_idx;;
> +		dxp = &vq->vq_descx[idx];
> +		dxp->cookie = (void *) cookie[i];
> +		dxp->ndescs = 1;
> +
> +		start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) +
> +				RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
> +		start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
> +
> +		vq->vq_desc_head_idx = dxp->next;
> +		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
> +			vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
> +		rte_smp_wmb();
> +		start_dp[idx].flags = flags;
> +		if (++vq->vq_avail_idx >= vq->vq_nentries) {
> +			vq->vq_avail_idx -= vq->vq_nentries;
> +			vq->avail_wrap_counter ^= 1;
> +			vq->avail_used_flags =
> +				VRING_DESC_F_AVAIL(vq->avail_wrap_counter) |
> +				VRING_DESC_F_USED(!vq->avail_wrap_counter);
> +			flags = VRING_DESC_F_WRITE | vq->avail_used_flags;
> +		}
> +	}
> +	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
> +	return 0;
> +}
> +
>   /* When doing TSO, the IP length is not included in the pseudo header
>    * checksum of the packet given to the PMD, but for virtio it is
>    * expected.
> @@ -801,7 +887,10 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx)
>   				break;
>   
>   			/* Enqueue allocated buffers */
> -			error = virtqueue_enqueue_recv_refill(vq, m);
> +			if (vtpci_packed_queue(vq->hw))
> +				error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1);
> +			else
> +				error = virtqueue_enqueue_recv_refill(vq, m);
>   			if (error) {
>   				rte_pktmbuf_free(m);
>   				break;
> @@ -809,7 +898,8 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx)
>   			nbufs++;
>   		}
>   
> -		vq_update_avail_idx(vq);
> +		if (!vtpci_packed_queue(vq->hw))
> +			vq_update_avail_idx(vq);
>   	}
>   
>   	PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
> @@ -896,7 +986,10 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
>   	 * Requeue the discarded mbuf. This should always be
>   	 * successful since it was just dequeued.
>   	 */
> -	error = virtqueue_enqueue_recv_refill(vq, m);
> +	if (vtpci_packed_queue(vq->hw))
> +		error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1);
> +	else
> +		error = virtqueue_enqueue_recv_refill(vq, m);
>   
>   	if (unlikely(error)) {
>   		RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
> @@ -1136,6 +1229,103 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
>   	return nb_rx;
>   }
>   
> +uint16_t
> +virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
> +{
> +	struct virtnet_rx *rxvq = rx_queue;
> +	struct virtqueue *vq = rxvq->vq;
> +	struct virtio_hw *hw = vq->hw;
> +	struct rte_mbuf *rxm, *new_mbuf;
> +	uint16_t num, nb_rx;
> +	uint32_t len[VIRTIO_MBUF_BURST_SZ];
> +	struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
> +	int error;
> +	uint32_t i, nb_enqueued;
> +	uint32_t hdr_size;
> +	struct virtio_net_hdr *hdr;
> +
> +	nb_rx = 0;
> +	if (unlikely(hw->started == 0))
> +		return nb_rx;
> +
> +	num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts);
> +	if (likely(num > DESC_PER_CACHELINE))
> +		num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
> +
> +	num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
> +	PMD_RX_LOG(DEBUG, "dequeue:%d", num);
> +
> +	nb_enqueued = 0;
> +	hdr_size = hw->vtnet_hdr_size;
> +
> +	for (i = 0; i < num; i++) {
> +		rxm = rcv_pkts[i];
> +
> +		PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
> +
> +		if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
> +			PMD_RX_LOG(ERR, "Packet drop");
> +			nb_enqueued++;
> +			virtio_discard_rxbuf(vq, rxm);
> +			rxvq->stats.errors++;
> +			continue;
> +		}
> +
> +		rxm->port = rxvq->port_id;
> +		rxm->data_off = RTE_PKTMBUF_HEADROOM;
> +		rxm->ol_flags = 0;
> +		rxm->vlan_tci = 0;
> +
> +		rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
> +		rxm->data_len = (uint16_t)(len[i] - hdr_size);
> +
> +		hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
> +			RTE_PKTMBUF_HEADROOM - hdr_size);
> +
> +		if (hw->vlan_strip)
> +			rte_vlan_strip(rxm);
> +
> +		if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
> +			virtio_discard_rxbuf(vq, rxm);
> +			rxvq->stats.errors++;
> +			continue;
> +		}
> +
> +		virtio_rx_stats_updated(rxvq, rxm);
> +
> +		rx_pkts[nb_rx++] = rxm;
> +	}
> +
> +	rxvq->stats.packets += nb_rx;
> +
> +	/* Allocate new mbuf for the used descriptor */
> +	while (likely(!virtqueue_full(vq))) {
> +		new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool);
> +		if (unlikely(new_mbuf == NULL)) {
> +			struct rte_eth_dev *dev
> +				= &rte_eth_devices[rxvq->port_id];
> +			dev->data->rx_mbuf_alloc_failed++;
> +			break;
> +		}
> +		error = virtqueue_enqueue_recv_refill_packed(vq, &new_mbuf, 1);
> +		if (unlikely(error)) {
> +			rte_pktmbuf_free(new_mbuf);
> +			break;
> +		}
> +		nb_enqueued++;
> +	}
> +
> +	if (likely(nb_enqueued)) {
> +		if (unlikely(virtqueue_kick_prepare_packed(vq))) {
> +			virtqueue_notify(vq);
> +			PMD_RX_LOG(DEBUG, "Notified");
> +		}
> +	}
> +
> +	return nb_rx;
> +}
> +
> +
>   uint16_t
>   virtio_recv_mergeable_pkts_inorder(void *rx_queue,
>   			struct rte_mbuf **rx_pkts,
> @@ -1342,6 +1532,7 @@ virtio_recv_mergeable_pkts(void *rx_queue,
>   	uint16_t extra_idx;
>   	uint32_t seg_res;
>   	uint32_t hdr_size;
> +	uint32_t rx_num = 0;
>   
>   	nb_rx = 0;
>   	if (unlikely(hw->started == 0))
> @@ -1367,6 +1558,8 @@ virtio_recv_mergeable_pkts(void *rx_queue,
>   			break;
>   
>   		num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1);
> +		if (num == 0)
> +			return nb_rx;
>   		if (num != 1)
>   			continue;
>   
> @@ -1419,11 +1612,8 @@ virtio_recv_mergeable_pkts(void *rx_queue,
>   			uint16_t  rcv_cnt =
>   				RTE_MIN(seg_res, RTE_DIM(rcv_pkts));
>   			if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) {
> -				uint32_t rx_num =
> -					virtqueue_dequeue_burst_rx(vq,
> -					rcv_pkts, len, rcv_cnt);
> -				i += rx_num;
> -				rcv_cnt = rx_num;
> +				rx_num = virtqueue_dequeue_burst_rx(vq,
> +					      rcv_pkts, len, rcv_cnt);
>   			} else {
>   				PMD_RX_LOG(ERR,
>   					   "No enough segments for packet.");
> @@ -1432,6 +1622,8 @@ virtio_recv_mergeable_pkts(void *rx_queue,
>   				rxvq->stats.errors++;
>   				break;
>   			}
> +			i += rx_num;
> +			rcv_cnt = rx_num;
>   
>   			extra_idx = 0;
>   
> @@ -1483,7 +1675,6 @@ virtio_recv_mergeable_pkts(void *rx_queue,
>   
>   	if (likely(nb_enqueued)) {
>   		vq_update_avail_idx(vq);
> -
>   		if (unlikely(virtqueue_kick_prepare(vq))) {
>   			virtqueue_notify(vq);
>   			PMD_RX_LOG(DEBUG, "Notified");
> @@ -1493,6 +1684,182 @@ virtio_recv_mergeable_pkts(void *rx_queue,
>   	return nb_rx;
>   }

Why are you doing changes in split function?

> +uint16_t
> +virtio_recv_mergeable_pkts_packed(void *rx_queue,
> +			struct rte_mbuf **rx_pkts,
> +			uint16_t nb_pkts)
> +{
> +	struct virtnet_rx *rxvq = rx_queue;
> +	struct virtqueue *vq = rxvq->vq;
> +	struct virtio_hw *hw = vq->hw;
> +	struct rte_mbuf *rxm;
> +	struct rte_mbuf *prev = NULL;
> +	uint16_t num, nb_rx = 0;
> +	uint32_t len[VIRTIO_MBUF_BURST_SZ];
> +	struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
> +	uint32_t nb_enqueued = 0;
> +	uint32_t seg_num = 0;
> +	uint32_t seg_res = 0;
> +	uint32_t hdr_size = hw->vtnet_hdr_size;
> +	int32_t i;
> +	int error;
> +
> +	if (unlikely(hw->started == 0))
> +		return nb_rx;
> +
> +
> +	num = nb_pkts;
> +	if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
> +		num = VIRTIO_MBUF_BURST_SZ;
> +	if (likely(num > DESC_PER_CACHELINE))
> +		num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
> +
> +	num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
> +
> +	for (i = 0; i < num; i++) {
> +		struct virtio_net_hdr_mrg_rxbuf *header;
> +
> +		PMD_RX_LOG(DEBUG, "dequeue:%d", num);
> +		PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
> +
> +		rxm = rcv_pkts[i];
> +
> +		if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
> +			PMD_RX_LOG(ERR, "Packet drop");
> +			nb_enqueued++;
> +			virtio_discard_rxbuf(vq, rxm);
> +			rxvq->stats.errors++;
> +			continue;
> +		}
> +
> +		header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr +
> +			RTE_PKTMBUF_HEADROOM - hdr_size);
> +		seg_num = header->num_buffers;
> +
> +		if (seg_num == 0)
> +			seg_num = 1;
> +
> +		rxm->data_off = RTE_PKTMBUF_HEADROOM;
> +		rxm->nb_segs = seg_num;
> +		rxm->ol_flags = 0;
> +		rxm->vlan_tci = 0;
> +		rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
> +		rxm->data_len = (uint16_t)(len[i] - hdr_size);
> +
> +		rxm->port = rxvq->port_id;
> +		rx_pkts[nb_rx] = rxm;
> +		prev = rxm;
> +
> +		if (hw->has_rx_offload &&
> +				virtio_rx_offload(rxm, &header->hdr) < 0) {
> +			virtio_discard_rxbuf(vq, rxm);
> +			rxvq->stats.errors++;
> +			continue;
> +		}
> +		
> +		if (hw->vlan_strip)
> +			rte_vlan_strip(rx_pkts[nb_rx]);		
> +
> +		seg_res = seg_num - 1;
> +
> +		/* Merge remaining segments */
> +		while (seg_res != 0 && i < (num - 1)) {
> +			i++;
> +
> +			rxm = rcv_pkts[i];
> +			rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
> +			rxm->pkt_len = (uint32_t)(len[i]);
> +			rxm->data_len = (uint16_t)(len[i]);
> +
> +			rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
> +			rx_pkts[nb_rx]->data_len += (uint16_t)(len[i]);
> +
> +			if (prev)
> +				prev->next = rxm;
> +
> +			prev = rxm;
> +			seg_res -= 1;
> +		}
> +
> +		if (!seg_res) {
> +			virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
> +			nb_rx++;
> +		}
> +	}
> +
> +	/* Last packet still need merge segments */
> +	while (seg_res != 0) {
> +		uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
> +					VIRTIO_MBUF_BURST_SZ);
> +		if (likely(vq->vq_free_cnt >= rcv_cnt)) {
> +			num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len,
> +							   rcv_cnt);
> +			uint16_t extra_idx = 0;
> +
> +			rcv_cnt = num;
> +
> +			while (extra_idx < rcv_cnt) {
> +				rxm = rcv_pkts[extra_idx];
> +
> +				rxm->data_off =
> +					RTE_PKTMBUF_HEADROOM - hdr_size;
> +				rxm->pkt_len = (uint32_t)(len[extra_idx]);
> +				rxm->data_len = (uint16_t)(len[extra_idx]);
> +
> +				prev->next = rxm;
> +				prev = rxm;
> +				rx_pkts[nb_rx]->pkt_len += len[extra_idx];
> +				rx_pkts[nb_rx]->data_len += len[extra_idx];
> +				extra_idx += 1;
> +			}
> +			seg_res -= rcv_cnt;
> +			if (!seg_res) {
> +				virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
> +				nb_rx++;
> +			}
> +		} else {
> +			PMD_RX_LOG(ERR,
> +					"No enough segments for packet.");
> +			if (prev)
> +				virtio_discard_rxbuf(vq, prev);
> +			rxvq->stats.errors++;
> +			break;
> +		}
> +	}
> +
> +	rxvq->stats.packets += nb_rx;
> +
> +	/* Allocate new mbuf for the used descriptor */
> +	if (likely(!virtqueue_full(vq))) {
> +		/* free_cnt may include mrg descs */
> +		uint16_t free_cnt = vq->vq_free_cnt;
> +		struct rte_mbuf *new_pkts[free_cnt];
> +
> +		if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
> +			error = virtqueue_enqueue_recv_refill_packed(vq, new_pkts,
> +					free_cnt);
> +			if (unlikely(error)) {
> +				for (i = 0; i < free_cnt; i++)
> +					rte_pktmbuf_free(new_pkts[i]);
> +			}
> +			nb_enqueued += free_cnt;
> +		} else {
> +			struct rte_eth_dev *dev =
> +				&rte_eth_devices[rxvq->port_id];
> +			dev->data->rx_mbuf_alloc_failed += free_cnt;
> +		}
> +	}
> +
> +	if (likely(nb_enqueued)) {
> +		if (unlikely(virtqueue_kick_prepare_packed(vq))) {
> +			virtqueue_notify(vq);
> +			PMD_RX_LOG(DEBUG, "Notified");
> +		}
> +	}
> +
> +	return nb_rx;
> +}
> +
>   uint16_t
>   virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
>   {
> diff --git a/drivers/net/virtio/virtqueue.c b/drivers/net/virtio/virtqueue.c
> index 56a77cc71..f6dbb7d82 100644
> --- a/drivers/net/virtio/virtqueue.c
> +++ b/drivers/net/virtio/virtqueue.c
> @@ -65,6 +65,28 @@ virtqueue_rxvq_flush(struct virtqueue *vq)
>   	uint16_t used_idx, desc_idx;
>   	uint16_t nb_used, i;
>   
> +	if (vtpci_packed_queue(vq->hw)) {
> +		struct vring_packed_desc *descs = vq->ring_packed.desc_packed;
> +		int cnt = 0;
> +
> +		i = vq->vq_used_cons_idx;
> +		while (desc_is_used(&descs[i], vq) && cnt++ < vq->vq_nentries) {
> +			dxp = &vq->vq_descx[descs[i].id];
> +			if (dxp->cookie != NULL) {
> +				rte_pktmbuf_free(dxp->cookie);
> +				dxp->cookie = NULL;
> +			}
> +			vq->vq_free_cnt++;
> +			vq->vq_used_cons_idx++;
> +			if (vq->vq_used_cons_idx >= vq->vq_nentries) {
> +				vq->vq_used_cons_idx -= vq->vq_nentries;
> +				vq->used_wrap_counter ^= 1;
> +			}
> +			i = vq->vq_used_cons_idx;
> +		}
> +		return;
> +	}
> +

I think I already asked it, but please create dedicated functions for
split and packed.

>   	nb_used = VIRTQUEUE_NUSED(vq);
>   
>   	for (i = 0; i < nb_used; i++) {
> diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
> index 5119818e1..bd8645019 100644
> --- a/drivers/net/virtio/virtqueue.h
> +++ b/drivers/net/virtio/virtqueue.h
> @@ -397,7 +397,7 @@ virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx)
>   #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx))
>   
>   void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx);
> -void vq_ring_free_chain_packed(struct virtqueue *vq, uint16_t desc_idx);
> +void vq_ring_free_chain_packed(struct virtqueue *vq, uint16_t used_idx);

Please squash this change in the original commit.

>   void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx,
>   			  uint16_t num);
>   
>