[dpdk-dev] [PATCH v4 18/20] vhost: add event suppression for packed queues

[Jens Freimann]

Signed-off-by: Jens Freimann <jfreimann at redhat.com>
---
 lib/librte_vhost/vhost.c      | 15 +++++++++++
 lib/librte_vhost/vhost.h      | 63 ++++++++++++++++++++++++++++++++++++-------
 lib/librte_vhost/vhost_user.c | 22 +++++++++++++++
 lib/librte_vhost/virtio_net.c | 15 ++++++-----
 4 files changed, 98 insertions(+), 17 deletions(-)

diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index f7989cfbd..d07a8a347 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -23,6 +23,7 @@
 #include "iotlb.h"
 #include "vhost.h"
 #include "vhost_user.h"
+#include "virtio-packed.h"
 
 struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
 
@@ -577,10 +578,24 @@ int
 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
 {
 	struct virtio_net *dev = get_device(vid);
+	struct vhost_virtqueue *vq;
 
 	if (!dev)
 		return -1;
 
+	vq = dev->virtqueue[queue_id];
+	if (!vq->enabled)
+		return 0;
+
+	if (vq_is_packed(dev)) {
+		if (!enable) {
+			vq->driver_event->desc_event_flags |=
+				RING_EVENT_FLAGS_DISABLE;
+		} else
+			vq->driver_event->desc_event_flags |=
+				RING_EVENT_FLAGS_ENABLE;
+	}
+
 	if (enable)
 		dev->virtqueue[queue_id]->used->flags &=
 			~VRING_USED_F_NO_NOTIFY;
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 272d45f54..3e189f22a 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -69,14 +69,31 @@ struct batch_copy_elem {
 	uint64_t log_addr;
 };
 
+#define RING_EVENT_FLAGS_ENABLE 0x0
+#define RING_EVENT_FLAGS_DISABLE 0x1
+#define RING_EVENT_FLAGS_DESC 0x2
+#define RING_EVENT_FLAGS_MASK 0xFFFC
+#define RING_EVENT_WRAP_MASK 0x8000
+#define RING_EVENT_OFF_MASK 0x7FFF
+struct vring_packed_desc_event {
+	uint16_t desc_event_off_wrap;
+	uint16_t desc_event_flags;
+};
+
 /**
  * Structure contains variables relevant to RX/TX virtqueues.
  */
 struct vhost_virtqueue {
 	struct vring_desc	*desc;
 	struct vring_desc_packed   *desc_packed;
-	struct vring_avail	*avail;
-	struct vring_used	*used;
+	union {
+		struct vring_avail	*avail;
+		struct vring_packed_desc_event *driver_event;
+	};
+	union {
+		struct vring_used	*used;
+		struct vring_packed_desc_event *device_event;
+	};
 	uint32_t		size;
 
 	uint16_t		last_avail_idx;
@@ -210,7 +227,6 @@ struct vhost_msg {
 				(1ULL << VIRTIO_NET_F_MTU) | \
 				(1ULL << VIRTIO_F_IOMMU_PLATFORM))
 
-
 struct guest_page {
 	uint64_t guest_phys_addr;
 	uint64_t host_phys_addr;
@@ -475,6 +491,11 @@ vhost_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old)
 static __rte_always_inline void
 vhost_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq)
 {
+	uint16_t off_wrap, wrap = 0;
+	uint16_t event_flags;
+	uint16_t event_idx = 0;
+	int do_kick = 0;
+
 	/* Flush used->idx update before we read avail->flags. */
 	rte_mb();
 
@@ -482,22 +503,44 @@ vhost_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq)
 	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
 		uint16_t old = vq->signalled_used;
 		uint16_t new = vq->last_used_idx;
+		if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
+			event_flags = vq->driver_event->desc_event_flags &
+					RING_EVENT_FLAGS_MASK;
+			if (!(event_flags & RING_EVENT_FLAGS_DESC))
+				do_kick = event_flags & RING_EVENT_FLAGS_ENABLE ? 1 : 0;
+			else {
+				off_wrap = vq->driver_event->desc_event_off_wrap;
+				wrap = off_wrap & RING_EVENT_WRAP_MASK;
+				event_idx = off_wrap & RING_EVENT_OFF_MASK;
+			}
+			if (vhost_need_event(event_idx, new, old) &&
+					(vq->callfd >= 0) &&
+					(wrap == vq->used_wrap_counter)) {
+				vq->signalled_used = vq->last_used_idx;
+				do_kick = 1;
+			}
+		} else {
+			event_idx = vhost_used_event(vq);
+			if (vhost_need_event(event_idx, new, old)
+				&& (vq->callfd >= 0)) {
+				vq->signalled_used = vq->last_used_idx;
+				do_kick = 1;
+			}
+		}
 
 		VHOST_LOG_DEBUG(VHOST_DATA, "%s: used_event_idx=%d, old=%d, new=%d\n",
 			__func__,
-			vhost_used_event(vq),
+			event_idx,
 			old, new);
-		if (vhost_need_event(vhost_used_event(vq), new, old)
-			&& (vq->callfd >= 0)) {
-			vq->signalled_used = vq->last_used_idx;
-			eventfd_write(vq->callfd, (eventfd_t) 1);
-		}
 	} else {
 		/* Kick the guest if necessary. */
 		if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)
 				&& (vq->callfd >= 0))
-			eventfd_write(vq->callfd, (eventfd_t)1);
+			do_kick = 1;
 	}
+
+	if (do_kick)
+		eventfd_write(vq->callfd, (eventfd_t)1);
 }
 
 #endif /* _VHOST_NET_CDEV_H_ */
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 27b10c00c..c62544b96 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -480,6 +480,28 @@ translate_ring_addresses(struct virtio_net *dev, int vq_index)
 		vq->avail = NULL;
 		vq->used = NULL;
 		vq->log_guest_addr = 0;
+		vq->driver_event = (struct vring_packed_desc_event *)
+					(uintptr_t)ring_addr_to_vva(dev,
+					vq, addr->avail_user_addr,
+					sizeof(struct vring_packed_desc_event));
+		if (vq->driver_event == 0) {
+			RTE_LOG(DEBUG, VHOST_CONFIG,
+				"(%d) failed to find driver area address.\n",
+				dev->vid);
+			return dev;
+		}
+
+		vq->device_event = (struct vring_packed_desc_event *)
+					(uintptr_t)ring_addr_to_vva(dev,
+					vq, addr->used_user_addr,
+					sizeof(struct vring_packed_desc_event));
+		if (vq->device_event == 0) {
+			RTE_LOG(DEBUG, VHOST_CONFIG,
+				"(%d) failed to find device area address.\n",
+				dev->vid);
+			return dev;
+		}
+
 
 		if (vq->last_used_idx != 0) {
 			RTE_LOG(WARNING, VHOST_CONFIG,
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index e6e75f9a3..523e7d9f6 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -490,14 +490,14 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	uint32_t vec_id = *vec_idx;
 	uint32_t len    = 0;
 
-	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
+	if (vq_is_packed(dev))
 		idx = vq->last_avail_idx;
 	else
 		idx = vq->avail->ring[avail_idx & (vq->size - 1)];
 
 	*desc_chain_head = idx;
 
-	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
+	if (vq_is_packed(dev)) {
 		if (__fill_vec_buf_packed(dev, vq,
 				buf_vec, &len, &vec_id))
 			return -1;
@@ -533,7 +533,7 @@ reserve_avail_buf_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 	while (size > 0) {
 		if (unlikely(cur_idx == avail_head) &&
-			!(dev->features & (1ull < VIRTIO_F_RING_PACKED)))
+			!vq_is_packed(dev))
 			return -1;
 
 		if (unlikely(fill_vec_buf(dev, vq, cur_idx, &vec_idx, buf_vec,
@@ -718,7 +718,7 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 
 	vq->batch_copy_nb_elems = 0;
 
-	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
+	if (vq_is_packed(dev)) {
 		avail_head = vq->last_avail_idx;
 		descs = vq->desc_packed;
 	} else {
@@ -737,7 +737,7 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 				"(%d) failed to get enough desc from vring\n",
 				dev->vid);
 
-			if (!dev->features & (1ULL & VIRTIO_F_RING_PACKED))
+			if (!vq_is_packed(dev))
 				vq->shadow_used_idx -= num_buffers;
 			break;
 		}
@@ -748,7 +748,7 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 
 		if (copy_mbuf_to_desc_mergeable(dev, vq, pkts[pkt_idx],
 						buf_vec, num_buffers) < 0) {
-			if (!dev->features & (1ULL & VIRTIO_F_RING_PACKED))
+			if (!vq_is_packed(dev))
 				vq->shadow_used_idx -= num_buffers;
 			break;
 		}
@@ -758,7 +758,7 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 
 	do_data_copy_enqueue(dev, vq);
 
-	if (!(dev->features & (1ULL << VIRTIO_F_RING_PACKED))) {
+	if (!vq_is_packed(dev)) {
 		if (likely(vq->shadow_used_idx)) {
 			flush_shadow_used_ring(dev, vq);
 			vhost_vring_call(dev, vq);
@@ -769,6 +769,7 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 			if ((i & (vq->size - 1)) == 0)
 				toggle_wrap_counter(vq);
 			set_desc_used(vq, &descs[i & (vq->size - 1)]);
+			vhost_vring_call(dev, vq);
 		}
 	}
 
-- 
2.14.3