[v7,05/10] vhost: checkout the resubmit inflight information
Checks
Commit Message
This patch shows how to checkout the inflight ring and construct
the resubmit information also include destroying resubmit info.
Signed-off-by: Lin Li <lilin24@baidu.com>
Signed-off-by: Xun Ni <nixun@baidu.com>
Signed-off-by: Yu Zhang <zhangyu31@baidu.com>
Signed-off-by: Jin Yu <jin.yu@intel.com>
---
lib/librte_vhost/rte_vhost.h | 19 +++
lib/librte_vhost/vhost.c | 29 ++++-
lib/librte_vhost/vhost.h | 9 ++
lib/librte_vhost/vhost_user.c | 212 +++++++++++++++++++++++++++++++++-
4 files changed, 267 insertions(+), 2 deletions(-)
Comments
On Fri, Sep 20, 2019 at 08:00:57PM +0800, Jin Yu wrote:
> @@ -1458,6 +1472,188 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
> return RTE_VHOST_MSG_RESULT_OK;
> }
>
> +static int
> +resubmit_desc_compare(const void *a, const void *b)
> +{
> + const struct rte_vhost_resubmit_desc *desc0 =
> + (const struct rte_vhost_resubmit_desc *)a;
> + const struct rte_vhost_resubmit_desc *desc1 =
> + (const struct rte_vhost_resubmit_desc *)b;
> +
> + if (desc1->counter > desc0->counter &&
> + (desc1->counter - desc0->counter) < VIRTQUEUE_MAX_SIZE * 2)
Why is this hardcoded to 1024 * 2 here?
> + return 1;
> +
> + return -1;
> +}
> +
> +static int
> +vhost_check_queue_inflights_split(struct virtio_net *dev,
> + struct vhost_virtqueue *vq)
> +{
> + uint16_t i = 0;
> + uint16_t resubmit_num = 0, last_io, num;
> + struct vring_used *used = vq->used;
> + struct rte_vhost_resubmit_info *resubmit = NULL;
> + struct rte_vhost_inflight_info_split *inflight_split;
> +
> + if (!(dev->protocol_features &
> + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
> + return RTE_VHOST_MSG_RESULT_OK;
> +
> + if ((!vq->inflight_split))
> + return RTE_VHOST_MSG_RESULT_ERR;
> +
> + if (!vq->inflight_split->version) {
> + vq->inflight_split->version = INFLIGHT_VERSION;
> + return RTE_VHOST_MSG_RESULT_OK;
> + }
> +
> + inflight_split = vq->inflight_split;
> + vq->resubmit_inflight = NULL;
This is invoked in vhost message handler, you need to
check whether it has been allocated first.
> + vq->global_counter = 0;
> + last_io = inflight_split->last_inflight_io;
> +
> + if (inflight_split->used_idx != used->idx) {
> + inflight_split->desc[last_io].inflight = 0;
> + rte_compiler_barrier();
> + inflight_split->used_idx = used->idx;
> + }
> +
> + for (i = 0; i < inflight_split->desc_num; i++) {
> + if (inflight_split->desc[i].inflight == 1)
> + resubmit_num++;
> + }
> +
> + vq->last_avail_idx += resubmit_num;
> +
> + if (resubmit_num) {
> + resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
> + if (!resubmit) {
> + RTE_LOG(ERR, VHOST_CONFIG,
> + "Failed to allocate memory for resubmit info.\n");
> + return RTE_VHOST_MSG_RESULT_ERR;
> + }
> +
> + resubmit->resubmit_list = calloc(resubmit_num,
> + sizeof(struct rte_vhost_resubmit_desc));
> + if (!resubmit->resubmit_list) {
> + RTE_LOG(ERR, VHOST_CONFIG,
> + "Failed to allocate memory for inflight desc.\n");
> + return RTE_VHOST_MSG_RESULT_ERR;
> + }
> +
> + num = 0;
> + for (i = 0; i < vq->inflight_split->desc_num; i++) {
> + if (vq->inflight_split->desc[i].inflight == 1) {
> + resubmit->resubmit_list[num].index = i;
> + resubmit->resubmit_list[num].counter =
> + inflight_split->desc[i].counter;
> + num++;
> + }
> + }
> + resubmit->resubmit_num = num;
> +
> + if (resubmit->resubmit_num > 1)
> + qsort(resubmit->resubmit_list, resubmit->resubmit_num,
> + sizeof(struct rte_vhost_resubmit_desc),
> + resubmit_desc_compare);
> +
> + vq->global_counter = resubmit->resubmit_list[0].counter + 1;
> + vq->resubmit_inflight = resubmit;
> + }
> +
> + return RTE_VHOST_MSG_RESULT_OK;
> +}
> +
> +static int
> +vhost_check_queue_inflights_packed(struct virtio_net *dev,
> + struct vhost_virtqueue *vq)
> +{
> + uint16_t i = 0;
> + uint16_t resubmit_num = 0, old_used_idx, num;
> + struct rte_vhost_resubmit_info *resubmit = NULL;
> + struct rte_vhost_inflight_info_packed *inflight_packed;
> +
> + if (!(dev->protocol_features &
> + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
> + return RTE_VHOST_MSG_RESULT_OK;
> +
> + if ((!vq->inflight_packed))
> + return RTE_VHOST_MSG_RESULT_ERR;
> +
> + if (!vq->inflight_packed->version) {
> + vq->inflight_packed->version = INFLIGHT_VERSION;
> + return RTE_VHOST_MSG_RESULT_OK;
> + }
> +
> + inflight_packed = vq->inflight_packed;
> + vq->resubmit_inflight = NULL;
Ditto.
> + vq->global_counter = 0;
> + old_used_idx = inflight_packed->old_used_idx;
> +
> + if (inflight_packed->used_idx != old_used_idx) {
> + if (inflight_packed->desc[old_used_idx].inflight == 0) {
> + inflight_packed->old_used_idx =
> + inflight_packed->used_idx;
> + inflight_packed->old_used_wrap_counter =
> + inflight_packed->used_wrap_counter;
> + inflight_packed->old_free_head =
> + inflight_packed->free_head;
> + } else {
> + inflight_packed->used_idx =
> + inflight_packed->old_used_idx;
> + inflight_packed->used_wrap_counter =
> + inflight_packed->old_used_wrap_counter;
> + inflight_packed->free_head =
> + inflight_packed->old_free_head;
> + }
> + }
> +
> + for (i = 0; i < inflight_packed->desc_num; i++) {
> + if (inflight_packed->desc[i].inflight == 1)
> + resubmit_num++;
> + }
> +
> + if (resubmit_num) {
> + resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
> + if (resubmit == NULL) {
> + RTE_LOG(ERR, VHOST_CONFIG,
> + "Failed to allocate memory for resubmit info.\n");
> + return RTE_VHOST_MSG_RESULT_ERR;
> + }
> +
> + resubmit->resubmit_list = calloc(resubmit_num,
> + sizeof(struct rte_vhost_resubmit_desc));
> + if (resubmit->resubmit_list == NULL) {
> + RTE_LOG(ERR, VHOST_CONFIG,
> + "Failed to allocate memory for resubmit desc.\n");
> + return RTE_VHOST_MSG_RESULT_ERR;
> + }
> +
> + num = 0;
> + for (i = 0; i < inflight_packed->desc_num; i++) {
> + if (vq->inflight_packed->desc[i].inflight == 1) {
> + resubmit->resubmit_list[num].index = i;
> + resubmit->resubmit_list[num].counter =
> + inflight_packed->desc[i].counter;
> + num++;
> + }
> + }
> + resubmit->resubmit_num = num;
> +
> + if (resubmit->resubmit_num > 1)
> + qsort(resubmit->resubmit_list, resubmit->resubmit_num,
> + sizeof(struct rte_vhost_resubmit_desc),
> + resubmit_desc_compare);
> +
> + vq->global_counter = resubmit->resubmit_list[0].counter + 1;
> + vq->resubmit_inflight = resubmit;
> + }
> +
> + return RTE_VHOST_MSG_RESULT_OK;
> +}
> +
> static int
> vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg,
> int main_fd __rte_unused)
> @@ -1499,6 +1695,20 @@ vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg,
> close(vq->kickfd);
> vq->kickfd = file.fd;
>
> + if (vq_is_packed(dev)) {
> + if (vhost_check_queue_inflights_packed(dev, vq)) {
> + RTE_LOG(ERR, VHOST_CONFIG,
> + "Failed to inflights for vq: %d\n", file.index);
> + return RTE_VHOST_MSG_RESULT_ERR;
> + }
> + } else {
> + if (vhost_check_queue_inflights_split(dev, vq)) {
> + RTE_LOG(ERR, VHOST_CONFIG,
> + "Failed to inflights for vq: %d\n", file.index);
> + return RTE_VHOST_MSG_RESULT_ERR;
> + }
> + }
> +
> return RTE_VHOST_MSG_RESULT_OK;
> }
>
> --
> 2.17.2
>
> -----Original Message-----
> From: Bie, Tiwei
> Sent: Thursday, September 26, 2019 4:05 PM
> To: Yu, Jin <jin.yu@intel.com>
> Cc: dev@dpdk.org; Liu, Changpeng <changpeng.liu@intel.com>;
> maxime.coquelin@redhat.com; Wang, Zhihong <zhihong.wang@intel.com>; Lin
> Li <lilin24@baidu.com>; Xun Ni <nixun@baidu.com>; Yu Zhang
> <zhangyu31@baidu.com>
> Subject: Re: [PATCH v7 05/10] vhost: checkout the resubmit inflight information
>
> On Fri, Sep 20, 2019 at 08:00:57PM +0800, Jin Yu wrote:
> > @@ -1458,6 +1472,188 @@ static int vhost_user_set_vring_err(struct
> virtio_net **pdev __rte_unused,
> > return RTE_VHOST_MSG_RESULT_OK;
> > }
> >
> > +static int
> > +resubmit_desc_compare(const void *a, const void *b) {
> > + const struct rte_vhost_resubmit_desc *desc0 =
> > + (const struct rte_vhost_resubmit_desc *)a;
> > + const struct rte_vhost_resubmit_desc *desc1 =
> > + (const struct rte_vhost_resubmit_desc *)b;
> > +
> > + if (desc1->counter > desc0->counter &&
> > + (desc1->counter - desc0->counter) < VIRTQUEUE_MAX_SIZE * 2)
>
> Why is this hardcoded to 1024 * 2 here?
I will check this. Thanks.
>
> > + return 1;
> > +
> > + return -1;
> > +}
> > +
> > +static int
> > +vhost_check_queue_inflights_split(struct virtio_net *dev,
> > + struct vhost_virtqueue *vq)
> > +{
> > + uint16_t i = 0;
> > + uint16_t resubmit_num = 0, last_io, num;
> > + struct vring_used *used = vq->used;
> > + struct rte_vhost_resubmit_info *resubmit = NULL;
> > + struct rte_vhost_inflight_info_split *inflight_split;
> > +
> > + if (!(dev->protocol_features &
> > + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
> > + return RTE_VHOST_MSG_RESULT_OK;
> > +
> > + if ((!vq->inflight_split))
> > + return RTE_VHOST_MSG_RESULT_ERR;
> > +
> > + if (!vq->inflight_split->version) {
> > + vq->inflight_split->version = INFLIGHT_VERSION;
> > + return RTE_VHOST_MSG_RESULT_OK;
> > + }
> > +
> > + inflight_split = vq->inflight_split;
> > + vq->resubmit_inflight = NULL;
>
> This is invoked in vhost message handler, you need to check whether it has been
> allocated first.
Got it, Thanks.
>
>
> > + vq->global_counter = 0;
> > + last_io = inflight_split->last_inflight_io;
> > +
> > + if (inflight_split->used_idx != used->idx) {
> > + inflight_split->desc[last_io].inflight = 0;
> > + rte_compiler_barrier();
> > + inflight_split->used_idx = used->idx;
> > + }
> > +
> > + for (i = 0; i < inflight_split->desc_num; i++) {
> > + if (inflight_split->desc[i].inflight == 1)
> > + resubmit_num++;
> > + }
> > +
> > + vq->last_avail_idx += resubmit_num;
> > +
> > + if (resubmit_num) {
> > + resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
> > + if (!resubmit) {
> > + RTE_LOG(ERR, VHOST_CONFIG,
> > + "Failed to allocate memory for resubmit info.\n");
> > + return RTE_VHOST_MSG_RESULT_ERR;
> > + }
> > +
> > + resubmit->resubmit_list = calloc(resubmit_num,
> > + sizeof(struct rte_vhost_resubmit_desc));
> > + if (!resubmit->resubmit_list) {
> > + RTE_LOG(ERR, VHOST_CONFIG,
> > + "Failed to allocate memory for inflight desc.\n");
> > + return RTE_VHOST_MSG_RESULT_ERR;
> > + }
> > +
> > + num = 0;
> > + for (i = 0; i < vq->inflight_split->desc_num; i++) {
> > + if (vq->inflight_split->desc[i].inflight == 1) {
> > + resubmit->resubmit_list[num].index = i;
> > + resubmit->resubmit_list[num].counter =
> > + inflight_split->desc[i].counter;
> > + num++;
> > + }
> > + }
> > + resubmit->resubmit_num = num;
> > +
> > + if (resubmit->resubmit_num > 1)
> > + qsort(resubmit->resubmit_list, resubmit-
> >resubmit_num,
> > + sizeof(struct rte_vhost_resubmit_desc),
> > + resubmit_desc_compare);
> > +
> > + vq->global_counter = resubmit->resubmit_list[0].counter + 1;
> > + vq->resubmit_inflight = resubmit;
> > + }
> > +
> > + return RTE_VHOST_MSG_RESULT_OK;
> > +}
> > +
> > +static int
> > +vhost_check_queue_inflights_packed(struct virtio_net *dev,
> > + struct vhost_virtqueue *vq)
> > +{
> > + uint16_t i = 0;
> > + uint16_t resubmit_num = 0, old_used_idx, num;
> > + struct rte_vhost_resubmit_info *resubmit = NULL;
> > + struct rte_vhost_inflight_info_packed *inflight_packed;
> > +
> > + if (!(dev->protocol_features &
> > + (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
> > + return RTE_VHOST_MSG_RESULT_OK;
> > +
> > + if ((!vq->inflight_packed))
> > + return RTE_VHOST_MSG_RESULT_ERR;
> > +
> > + if (!vq->inflight_packed->version) {
> > + vq->inflight_packed->version = INFLIGHT_VERSION;
> > + return RTE_VHOST_MSG_RESULT_OK;
> > + }
> > +
> > + inflight_packed = vq->inflight_packed;
> > + vq->resubmit_inflight = NULL;
>
> Ditto.
>
>
> > + vq->global_counter = 0;
> > + old_used_idx = inflight_packed->old_used_idx;
> > +
> > + if (inflight_packed->used_idx != old_used_idx) {
> > + if (inflight_packed->desc[old_used_idx].inflight == 0) {
> > + inflight_packed->old_used_idx =
> > + inflight_packed->used_idx;
> > + inflight_packed->old_used_wrap_counter =
> > + inflight_packed->used_wrap_counter;
> > + inflight_packed->old_free_head =
> > + inflight_packed->free_head;
> > + } else {
> > + inflight_packed->used_idx =
> > + inflight_packed->old_used_idx;
> > + inflight_packed->used_wrap_counter =
> > + inflight_packed->old_used_wrap_counter;
> > + inflight_packed->free_head =
> > + inflight_packed->old_free_head;
> > + }
> > + }
> > +
> > + for (i = 0; i < inflight_packed->desc_num; i++) {
> > + if (inflight_packed->desc[i].inflight == 1)
> > + resubmit_num++;
> > + }
> > +
> > + if (resubmit_num) {
> > + resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
> > + if (resubmit == NULL) {
> > + RTE_LOG(ERR, VHOST_CONFIG,
> > + "Failed to allocate memory for resubmit info.\n");
> > + return RTE_VHOST_MSG_RESULT_ERR;
> > + }
> > +
> > + resubmit->resubmit_list = calloc(resubmit_num,
> > + sizeof(struct rte_vhost_resubmit_desc));
> > + if (resubmit->resubmit_list == NULL) {
> > + RTE_LOG(ERR, VHOST_CONFIG,
> > + "Failed to allocate memory for resubmit desc.\n");
> > + return RTE_VHOST_MSG_RESULT_ERR;
> > + }
> > +
> > + num = 0;
> > + for (i = 0; i < inflight_packed->desc_num; i++) {
> > + if (vq->inflight_packed->desc[i].inflight == 1) {
> > + resubmit->resubmit_list[num].index = i;
> > + resubmit->resubmit_list[num].counter =
> > + inflight_packed->desc[i].counter;
> > + num++;
> > + }
> > + }
> > + resubmit->resubmit_num = num;
> > +
> > + if (resubmit->resubmit_num > 1)
> > + qsort(resubmit->resubmit_list, resubmit-
> >resubmit_num,
> > + sizeof(struct rte_vhost_resubmit_desc),
> > + resubmit_desc_compare);
> > +
> > + vq->global_counter = resubmit->resubmit_list[0].counter + 1;
> > + vq->resubmit_inflight = resubmit;
> > + }
> > +
> > + return RTE_VHOST_MSG_RESULT_OK;
> > +}
> > +
> > static int
> > vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg
> *msg,
> > int main_fd __rte_unused)
> > @@ -1499,6 +1695,20 @@ vhost_user_set_vring_kick(struct virtio_net
> **pdev, struct VhostUserMsg *msg,
> > close(vq->kickfd);
> > vq->kickfd = file.fd;
> >
> > + if (vq_is_packed(dev)) {
> > + if (vhost_check_queue_inflights_packed(dev, vq)) {
> > + RTE_LOG(ERR, VHOST_CONFIG,
> > + "Failed to inflights for vq: %d\n", file.index);
> > + return RTE_VHOST_MSG_RESULT_ERR;
> > + }
> > + } else {
> > + if (vhost_check_queue_inflights_split(dev, vq)) {
> > + RTE_LOG(ERR, VHOST_CONFIG,
> > + "Failed to inflights for vq: %d\n", file.index);
> > + return RTE_VHOST_MSG_RESULT_ERR;
> > + }
> > + }
> > +
> > return RTE_VHOST_MSG_RESULT_OK;
> > }
> >
> > --
> > 2.17.2
> >
@@ -145,6 +145,25 @@ struct rte_vhost_inflight_info_packed {
struct rte_vhost_inflight_desc_packed desc[0];
};
+struct rte_vhost_resubmit_desc {
+ uint16_t index;
+ uint64_t counter;
+};
+
+struct rte_vhost_resubmit_info {
+ struct rte_vhost_resubmit_desc *resubmit_list;
+ uint16_t resubmit_num;
+};
+
+struct rte_vhost_ring_inflight {
+ union {
+ struct rte_vhost_inflight_info_split *inflight_split;
+ struct rte_vhost_inflight_info_packed *inflight_packed;
+ };
+
+ struct rte_vhost_resubmit_info *resubmit_inflight;
+};
+
struct rte_vhost_vring {
union {
struct vring_desc *desc;
@@ -242,6 +242,31 @@ cleanup_vq(struct vhost_virtqueue *vq, int destroy)
close(vq->kickfd);
}
+void
+cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq)
+{
+ if (!(dev->protocol_features &
+ (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
+ return;
+
+ if (vq_is_packed(dev)) {
+ if (vq->inflight_packed)
+ vq->inflight_packed = NULL;
+ } else {
+ if (vq->inflight_split)
+ vq->inflight_split = NULL;
+ }
+
+ if (vq->resubmit_inflight) {
+ if (vq->resubmit_inflight->resubmit_list) {
+ free(vq->resubmit_inflight->resubmit_list);
+ vq->resubmit_inflight->resubmit_list = NULL;
+ }
+ free(vq->resubmit_inflight);
+ vq->resubmit_inflight = NULL;
+ }
+}
+
/*
* Unmap any memory, close any file descriptors and
* free any memory owned by a device.
@@ -253,8 +278,10 @@ cleanup_device(struct virtio_net *dev, int destroy)
vhost_backend_cleanup(dev);
- for (i = 0; i < dev->nr_vring; i++)
+ for (i = 0; i < dev->nr_vring; i++) {
cleanup_vq(dev->virtqueue[i], destroy);
+ cleanup_vq_inflight(dev, dev->virtqueue[i]);
+ }
}
void
@@ -128,6 +128,14 @@ struct vhost_virtqueue {
/* Physical address of used ring, for logging */
uint64_t log_guest_addr;
+ /* inflight share memory info */
+ union {
+ struct rte_vhost_inflight_info_split *inflight_split;
+ struct rte_vhost_inflight_info_packed *inflight_packed;
+ };
+ struct rte_vhost_resubmit_info *resubmit_inflight;
+ uint64_t global_counter;
+
uint16_t nr_zmbuf;
uint16_t zmbuf_size;
uint16_t last_zmbuf_idx;
@@ -474,6 +482,7 @@ void vhost_destroy_device(int);
void vhost_destroy_device_notify(struct virtio_net *dev);
void cleanup_vq(struct vhost_virtqueue *vq, int destroy);
+void cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq);
void free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq);
int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx);
@@ -336,6 +336,7 @@ vhost_user_set_features(struct virtio_net **pdev, struct VhostUserMsg *msg,
dev->virtqueue[dev->nr_vring] = NULL;
cleanup_vq(vq, 1);
+ cleanup_vq_inflight(dev, vq);
free_vq(dev, vq);
}
}
@@ -1353,11 +1354,12 @@ static int
vhost_user_set_inflight_fd(struct virtio_net **pdev, VhostUserMsg *msg,
int main_fd __rte_unused)
{
- int fd;
+ int fd, i;
uint64_t mmap_size, mmap_offset;
uint16_t num_queues, queue_size;
uint32_t pervq_inflight_size;
void *addr;
+ struct vhost_virtqueue *vq;
struct virtio_net *dev = *pdev;
fd = msg->fds[0];
@@ -1419,6 +1421,18 @@ vhost_user_set_inflight_fd(struct virtio_net **pdev, VhostUserMsg *msg,
dev->inflight_info->addr = addr;
dev->inflight_info->size = mmap_size;
+ for (i = 0; i < num_queues; i++) {
+ vq = dev->virtqueue[i];
+ if (vq_is_packed(dev)) {
+ vq->inflight_packed = addr;
+ vq->inflight_packed->desc_num = queue_size;
+ } else {
+ vq->inflight_split = addr;
+ vq->inflight_split->desc_num = queue_size;
+ }
+ addr = (void *)((char *)addr + pervq_inflight_size);
+ }
+
return RTE_VHOST_MSG_RESULT_OK;
}
@@ -1458,6 +1472,188 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,
return RTE_VHOST_MSG_RESULT_OK;
}
+static int
+resubmit_desc_compare(const void *a, const void *b)
+{
+ const struct rte_vhost_resubmit_desc *desc0 =
+ (const struct rte_vhost_resubmit_desc *)a;
+ const struct rte_vhost_resubmit_desc *desc1 =
+ (const struct rte_vhost_resubmit_desc *)b;
+
+ if (desc1->counter > desc0->counter &&
+ (desc1->counter - desc0->counter) < VIRTQUEUE_MAX_SIZE * 2)
+ return 1;
+
+ return -1;
+}
+
+static int
+vhost_check_queue_inflights_split(struct virtio_net *dev,
+ struct vhost_virtqueue *vq)
+{
+ uint16_t i = 0;
+ uint16_t resubmit_num = 0, last_io, num;
+ struct vring_used *used = vq->used;
+ struct rte_vhost_resubmit_info *resubmit = NULL;
+ struct rte_vhost_inflight_info_split *inflight_split;
+
+ if (!(dev->protocol_features &
+ (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
+ return RTE_VHOST_MSG_RESULT_OK;
+
+ if ((!vq->inflight_split))
+ return RTE_VHOST_MSG_RESULT_ERR;
+
+ if (!vq->inflight_split->version) {
+ vq->inflight_split->version = INFLIGHT_VERSION;
+ return RTE_VHOST_MSG_RESULT_OK;
+ }
+
+ inflight_split = vq->inflight_split;
+ vq->resubmit_inflight = NULL;
+ vq->global_counter = 0;
+ last_io = inflight_split->last_inflight_io;
+
+ if (inflight_split->used_idx != used->idx) {
+ inflight_split->desc[last_io].inflight = 0;
+ rte_compiler_barrier();
+ inflight_split->used_idx = used->idx;
+ }
+
+ for (i = 0; i < inflight_split->desc_num; i++) {
+ if (inflight_split->desc[i].inflight == 1)
+ resubmit_num++;
+ }
+
+ vq->last_avail_idx += resubmit_num;
+
+ if (resubmit_num) {
+ resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
+ if (!resubmit) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "Failed to allocate memory for resubmit info.\n");
+ return RTE_VHOST_MSG_RESULT_ERR;
+ }
+
+ resubmit->resubmit_list = calloc(resubmit_num,
+ sizeof(struct rte_vhost_resubmit_desc));
+ if (!resubmit->resubmit_list) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "Failed to allocate memory for inflight desc.\n");
+ return RTE_VHOST_MSG_RESULT_ERR;
+ }
+
+ num = 0;
+ for (i = 0; i < vq->inflight_split->desc_num; i++) {
+ if (vq->inflight_split->desc[i].inflight == 1) {
+ resubmit->resubmit_list[num].index = i;
+ resubmit->resubmit_list[num].counter =
+ inflight_split->desc[i].counter;
+ num++;
+ }
+ }
+ resubmit->resubmit_num = num;
+
+ if (resubmit->resubmit_num > 1)
+ qsort(resubmit->resubmit_list, resubmit->resubmit_num,
+ sizeof(struct rte_vhost_resubmit_desc),
+ resubmit_desc_compare);
+
+ vq->global_counter = resubmit->resubmit_list[0].counter + 1;
+ vq->resubmit_inflight = resubmit;
+ }
+
+ return RTE_VHOST_MSG_RESULT_OK;
+}
+
+static int
+vhost_check_queue_inflights_packed(struct virtio_net *dev,
+ struct vhost_virtqueue *vq)
+{
+ uint16_t i = 0;
+ uint16_t resubmit_num = 0, old_used_idx, num;
+ struct rte_vhost_resubmit_info *resubmit = NULL;
+ struct rte_vhost_inflight_info_packed *inflight_packed;
+
+ if (!(dev->protocol_features &
+ (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
+ return RTE_VHOST_MSG_RESULT_OK;
+
+ if ((!vq->inflight_packed))
+ return RTE_VHOST_MSG_RESULT_ERR;
+
+ if (!vq->inflight_packed->version) {
+ vq->inflight_packed->version = INFLIGHT_VERSION;
+ return RTE_VHOST_MSG_RESULT_OK;
+ }
+
+ inflight_packed = vq->inflight_packed;
+ vq->resubmit_inflight = NULL;
+ vq->global_counter = 0;
+ old_used_idx = inflight_packed->old_used_idx;
+
+ if (inflight_packed->used_idx != old_used_idx) {
+ if (inflight_packed->desc[old_used_idx].inflight == 0) {
+ inflight_packed->old_used_idx =
+ inflight_packed->used_idx;
+ inflight_packed->old_used_wrap_counter =
+ inflight_packed->used_wrap_counter;
+ inflight_packed->old_free_head =
+ inflight_packed->free_head;
+ } else {
+ inflight_packed->used_idx =
+ inflight_packed->old_used_idx;
+ inflight_packed->used_wrap_counter =
+ inflight_packed->old_used_wrap_counter;
+ inflight_packed->free_head =
+ inflight_packed->old_free_head;
+ }
+ }
+
+ for (i = 0; i < inflight_packed->desc_num; i++) {
+ if (inflight_packed->desc[i].inflight == 1)
+ resubmit_num++;
+ }
+
+ if (resubmit_num) {
+ resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
+ if (resubmit == NULL) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "Failed to allocate memory for resubmit info.\n");
+ return RTE_VHOST_MSG_RESULT_ERR;
+ }
+
+ resubmit->resubmit_list = calloc(resubmit_num,
+ sizeof(struct rte_vhost_resubmit_desc));
+ if (resubmit->resubmit_list == NULL) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "Failed to allocate memory for resubmit desc.\n");
+ return RTE_VHOST_MSG_RESULT_ERR;
+ }
+
+ num = 0;
+ for (i = 0; i < inflight_packed->desc_num; i++) {
+ if (vq->inflight_packed->desc[i].inflight == 1) {
+ resubmit->resubmit_list[num].index = i;
+ resubmit->resubmit_list[num].counter =
+ inflight_packed->desc[i].counter;
+ num++;
+ }
+ }
+ resubmit->resubmit_num = num;
+
+ if (resubmit->resubmit_num > 1)
+ qsort(resubmit->resubmit_list, resubmit->resubmit_num,
+ sizeof(struct rte_vhost_resubmit_desc),
+ resubmit_desc_compare);
+
+ vq->global_counter = resubmit->resubmit_list[0].counter + 1;
+ vq->resubmit_inflight = resubmit;
+ }
+
+ return RTE_VHOST_MSG_RESULT_OK;
+}
+
static int
vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg,
int main_fd __rte_unused)
@@ -1499,6 +1695,20 @@ vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg,
close(vq->kickfd);
vq->kickfd = file.fd;
+ if (vq_is_packed(dev)) {
+ if (vhost_check_queue_inflights_packed(dev, vq)) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "Failed to inflights for vq: %d\n", file.index);
+ return RTE_VHOST_MSG_RESULT_ERR;
+ }
+ } else {
+ if (vhost_check_queue_inflights_split(dev, vq)) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "Failed to inflights for vq: %d\n", file.index);
+ return RTE_VHOST_MSG_RESULT_ERR;
+ }
+ }
+
return RTE_VHOST_MSG_RESULT_OK;
}