[dpdk-stable] [PATCH v16.11 LTS 3/6] vhost: add support for non-contiguous indirect descs tables
Maxime Coquelin
maxime.coquelin at redhat.com
Mon Apr 23 17:59:15 CEST 2018
This patch adds support for non-contiguous indirect descriptor
tables in VA space.
When it happens, which is unlikely, a table is allocated and the
non-contiguous content is copied into it.
Reported-by: Yongji Xie <xieyongji at baidu.com>
Signed-off-by: Maxime Coquelin <maxime.coquelin at redhat.com>
---
lib/librte_vhost/virtio_net.c | 105 +++++++++++++++++++++++++++++++++++++++---
1 file changed, 98 insertions(+), 7 deletions(-)
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index f1b1a4ec7..352f6ada0 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -45,6 +45,7 @@
#include <rte_sctp.h>
#include <rte_arp.h>
#include <rte_spinlock.h>
+#include <rte_malloc.h>
#include "vhost.h"
@@ -101,6 +102,44 @@ is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t qp_nb)
return (is_tx ^ (idx & 1)) == 0 && idx < qp_nb * VIRTIO_QNUM;
}
+static inline struct vring_desc *__attribute__((always_inline))
+alloc_copy_ind_table(struct virtio_net *dev, struct vring_desc *desc)
+{
+ struct vring_desc *idesc;
+ uint64_t src, dst;
+ uint64_t len, remain = desc->len;
+ uint64_t desc_addr = desc->addr;
+
+ idesc = rte_malloc(__func__, desc->len, 0);
+ if (unlikely(!idesc))
+ return 0;
+
+ dst = (uint64_t)(uintptr_t)idesc;
+
+ while (remain) {
+ len = remain;
+ src = gpa_to_vva(dev, desc_addr, &len);
+ if (unlikely(!src || !len)) {
+ rte_free(idesc);
+ return 0;
+ }
+
+ rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len);
+
+ remain -= len;
+ dst += len;
+ desc_addr += len;
+ }
+
+ return idesc;
+}
+
+static inline void __attribute__((always_inline))
+free_ind_table(struct vring_desc *idesc)
+{
+ rte_free(idesc);
+}
+
static inline void __attribute__((always_inline))
do_flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
uint16_t to, uint16_t from, uint16_t size)
@@ -351,6 +390,7 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
rte_prefetch0(&vq->desc[desc_indexes[0]]);
for (i = 0; i < count; i++) {
+ struct vring_desc *idesc = NULL;
uint16_t desc_idx = desc_indexes[i];
int err;
@@ -358,12 +398,24 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
dlen = vq->desc[desc_idx].len;
descs = (struct vring_desc *)(uintptr_t)gpa_to_vva(dev,
vq->desc[desc_idx].addr, &dlen);
- if (unlikely(!descs ||
- dlen != vq->desc[desc_idx].len)) {
+ if (unlikely(!descs)) {
count = i;
break;
}
+ if (unlikely(dlen < vq->desc[desc_idx].len)) {
+ /*
+ * The indirect desc table is not contiguous
+ * in process VA space, we have to copy it.
+ */
+ idesc = alloc_copy_ind_table(dev,
+ &vq->desc[desc_idx]);
+ if (unlikely(!idesc))
+ break;
+
+ descs = idesc;
+ }
+
desc_idx = 0;
sz = vq->desc[desc_idx].len / sizeof(*descs);
} else {
@@ -382,6 +434,9 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
if (i + 1 < count)
rte_prefetch0(&vq->desc[desc_indexes[i+1]]);
+
+ if (unlikely(!!idesc))
+ free_ind_table(idesc);
}
rte_smp_wmb();
@@ -417,6 +472,7 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
uint32_t len = 0;
uint64_t dlen;
struct vring_desc *descs = vq->desc;
+ struct vring_desc *idesc = NULL;
*desc_chain_head = idx;
@@ -425,15 +481,29 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
descs = (struct vring_desc *)(uintptr_t)
gpa_to_vva(dev, vq->desc[idx].addr,
&dlen);
- if (unlikely(!descs || dlen != vq->desc[idx].len))
+ if (unlikely(!descs))
return -1;
+ if (unlikely(dlen < vq->desc[idx].len)) {
+ /*
+ * The indirect desc table is not contiguous
+ * in process VA space, we have to copy it.
+ */
+ idesc = alloc_copy_ind_table(dev, &vq->desc[idx]);
+ if (unlikely(!idesc))
+ return -1;
+
+ descs = idesc;
+ }
+
idx = 0;
}
while (1) {
- if (unlikely(vec_id >= BUF_VECTOR_MAX || idx >= vq->size))
+ if (unlikely(vec_id >= BUF_VECTOR_MAX || idx >= vq->size)) {
+ free_ind_table(idesc);
return -1;
+ }
len += descs[idx].len;
buf_vec[vec_id].buf_addr = descs[idx].addr;
@@ -450,6 +520,9 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
*desc_chain_len = len;
*vec_idx = vec_id;
+ if (unlikely(!!idesc))
+ free_ind_table(idesc);
+
return 0;
}
@@ -1200,7 +1273,7 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
/* Prefetch descriptor index. */
rte_prefetch0(&vq->desc[desc_indexes[0]]);
for (i = 0; i < count; i++) {
- struct vring_desc *desc;
+ struct vring_desc *desc, *idesc = NULL;
uint16_t sz, idx;
uint64_t dlen;
int err;
@@ -1213,10 +1286,22 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
desc = (struct vring_desc *)(uintptr_t)gpa_to_vva(dev,
vq->desc[desc_indexes[i]].addr,
&dlen);
- if (unlikely(!desc ||
- dlen != vq->desc[desc_indexes[i]].len))
+ if (unlikely(!desc))
break;
+ if (unlikely(dlen < vq->desc[desc_indexes[i]].len)) {
+ /*
+ * The indirect desc table is not contiguous
+ * in process VA space, we have to copy it.
+ */
+ idesc = alloc_copy_ind_table(dev,
+ &vq->desc[desc_indexes[i]]);
+ if (unlikely(!idesc))
+ break;
+
+ desc = idesc;
+ }
+
rte_prefetch0(desc);
sz = vq->desc[desc_indexes[i]].len / sizeof(*desc);
idx = 0;
@@ -1230,12 +1315,14 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
if (unlikely(pkts[i] == NULL)) {
RTE_LOG(ERR, VHOST_DATA,
"Failed to allocate memory for mbuf.\n");
+ free_ind_table(idesc);
break;
}
err = copy_desc_to_mbuf(dev, desc, sz, pkts[i], idx, mbuf_pool);
if (unlikely(err)) {
rte_pktmbuf_free(pkts[i]);
+ free_ind_table(idesc);
break;
}
@@ -1245,6 +1332,7 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
zmbuf = get_zmbuf(vq);
if (!zmbuf) {
rte_pktmbuf_free(pkts[i]);
+ free_ind_table(idesc);
break;
}
zmbuf->mbuf = pkts[i];
@@ -1261,6 +1349,9 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
vq->nr_zmbuf += 1;
TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
}
+
+ if (unlikely(!!idesc))
+ free_ind_table(idesc);
}
vq->last_avail_idx += i;
--
2.14.3
More information about the stable
mailing list