[dpdk-dev] [PATCH 3/4] net/virtio-user: support server mode
Zhiyong Yang
zhiyong.yang at intel.com
Wed Feb 14 15:53:29 CET 2018
virtio user adds to support for server mode.
Virtio user with server mode creates socket file and then starts to wait
for first connection from vhost user with client mode in blocking mode.
Server mode virtio user supports many times' vhost reconnections with
same configurations.
Support only one connection at the same time in server mode.
Signed-off-by: Zhiyong Yang <zhiyong.yang at intel.com>
---
drivers/net/virtio/virtio_ethdev.c | 9 ++-
drivers/net/virtio/virtio_user/vhost_user.c | 77 ++++++++++++++++++++--
drivers/net/virtio/virtio_user/virtio_user_dev.c | 44 +++++++++----
drivers/net/virtio/virtio_user_ethdev.c | 81 ++++++++++++++++++++++--
4 files changed, 186 insertions(+), 25 deletions(-)
diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
index 884f74ad0..44d037d6b 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1273,9 +1273,13 @@ static void
virtio_notify_peers(struct rte_eth_dev *dev)
{
struct virtio_hw *hw = dev->data->dev_private;
- struct virtnet_rx *rxvq = dev->data->rx_queues[0];
+ struct virtnet_rx *rxvq = NULL;
struct rte_mbuf *rarp_mbuf;
+ if (!dev->data->rx_queues)
+ return;
+
+ rxvq = dev->data->rx_queues[0];
rarp_mbuf = rte_net_make_rarp_packet(rxvq->mpool,
(struct ether_addr *)hw->mac_addr);
if (rarp_mbuf == NULL) {
@@ -1333,7 +1337,8 @@ virtio_interrupt_handler(void *param)
if (isr & VIRTIO_NET_S_ANNOUNCE) {
virtio_notify_peers(dev);
- virtio_ack_link_announce(dev);
+ if (hw->cvq)
+ virtio_ack_link_announce(dev);
}
}
diff --git a/drivers/net/virtio/virtio_user/vhost_user.c b/drivers/net/virtio/virtio_user/vhost_user.c
index 91c6449bb..fd806e106 100644
--- a/drivers/net/virtio/virtio_user/vhost_user.c
+++ b/drivers/net/virtio/virtio_user/vhost_user.c
@@ -378,6 +378,55 @@ vhost_user_sock(struct virtio_user_dev *dev,
return 0;
}
+static void
+virtio_user_set_block(int fd, bool enabled)
+{
+ int f;
+
+ f = fcntl(fd, F_GETFL);
+ if (enabled)
+ fcntl(fd, F_SETFL, f & ~O_NONBLOCK);
+ else
+ fcntl(fd, F_SETFL, f | O_NONBLOCK);
+}
+
+#define MAX_VIRTIO_USER_BACKLOG 128
+static int
+virtio_user_start_server(struct virtio_user_dev *dev, struct sockaddr_un *un)
+{
+ int ret;
+ int fd = dev->listenfd;
+ int connectfd;
+
+ ret = bind(fd, (struct sockaddr *)un, sizeof(*un));
+ if (ret < 0) {
+ PMD_DRV_LOG(ERR, "failed to bind to %s: %s; remove it and try again\n",
+ dev->path, strerror(errno));
+ goto err;
+ }
+ ret = listen(fd, MAX_VIRTIO_USER_BACKLOG);
+ if (ret < 0)
+ goto err;
+
+ virtio_user_set_block(fd, true);
+ PMD_DRV_LOG(NOTICE, "virtio user server mode is waiting for connection from vhost user.");
+ while (1) {
+ connectfd = accept(fd, NULL, NULL);
+ if (connectfd >= 0) {
+ dev->connected = true;
+ break;
+ }
+ }
+
+ dev->vhostfd = connectfd;
+ virtio_user_set_block(connectfd, true);
+
+ return 0;
+err:
+ close(fd);
+ return -1;
+}
+
/**
* Set up environment to talk with a vhost user backend.
*
@@ -390,6 +439,7 @@ vhost_user_setup(struct virtio_user_dev *dev)
{
int fd;
int flag;
+ int ret;
struct sockaddr_un un;
fd = socket(AF_UNIX, SOCK_STREAM, 0);
@@ -405,13 +455,30 @@ vhost_user_setup(struct virtio_user_dev *dev)
memset(&un, 0, sizeof(un));
un.sun_family = AF_UNIX;
snprintf(un.sun_path, sizeof(un.sun_path), "%s", dev->path);
- if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
- PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
- close(fd);
- return -1;
+
+ if (dev->is_server) {
+ static pthread_t fdset_tid;
+
+ dev->listenfd = fd;
+ if (fdset_tid == 0) {
+ ret = pthread_create(&fdset_tid, NULL,
+ fdset_event_dispatch,
+ &dev->fdset);
+ if (ret < 0)
+ PMD_DRV_LOG(ERR, "failed to create fdset handling thread");
+ }
+ return virtio_user_start_server(dev, &un);
+
+ } else {
+ dev->vhostfd = fd;
+ if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
+ PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
+ close(fd);
+ return -1;
+ }
+ dev->connected = true;
}
- dev->vhostfd = fd;
return 0;
}
diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c
index f90fee9e5..23312344f 100644
--- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
+++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
@@ -142,6 +142,9 @@ int virtio_user_stop_device(struct virtio_user_dev *dev)
{
uint32_t i;
+ if (!dev->connected)
+ return -1;
+
for (i = 0; i < dev->max_queue_pairs; ++i)
dev->ops->enable_qp(dev, i, 0);
@@ -267,21 +270,27 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
dev->vhostfds = NULL;
dev->tapfds = NULL;
- if (is_vhost_user_by_type(dev->path)) {
- dev->ops = &ops_user;
+ if (dev->is_server) {
+ dev->ops = &ops_user;/* server mode only supports vhost user */
} else {
- dev->ops = &ops_kernel;
-
- dev->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
- dev->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
- if (!dev->vhostfds || !dev->tapfds) {
- PMD_INIT_LOG(ERR, "Failed to malloc");
- return -1;
- }
-
- for (q = 0; q < dev->max_queue_pairs; ++q) {
- dev->vhostfds[q] = -1;
- dev->tapfds[q] = -1;
+ if (is_vhost_user_by_type(dev->path)) {
+ dev->ops = &ops_user;
+ } else {
+ dev->ops = &ops_kernel;
+
+ dev->vhostfds = malloc(dev->max_queue_pairs *
+ sizeof(int));
+ dev->tapfds = malloc(dev->max_queue_pairs *
+ sizeof(int));
+ if (!dev->vhostfds || !dev->tapfds) {
+ PMD_INIT_LOG(ERR, "Failed to malloc");
+ return -1;
+ }
+
+ for (q = 0; q < dev->max_queue_pairs; ++q) {
+ dev->vhostfds[q] = -1;
+ dev->tapfds[q] = -1;
+ }
}
}
@@ -388,6 +397,10 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
close(dev->vhostfd);
+ if (dev->is_server && dev->listenfd >= 0)
+ close(dev->listenfd);
+
+ dev->connected = false;
if (dev->vhostfds) {
for (i = 0; i < dev->max_queue_pairs; ++i)
close(dev->vhostfds[i]);
@@ -396,6 +409,9 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
}
free(dev->ifname);
+
+ if (dev->is_server)
+ unlink(dev->path);
}
static uint8_t
diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c
index 263649006..43fde6840 100644
--- a/drivers/net/virtio/virtio_user_ethdev.c
+++ b/drivers/net/virtio/virtio_user_ethdev.c
@@ -65,8 +65,7 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
r = recv(dev->vhostfd, buf, 128, MSG_PEEK);
if (r == 0 || (r < 0 && errno != EAGAIN)) {
dev->status &= (~VIRTIO_NET_S_LINK_UP);
- PMD_DRV_LOG(ERR, "virtio-user port %u is down",
- hw->port_id);
+
/* Only client mode is available now. Once the
* connection is broken, it can never be up
* again. Besides, this function could be called
@@ -74,9 +73,15 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
* callback cannot be unregistered here, set an
* alarm to do it.
*/
- rte_eal_alarm_set(1,
+ if (dev->connected) {
+ dev->connected = false;
+ PMD_DRV_LOG(ERR, "virtio-user port %u is down",
+ hw->port_id);
+ rte_eal_alarm_set(1,
virtio_user_delayed_handler,
(void *)hw);
+ hw->started = 0;
+ }
} else {
dev->status |= VIRTIO_NET_S_LINK_UP;
}
@@ -278,12 +283,15 @@ static const char *valid_args[] = {
VIRTIO_USER_ARG_QUEUE_SIZE,
#define VIRTIO_USER_ARG_INTERFACE_NAME "iface"
VIRTIO_USER_ARG_INTERFACE_NAME,
+#define VIRTIO_USER_ARG_SERVER_MODE "server"
+ VIRTIO_USER_ARG_SERVER_MODE,
NULL
};
#define VIRTIO_USER_DEF_CQ_EN 0
#define VIRTIO_USER_DEF_Q_NUM 1
#define VIRTIO_USER_DEF_Q_SZ 256
+#define VIRTIO_USER_DEF_SERVER_MODE 0
static int
get_string_arg(const char *key __rte_unused,
@@ -365,6 +373,49 @@ virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev)
rte_eth_dev_release_port(eth_dev);
}
+static void
+virtio_user_server_reconnection(int fd, void *dat, int *remove __rte_unused)
+{
+ int ret;
+ int flag;
+ int connectfd;
+ struct virtio_user_dev *dev = dat;
+ struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
+ struct virtio_hw *hw = eth_dev->data->dev_private;
+
+ if (dev->connected)
+ return;
+
+ connectfd = accept(fd, NULL, NULL);
+ if (connectfd < 0)
+ return;
+
+ if (dev->vhostfd >= 0)
+ close(dev->vhostfd);
+
+ dev->vhostfd = connectfd;
+ flag = fcntl(connectfd, F_GETFD);
+ fcntl(connectfd, F_SETFL, flag & ~O_NONBLOCK);
+
+ ret = virtio_user_start_device(dev);
+ if (ret < 0)
+ return;
+
+ if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
+ eth_dev->intr_handle->fd = connectfd;
+ rte_intr_callback_register(eth_dev->intr_handle,
+ virtio_interrupt_handler, eth_dev);
+
+ if (rte_intr_enable(eth_dev->intr_handle) < 0) {
+ PMD_DRV_LOG(ERR, "interrupt enable failed");
+ return;
+ }
+ }
+
+ hw->started = 1;
+ dev->connected = true;
+ PMD_INIT_LOG(NOTICE, "virtio_user_server_reconnection succeeds!");
+}
/* Dev initialization routine. Invoked once for each virtio vdev at
* EAL init time, see rte_bus_probe().
* Returns 0 on success.
@@ -378,11 +429,12 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
uint64_t queues = VIRTIO_USER_DEF_Q_NUM;
uint64_t cq = VIRTIO_USER_DEF_CQ_EN;
uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ;
+ uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE;
char *path = NULL;
char *ifname = NULL;
char *mac_addr = NULL;
int ret = -1;
-
+ struct virtio_user_dev *vu_dev = NULL;
kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_args);
if (!kvlist) {
PMD_INIT_LOG(ERR, "error when parsing param");
@@ -445,6 +497,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
}
+ if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) {
+ if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE,
+ &get_integer_arg, &server_mode) < 0) {
+ PMD_INIT_LOG(ERR, "error to parse %s",
+ VIRTIO_USER_ARG_SERVER_MODE);
+ goto end;
+ }
+ }
+
if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) {
if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM,
&get_integer_arg, &cq) < 0) {
@@ -476,6 +537,11 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
hw = eth_dev->data->dev_private;
+ vu_dev = virtio_user_get_dev(hw);
+ if (server_mode == 1)
+ vu_dev->is_server = true;
+ else
+ vu_dev->is_server = false;
if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq,
queue_size, mac_addr, &ifname) < 0) {
PMD_INIT_LOG(ERR, "virtio_user_dev_init fails");
@@ -488,6 +554,13 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
goto end;
}
+ if (vu_dev->is_server) {
+ ret = fdset_add(&vu_dev->fdset, vu_dev->listenfd,
+ virtio_user_server_reconnection, NULL, vu_dev);
+ if (ret < 0)
+ goto end;
+ }
+
/* previously called by rte_pci_probe() for physical dev */
if (eth_virtio_dev_init(eth_dev) < 0) {
PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");
--
2.13.3
More information about the dev
mailing list