[dpdk-dev] [PATCH v5 3/6] virtio, qtest: Add functionality to share memory between QTest guest

[Tetsuya Mukawa]

The patch adds functionality to share memory between QTest guest and
DPDK application using ivshmem device.
The shared memory will be all EAL memory on hugepages. This memory will
be accessed by QEMU vcpu and DPDK application using same address.

Signed-off-by: Tetsuya Mukawa <mukawa at igel.co.jp>
---
 drivers/net/virtio/virtio_qtest/qtest_utils.c | 189 +++++++++++++++++++++++++-
 drivers/net/virtio/virtio_qtest/qtest_utils.h |   4 +-
 2 files changed, 189 insertions(+), 4 deletions(-)

diff --git a/drivers/net/virtio/virtio_qtest/qtest_utils.c b/drivers/net/virtio/virtio_qtest/qtest_utils.c
index 2c088f0..9bc1fca 100644
--- a/drivers/net/virtio/virtio_qtest/qtest_utils.c
+++ b/drivers/net/virtio/virtio_qtest/qtest_utils.c
@@ -43,6 +43,9 @@
 #include "../virtio_ethdev.h"
 #include "qtest_utils.h"
 
+/* ivshmem configuration */
+#define IVSHMEM_PROTOCOL_VERSION        0
+
 #define PCI_CONFIG_ADDR(_bus, _device, _function, _offset) ( \
 	(1 << 31) | ((_bus) & 0xff) << 16 | ((_device) & 0x1f) << 11 | \
 	((_function) & 0x7) << 8 | ((_offset) & 0xfc))
@@ -59,6 +62,7 @@ union qtest_pipefds {
 
 struct qtest_session {
 	int qtest_socket;
+	int ivshmem_socket;
 	pthread_mutex_t qtest_session_lock;
 
 	struct qtest_pci_device_list head;
@@ -411,6 +415,7 @@ qtest_close_sockets(struct qtest_session *s)
 	qtest_close_one_socket(&s->qtest_socket);
 	qtest_close_one_socket(&s->msgfds.readfd);
 	qtest_close_one_socket(&s->msgfds.writefd);
+	qtest_close_one_socket(&s->ivshmem_socket);
 }
 
 static void
@@ -716,6 +721,172 @@ qtest_register_target_devices(struct qtest_session *s,
 }
 
 static int
+qtest_send_message_to_ivshmem(int sock_fd, uint64_t client_id, int shm_fd)
+{
+	struct iovec iov;
+	struct msghdr msgh;
+	size_t fdsize = sizeof(int);
+	char control[CMSG_SPACE(fdsize)];
+	struct cmsghdr *cmsg;
+	int ret;
+
+	memset(&msgh, 0, sizeof(msgh));
+	iov.iov_base = &client_id;
+	iov.iov_len = sizeof(client_id);
+
+	msgh.msg_iov = &iov;
+	msgh.msg_iovlen = 1;
+
+	if (shm_fd >= 0) {
+		msgh.msg_control = &control;
+		msgh.msg_controllen = sizeof(control);
+		cmsg = CMSG_FIRSTHDR(&msgh);
+		cmsg->cmsg_len = CMSG_LEN(fdsize);
+		cmsg->cmsg_level = SOL_SOCKET;
+		cmsg->cmsg_type = SCM_RIGHTS;
+		memcpy(CMSG_DATA(cmsg), &shm_fd, fdsize);
+	}
+
+	do {
+		ret = sendmsg(sock_fd, &msgh, 0);
+	} while (ret < 0 && errno == EINTR);
+
+	if (ret < 0) {
+		PMD_DRV_LOG(ERR, "sendmsg error");
+		return ret;
+	}
+
+	return ret;
+}
+
+/* This function is came from ../virtio_user/vhost_user.c
+ *
+ * Two possible options:
+ * 1. Match HUGEPAGE_INFO_FMT to find the file storing struct hugepage_file
+ * array. This is simple but cannot be used in secondary process because
+ * secondary process will close and munmap that file.
+ * 2. Match HUGEFILE_FMT to find hugepage files directly.
+ *
+ * We choose option 2.
+ */
+struct hugepage_file_info {
+	uint64_t addr;            /**< virtual addr */
+	size_t   size;            /**< the file size */
+	char     path[PATH_MAX];  /**< path to backing file */
+};
+
+static int
+get_hugepage_file_info(struct hugepage_file_info huges[], int max)
+{
+	int idx;
+	FILE *f;
+	char buf[BUFSIZ], *tmp, *tail;
+	char *str_underline, *str_start;
+	int huge_index;
+	uint64_t v_start, v_end;
+
+	f = fopen("/proc/self/maps", "r");
+	if (!f) {
+		PMD_DRV_LOG(ERR, "cannot open /proc/self/maps");
+		return -1;
+	}
+
+	idx = 0;
+	while (fgets(buf, sizeof(buf), f) != NULL) {
+		sscanf(buf, "%" PRIx64 "-%" PRIx64, &v_start, &v_end);
+
+		tmp = strchr(buf, ' ') + 1; /** skip address */
+		tmp = strchr(tmp, ' ') + 1; /** skip perm */
+		tmp = strchr(tmp, ' ') + 1; /** skip offset */
+		tmp = strchr(tmp, ' ') + 1; /** skip dev */
+		tmp = strchr(tmp, ' ') + 1; /** skip inode */
+		while (*tmp == ' ')         /** skip spaces */
+			tmp++;
+		tail = strrchr(tmp, '\n');  /** remove newline if exists */
+		if (tail)
+			*tail = '\0';
+
+		/* Match HUGEFILE_FMT, aka "%s/%smap_%d",
+		 * which is defined in eal_filesystem.h
+		 */
+		str_underline = strrchr(tmp, '_');
+		if (!str_underline)
+			continue;
+
+		str_start = str_underline - strlen("map");
+		if (str_start < tmp)
+			continue;
+
+		if (sscanf(str_start, "map_%d", &huge_index) != 1)
+			continue;
+
+		if (idx >= max) {
+			PMD_DRV_LOG(ERR, "Exceed maximum of %d", max);
+			goto error;
+		}
+		huges[idx].addr = v_start;
+		huges[idx].size = v_end - v_start;
+		strcpy(huges[idx].path, tmp);
+		idx++;
+	}
+
+	fclose(f);
+	return idx;
+
+error:
+	fclose(f);
+	return -1;
+}
+
+static int
+qtest_setup_shared_memory(struct qtest_session *s)
+{
+	int shm_fd, num, ret;
+	struct hugepage_file_info huges[1];
+
+	num = get_hugepage_file_info(huges, 1);
+	if (num != 1) {
+		PMD_DRV_LOG(ERR,
+			"Not supported memory configuration");
+		return -1;
+	}
+
+	shm_fd = open(huges[0].path, O_RDWR);
+	if (shm_fd < 0) {
+		PMD_DRV_LOG(ERR,
+			"Cannot open file: %s", huges[0].path);
+		return -1;
+	}
+
+	/* send our protocol version first */
+	ret = qtest_send_message_to_ivshmem(s->ivshmem_socket,
+			IVSHMEM_PROTOCOL_VERSION, -1);
+	if (ret < 0) {
+		PMD_DRV_LOG(ERR,
+			"Failed to send protocol version to ivshmem");
+		return -1;
+	}
+
+	/* send client id */
+	ret = qtest_send_message_to_ivshmem(s->ivshmem_socket, 0, -1);
+	if (ret < 0) {
+		PMD_DRV_LOG(ERR, "Failed to send VMID to ivshmem");
+		return -1;
+	}
+
+	/* send message to ivshmem */
+	ret = qtest_send_message_to_ivshmem(s->ivshmem_socket, -1, shm_fd);
+	if (ret < 0) {
+		PMD_DRV_LOG(ERR, "Failed to file descriptor to ivshmem");
+		return -1;
+	}
+
+	close(shm_fd);
+
+	return 0;
+}
+
+static int
 qtest_open_socket(char *path)
 {
 	struct sockaddr_un sa = {0};
@@ -769,7 +940,7 @@ qtest_vdev_uninit(struct qtest_session *s)
 }
 
 struct qtest_session *
-qtest_vdev_init(char *qtest_path,
+qtest_vdev_init(char *qtest_path, char *ivshmem_path,
 		struct qtest_pci_device *devices, int devnum)
 {
 	struct qtest_session *s;
@@ -796,7 +967,13 @@ qtest_vdev_init(char *qtest_path,
 
 	ret = qtest_register_target_devices(s, devices, devnum);
 	if (ret != 0) {
-		PMD_DRV_LOG(ERR, "Failed to initialize qtest session\n");
+		PMD_DRV_LOG(ERR, "Failed to initialize qtest session");
+		goto error;
+	}
+
+	s->ivshmem_socket = qtest_open_socket(ivshmem_path);
+	if (s->ivshmem_socket < 0) {
+		PMD_DRV_LOG(ERR, "Failed to open %s", ivshmem_path);
 		goto error;
 	}
 
@@ -813,9 +990,15 @@ qtest_vdev_init(char *qtest_path,
 	}
 	s->event_th_started = 1;
 
+	ret = qtest_setup_shared_memory(s);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Failed to setup shared memory");
+		goto error;
+	}
+
 	ret = qtest_init_pci_devices(s, devices, devnum);
 	if (ret != 0) {
-		PMD_DRV_LOG(ERR, "Failed to initialize devices\n");
+		PMD_DRV_LOG(ERR, "Failed to initialize devices");
 		goto error;
 	}
 
diff --git a/drivers/net/virtio/virtio_qtest/qtest_utils.h b/drivers/net/virtio/virtio_qtest/qtest_utils.h
index a3d8176..6c70552 100644
--- a/drivers/net/virtio/virtio_qtest/qtest_utils.h
+++ b/drivers/net/virtio/virtio_qtest/qtest_utils.h
@@ -132,6 +132,8 @@ struct qtest_pci_device {
  *
  * @param qtest_path
  *   Path of qtest socket.
+ * @param ivshmem_path
+ *   Path of ivshmem socket.
  * @param devices
  *   Array of device information. It should contain piix3, ivshmem and target
  *   device(virtio-net device).
@@ -140,7 +142,7 @@ struct qtest_pci_device {
  * @return
  *   The pointer to qtest session structure.
  */
-struct qtest_session *qtest_vdev_init(char *qtest_path,
+struct qtest_session *qtest_vdev_init(char *qtest_path, char *ivshmem_path,
 		struct qtest_pci_device *devices, int devnum);
 
 /**
-- 
2.7.4