[dpdk-dev] [PATCH V10 1/2] eal: add uevent monitor api and callback func

[Jeff Guo]

This patch aim to add a general uevent mechanism in eal device layer,
to enable all linux kernel object uevent monitoring, user could use these
APIs to monitor and read out the device status info that sent from the
kernel side, then corresponding to handle it, such as when detect hotplug
uevent type, user could detach or attach the device, and more it benefit
to use to do smoothly fail safe work.

About uevent monitoring:
a: add one epolling to poll the netlink socket, to monitor the uevent of
   the device.
b: add enum of rte_eal_dev_event_type and struct of rte_eal_uevent.
c: add below APIs in rte eal device layer.
   rte_dev_callback_register
   rte_dev_callback_unregister
   _rte_dev_callback_process
   rte_dev_evt_mntr_start
   rte_dev_evt_mntr_stop

Signed-off-by: Jeff Guo <jia.guo at intel.com>
---
V10->V9:
a.fix prefix issue.
b.use a common callback lists for all device and all type to replace
add callback parameter into device struct.
c.delete some unuse part.
---
 lib/librte_eal/bsdapp/eal/eal_dev.c     |  38 ++++++
 lib/librte_eal/common/eal_common_dev.c  | 124 ++++++++++++++++++
 lib/librte_eal/common/include/rte_dev.h | 101 +++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile    |   1 +
 lib/librte_eal/linuxapp/eal/eal_dev.c   | 223 ++++++++++++++++++++++++++++++++
 5 files changed, 487 insertions(+)
 create mode 100644 lib/librte_eal/bsdapp/eal/eal_dev.c
 create mode 100644 lib/librte_eal/linuxapp/eal/eal_dev.c

diff --git a/lib/librte_eal/bsdapp/eal/eal_dev.c b/lib/librte_eal/bsdapp/eal/eal_dev.c
new file mode 100644
index 0000000..32c17e8
--- /dev/null
+++ b/lib/librte_eal/bsdapp/eal/eal_dev.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/queue.h>
+#include <sys/signalfd.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/epoll.h>
+#include <unistd.h>
+#include <signal.h>
+#include <stdbool.h>
+
+#include <rte_malloc.h>
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+
+#include "eal_thread.h"
+
+int
+rte_dev_evt_mntr_start(void)
+{
+	RTE_LOG(ERR, EAL, "Not support event monitor for FreeBSD\n");
+	return -1;
+}
+
+int
+rte_dev_evt_mntr_stop(void)
+{
+	RTE_LOG(ERR, EAL, "Not support event monitor for FreeBSD\n");
+	return -1;
+}
diff --git a/lib/librte_eal/common/eal_common_dev.c b/lib/librte_eal/common/eal_common_dev.c
index dda8f58..b602535 100644
--- a/lib/librte_eal/common/eal_common_dev.c
+++ b/lib/librte_eal/common/eal_common_dev.c
@@ -42,9 +42,31 @@
 #include <rte_devargs.h>
 #include <rte_debug.h>
 #include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_malloc.h>
 
 #include "eal_private.h"
 
+/* spinlock for device callbacks */
+static rte_spinlock_t rte_dev_event_lock = RTE_SPINLOCK_INITIALIZER;
+
+/**
+ * The user application callback description.
+ *
+ * It contains callback address to be registered by user application,
+ * the pointer to the parameters for callback, and the event type.
+ */
+struct rte_dev_event_callback {
+	TAILQ_ENTRY(rte_dev_event_callback) next; /**< Callbacks list */
+	rte_dev_event_cb_fn cb_fn;                /**< Callback address */
+	void *cb_arg;                           /**< Callback parameter */
+	char *dev_name; /**< device name */
+	uint32_t active;                        /**< Callback is executing */
+};
+
+/* A genaral callback for all registerd devices */
+static struct rte_dev_event_cb_list dev_event_cbs;
+
 static int cmp_detached_dev_name(const struct rte_device *dev,
 	const void *_name)
 {
@@ -234,3 +256,105 @@ int rte_eal_hotplug_remove(const char *busname, const char *devname)
 	rte_eal_devargs_remove(busname, devname);
 	return ret;
 }
+
+int
+rte_dev_callback_register(char *dev_name,
+			rte_dev_event_cb_fn cb_fn, void *cb_arg)
+{
+	struct rte_dev_event_callback *event_cb = NULL;
+
+	rte_spinlock_lock(&rte_dev_event_lock);
+
+	if (TAILQ_EMPTY(&(dev_event_cbs)))
+		TAILQ_INIT(&(dev_event_cbs));
+
+	TAILQ_FOREACH(event_cb, &(dev_event_cbs), next) {
+		if (event_cb->cb_fn == cb_fn &&
+			event_cb->cb_arg == cb_arg &&
+			event_cb->dev_name == dev_name)
+			break;
+	}
+
+	/* create a new callback. */
+	if (event_cb == NULL) {
+		/* allocate a new user callback entity */
+		event_cb = malloc(sizeof(struct rte_dev_event_callback));
+		if (event_cb != NULL) {
+			event_cb->cb_fn = cb_fn;
+			event_cb->cb_arg = cb_arg;
+			event_cb->dev_name = dev_name;
+		}
+		TAILQ_INSERT_TAIL(&(dev_event_cbs), event_cb, next);
+	}
+
+	rte_spinlock_unlock(&rte_dev_event_lock);
+	return (event_cb == NULL) ? -1 : 0;
+}
+
+int
+rte_dev_callback_unregister(char *dev_name,
+			rte_dev_event_cb_fn cb_fn, void *cb_arg)
+{
+	int ret;
+	struct rte_dev_event_callback *event_cb, *next;
+
+	if (!cb_fn || dev_name == NULL)
+		return -EINVAL;
+
+	rte_spinlock_lock(&rte_dev_event_lock);
+
+	ret = 0;
+
+	for (event_cb = TAILQ_FIRST(&(dev_event_cbs)); event_cb != NULL;
+	      event_cb = next) {
+
+		next = TAILQ_NEXT(event_cb, next);
+
+		if (event_cb->cb_fn != cb_fn ||
+				(event_cb->cb_arg != (void *)-1 &&
+				event_cb->cb_arg != cb_arg) ||
+				event_cb->dev_name != dev_name)
+			continue;
+
+		/*
+		 * if this callback is not executing right now,
+		 * then remove it.
+		 */
+		if (event_cb->active == 0) {
+			TAILQ_REMOVE(&(dev_event_cbs), event_cb, next);
+			rte_free(event_cb);
+		} else {
+			ret = -EAGAIN;
+		}
+	}
+
+	rte_spinlock_unlock(&rte_dev_event_lock);
+	return ret;
+}
+
+int
+_rte_dev_callback_process(char *dev_name,
+			enum rte_dev_event_type event)
+{
+	struct rte_dev_event_callback dev_cb;
+	struct rte_dev_event_callback *cb_lst;
+	int rc = 0;
+
+	rte_spinlock_lock(&rte_dev_event_lock);
+
+	if (dev_name == NULL)
+		return -EINVAL;
+
+	TAILQ_FOREACH(cb_lst, &(dev_event_cbs), next) {
+		if (cb_lst->cb_fn == NULL || cb_lst->dev_name != dev_name)
+			continue;
+		dev_cb = *cb_lst;
+		cb_lst->active = 1;
+		rc = dev_cb.cb_fn(event,
+				dev_cb.cb_arg);
+		cb_lst->active = 0;
+	}
+
+	rte_spinlock_unlock(&rte_dev_event_lock);
+	return rc;
+}
diff --git a/lib/librte_eal/common/include/rte_dev.h b/lib/librte_eal/common/include/rte_dev.h
index 9342e0c..fea037a 100644
--- a/lib/librte_eal/common/include/rte_dev.h
+++ b/lib/librte_eal/common/include/rte_dev.h
@@ -51,6 +51,29 @@ extern "C" {
 
 #include <rte_log.h>
 
+/**
+ * The device event type.
+ */
+enum rte_dev_event_type {
+	RTE_DEV_EVENT_UNKNOWN,	/**< unknown event type */
+	RTE_DEV_EVENT_ADD,	/**< device being added */
+	RTE_DEV_EVENT_REMOVE,	/**< device being removed */
+	RTE_DEV_EVENT_MAX	/**< max value of this enum */
+};
+
+struct rte_dev_event {
+	enum rte_dev_event_type type;	/**< device event type */
+	int subsystem;				/**< subsystem id */
+	char *devname;				/**< device name */
+};
+
+typedef int (*rte_dev_event_cb_fn)(enum rte_dev_event_type event,
+					void *cb_arg);
+
+struct rte_dev_event_callback;
+/** @internal Structure to keep track of registered callbacks */
+TAILQ_HEAD(rte_dev_event_cb_list, rte_dev_event_callback);
+
 __attribute__((format(printf, 2, 0)))
 static inline void
 rte_pmd_debug_trace(const char *func_name, const char *fmt, ...)
@@ -293,4 +316,82 @@ __attribute__((used)) = str
 }
 #endif
 
+/**
+ * It registers the callback for the specific event. Multiple
+ * callbacks cal be registered at the same time.
+ * @param event
+ *  The device event type.
+ * @param cb_fn
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback.
+ *
+ * @return
+ *  - On success, zero.
+ *  - On failure, a negative value.
+ */
+int rte_dev_callback_register(char *dev_name,
+			rte_dev_event_cb_fn cb_fn, void *cb_arg);
+
+/**
+ * It unregisters the callback according to the specified event.
+ *
+ * @param event
+ *  The event type which corresponding to the callback.
+ * @param cb_fn
+ *  callback address.
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ *
+ * @return
+ *  - On success, return the number of callback entities removed.
+ *  - On failure, a negative value.
+ */
+int rte_dev_callback_unregister(char *dev_name,
+			rte_dev_event_cb_fn cb_fn, void *cb_arg);
+
+/**
+ * @internal Executes all the user application registered callbacks for
+ * the specific device. It is for DPDK internal user only. User
+ * application should not call it directly.
+ *
+ * @param event
+ *  The device event type.
+ * @param cb_arg
+ *  callback parameter.
+ * @param ret_param
+ *  To pass data back to user application.
+ *  This allows the user application to decide if a particular function
+ *  is permitted or not.
+ *
+ * @return
+ *  - On success, return zero.
+ *  - On failure, a negative value.
+ */
+int
+_rte_dev_callback_process(char *dev_name,
+			enum rte_dev_event_type event);
+
+/**
+ * Start the device event monitoring.
+ *
+ * @param none
+ * @return
+ *   - On success, zero.
+ *   - On failure, a negative value.
+ */
+int
+rte_dev_evt_mntr_start(void);
+
+/**
+ * Stop the device event monitoring .
+ *
+ * @param none
+ * @return
+ *   - On success, zero.
+ *   - On failure, a negative value.
+ */
+
+int
+rte_dev_evt_mntr_stop(void);
 #endif /* _RTE_DEV_H_ */
diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile
index 588c0bd..43b00e5 100644
--- a/lib/librte_eal/linuxapp/eal/Makefile
+++ b/lib/librte_eal/linuxapp/eal/Makefile
@@ -39,6 +39,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_lcore.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_timer.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_interrupts.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_alarm.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_dev.c
 
 # from common dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_lcore.c
diff --git a/lib/librte_eal/linuxapp/eal/eal_dev.c b/lib/librte_eal/linuxapp/eal/eal_dev.c
new file mode 100644
index 0000000..bc32aab
--- /dev/null
+++ b/lib/librte_eal/linuxapp/eal/eal_dev.c
@@ -0,0 +1,223 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/queue.h>
+#include <sys/signalfd.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <linux/netlink.h>
+#include <sys/epoll.h>
+#include <unistd.h>
+#include <signal.h>
+#include <stdbool.h>
+
+#include <rte_malloc.h>
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_service.h>
+#include <rte_service_component.h>
+
+#include "eal_thread.h"
+
+bool service_exit = true;
+
+bool service_no_init = true;
+
+#define DEV_EV_MNT_SERVICE_NAME "device_event_monitor_service"
+
+static int
+dev_monitor_fd_new(void)
+{
+
+	int uevent_fd;
+
+	uevent_fd = socket(PF_NETLINK, SOCK_RAW | SOCK_CLOEXEC |
+			SOCK_NONBLOCK,
+			NETLINK_KOBJECT_UEVENT);
+	if (uevent_fd < 0) {
+		RTE_LOG(ERR, EAL, "create uevent fd failed\n");
+		return -1;
+	}
+	return uevent_fd;
+}
+
+static int
+dev_monitor_enable(int netlink_fd)
+{
+	struct sockaddr_nl addr;
+	int ret;
+	int size = 64 * 1024;
+	int nonblock = 1;
+
+	memset(&addr, 0, sizeof(addr));
+	addr.nl_family = AF_NETLINK;
+	addr.nl_pid = 0;
+	addr.nl_groups = 0xffffffff;
+
+	if (bind(netlink_fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) {
+		RTE_LOG(ERR, EAL, "bind failed\n");
+		goto err;
+	}
+
+	setsockopt(netlink_fd, SOL_SOCKET, SO_PASSCRED, &size, sizeof(size));
+
+	ret = ioctl(netlink_fd, FIONBIO, &nonblock);
+	if (ret != 0) {
+		RTE_LOG(ERR, EAL, "ioctl(FIONBIO) failed\n");
+		goto err;
+	}
+	return 0;
+err:
+	close(netlink_fd);
+	return -1;
+}
+
+static int
+dev_uev_process(__rte_unused struct epoll_event *events, __rte_unused int nfds)
+{
+	/* TODO: device uevent processing */
+	return 0;
+}
+
+/**
+ * It builds/rebuilds up the epoll file descriptor with all the
+ * file descriptors being waited on. Then handles the interrupts.
+ *
+ * @param arg
+ *  pointer. (unused)
+ *
+ * @return
+ *   - On success, zero.
+ *   - On failure, a negative value.
+ */
+static int32_t dev_uev_monitoring(__rte_unused void *arg)
+{
+	int netlink_fd = -1;
+	struct epoll_event ep_kernel;
+	int fd_ep = -1;
+
+	service_exit = false;
+
+	fd_ep = epoll_create1(EPOLL_CLOEXEC);
+	if (fd_ep < 0) {
+		RTE_LOG(ERR, EAL, "error creating epoll fd: %m\n");
+		goto out;
+	}
+
+	netlink_fd = dev_monitor_fd_new();
+
+	if (dev_monitor_enable(netlink_fd) < 0) {
+		RTE_LOG(ERR, EAL, "error subscribing to kernel events\n");
+		goto out;
+	}
+
+	memset(&ep_kernel, 0, sizeof(struct epoll_event));
+	ep_kernel.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP;
+	ep_kernel.data.fd = netlink_fd;
+	if (epoll_ctl(fd_ep, EPOLL_CTL_ADD, netlink_fd,
+		&ep_kernel) < 0) {
+		RTE_LOG(ERR, EAL, "error addding fd to epoll: %m\n");
+		goto out;
+	}
+
+	while (!service_exit) {
+		int fdcount;
+		struct epoll_event ev[1];
+
+		fdcount = epoll_wait(fd_ep, ev, 1, -1);
+		if (fdcount < 0) {
+			if (errno != EINTR)
+				RTE_LOG(ERR, EAL, "error receiving uevent "
+					"message: %m\n");
+				continue;
+			}
+
+		/* epoll_wait has at least one fd ready to read */
+		if (dev_uev_process(ev, fdcount) < 0) {
+			if (errno != EINTR)
+				RTE_LOG(ERR, EAL, "error processing uevent "
+					"message: %m\n");
+		}
+	}
+	return 0;
+out:
+	if (fd_ep >= 0)
+		close(fd_ep);
+	if (netlink_fd >= 0)
+		close(netlink_fd);
+	rte_panic("uev monitoring fail\n");
+	return -1;
+}
+
+int
+rte_dev_evt_mntr_start(void)
+{
+	int ret;
+	struct rte_service_spec service;
+	uint32_t id;
+	const uint32_t sid = 0;
+
+	if (!service_no_init)
+		return 0;
+
+	uint32_t slcore_1 = rte_get_next_lcore(/* start core */ -1,
+					       /* skip master */ 1,
+					       /* wrap */ 0);
+
+	ret = rte_service_lcore_add(slcore_1);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "dev event monitor lcore add fail");
+		return ret;
+	}
+
+	memset(&service, 0, sizeof(service));
+	snprintf(service.name, sizeof(service.name), DEV_EV_MNT_SERVICE_NAME);
+
+	service.socket_id = rte_socket_id();
+	service.callback = dev_uev_monitoring;
+	service.callback_userdata = NULL;
+	service.capabilities = 0;
+	ret = rte_service_component_register(&service, &id);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "Failed to register service %s "
+			"err = %" PRId32,
+			service.name, ret);
+		return ret;
+	}
+	ret = rte_service_runstate_set(sid, 1);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "Failed to set the runstate of "
+			"the service");
+		return ret;
+	}
+	ret = rte_service_component_runstate_set(id, 1);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "Failed to set the backend runstate"
+			" of a component");
+		return ret;
+	}
+	ret = rte_service_map_lcore_set(sid, slcore_1, 1);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "Failed to enable lcore 1 on "
+			"dev event monitor service");
+		return ret;
+	}
+	rte_service_lcore_start(slcore_1);
+	service_no_init = false;
+	return 0;
+}
+
+int
+rte_dev_evt_mntr_stop(void)
+{
+	service_exit = true;
+	service_no_init = true;
+	return 0;
+}
-- 
2.7.4