[dpdk-dev] [PATCH v4 2/5] eal: use file to check if secondary process is ready
Anatoly Burakov
anatoly.burakov at intel.com
Fri Mar 2 16:14:09 CET 2018
Previously, IPC would remove sockets it considers to be "inactive"
based on whether they have responded. We also need to prevent
sending messages to processes that are active, but haven't yet
finished initialization.
This will create a "init file" per socket which will be removed
after initialization is complete, to prevent primary process from
sending messages to a process that hasn't finished its
initialization.
Signed-off-by: Anatoly Burakov <anatoly.burakov at intel.com>
---
Notes:
v4: changed from "don't process messages until init
complete" to "don't send messages to processes
which haven't finished initializing", as the
former would have resulted in timeouts if init
took too long to complete.
v4: rework to not send any messages to secondary processes that
haven't yet initialized, so no need for message queue
v3: no changes
v2: no changes
lib/librte_eal/bsdapp/eal/eal.c | 7 ++
lib/librte_eal/common/eal_common_proc.c | 152 +++++++++++++++++++++++++++-----
lib/librte_eal/common/eal_private.h | 10 ++-
lib/librte_eal/linuxapp/eal/eal.c | 7 ++
4 files changed, 151 insertions(+), 25 deletions(-)
diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
index 4eafcb5..c003c16 100644
--- a/lib/librte_eal/bsdapp/eal/eal.c
+++ b/lib/librte_eal/bsdapp/eal/eal.c
@@ -694,6 +694,13 @@ rte_eal_init(int argc, char **argv)
return -1;
}
+ ret = rte_mp_channel_set_ready();
+ if (ret < 0) {
+ rte_eal_init_alert("Cannot finalize mp channel init\n");
+ rte_errno = EFAULT;
+ return -1;
+ }
+
rte_eal_mcfg_complete();
return fctret;
diff --git a/lib/librte_eal/common/eal_common_proc.c b/lib/librte_eal/common/eal_common_proc.c
index da7930f..4d227fe 100644
--- a/lib/librte_eal/common/eal_common_proc.c
+++ b/lib/librte_eal/common/eal_common_proc.c
@@ -13,6 +13,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/file.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>
@@ -33,6 +34,7 @@
static int mp_fd = -1;
static char mp_filter[PATH_MAX]; /* Filter for secondary process sockets */
static char mp_dir_path[PATH_MAX]; /* The directory path for all mp sockets */
+static char process_peer_name[PATH_MAX] = {0}; /* this process's peer name */
static pthread_mutex_t mp_mutex_action = PTHREAD_MUTEX_INITIALIZER;
struct action_entry {
@@ -91,6 +93,48 @@ find_sync_request(const char *dst, const char *act_name)
return r;
}
+static void
+create_socket_path(const char *name, char *buf, int len)
+{
+ const char *prefix = eal_mp_socket_path();
+
+ if (strlen(name) > 0)
+ snprintf(buf, len, "%s_%s", prefix, name);
+ else
+ snprintf(buf, len, "%s", prefix);
+}
+
+static void
+create_initfile_path(const char *name, char *buf, int len)
+{
+ const char *prefix = eal_mp_socket_path();
+
+ if (strlen(name) > 1)
+ snprintf(buf, len, "%sinit_%s", prefix, name);
+ else
+ snprintf(buf, len, "%sinit", prefix);
+}
+
+static const char *
+get_peer_name(const char *socket_full_path)
+{
+ char buf[PATH_MAX] = {0};
+ int len;
+
+ /* primary process has no peer name */
+ if (strcmp(socket_full_path, eal_mp_socket_path()) == 0)
+ return NULL;
+
+ /* construct dummy socket file name - make it one character long so that
+ * we hit the code path where underscores are added
+ */
+ create_socket_path("a", buf, sizeof(buf));
+
+ /* we want to get everything after /path/.rte_unix_, so discard 'a' */
+ len = strlen(buf) - 1;
+ return &socket_full_path[len];
+}
+
int
rte_eal_primary_proc_alive(const char *config_file_path)
{
@@ -290,8 +334,23 @@ mp_handle(void *arg __rte_unused)
static int
open_socket_fd(void)
{
+ char initfile[PATH_MAX] = {0};
struct sockaddr_un un;
- const char *prefix = eal_mp_socket_path();
+ int init_fd;
+
+ if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+ snprintf(process_peer_name, sizeof(process_peer_name),
+ "%d_%"PRIx64, getpid(), rte_rdtsc());
+
+ /* try to create initfile */
+ create_initfile_path(process_peer_name, initfile, sizeof(initfile));
+
+ init_fd = open(initfile, O_CREAT | O_RDWR);
+ if (init_fd < 0) {
+ RTE_LOG(ERR, EAL, "failed to open '%s': %s\n", initfile,
+ strerror(errno));
+ return -1;
+ }
mp_fd = socket(AF_UNIX, SOCK_DGRAM, 0);
if (mp_fd < 0) {
@@ -301,13 +360,11 @@ open_socket_fd(void)
memset(&un, 0, sizeof(un));
un.sun_family = AF_UNIX;
- if (rte_eal_process_type() == RTE_PROC_PRIMARY)
- snprintf(un.sun_path, sizeof(un.sun_path), "%s", prefix);
- else {
- snprintf(un.sun_path, sizeof(un.sun_path), "%s_%d_%"PRIx64,
- prefix, getpid(), rte_rdtsc());
- }
+
+ create_socket_path(process_peer_name, un.sun_path, sizeof(un.sun_path));
+
unlink(un.sun_path); /* May still exist since last run */
+
if (bind(mp_fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
RTE_LOG(ERR, EAL, "failed to bind %s: %s\n",
un.sun_path, strerror(errno));
@@ -319,6 +376,31 @@ open_socket_fd(void)
return mp_fd;
}
+/* find corresponding init file */
+static int
+socket_is_ready(const char *peer_name)
+{
+ char initfile[PATH_MAX] = {0};
+ int fd;
+
+ /* construct lockfile filename */
+ create_initfile_path(peer_name, initfile, sizeof(initfile));
+
+ fd = open(initfile, O_RDWR);
+ if (fd >= 0) {
+ /* init file still exists, socket is not ready */
+ close(fd);
+ return 0;
+ }
+ if (fd < 0 && errno == ENOENT) {
+ /* init file not found, socket is ready */
+ return 1;
+ }
+ RTE_LOG(ERR, EAL, "Cannot open '%s': %s\n", initfile,
+ strerror(errno));
+ return -1;
+}
+
static int
unlink_sockets(const char *filter)
{
@@ -334,26 +416,30 @@ unlink_sockets(const char *filter)
dir_fd = dirfd(mp_dir);
while ((ent = readdir(mp_dir))) {
- if (fnmatch(filter, ent->d_name, 0) == 0)
+ if (fnmatch(filter, ent->d_name, 0) == 0) {
+ char path[PATH_MAX];
+
+ snprintf(path, sizeof(path), "%s/%s", mp_dir_path,
+ ent->d_name);
+
+ RTE_LOG(DEBUG, EAL, "Removing stale socket file '%s'\n",
+ ent->d_name);
unlinkat(dir_fd, ent->d_name, 0);
+ }
}
closedir(mp_dir);
return 0;
}
-static void
-unlink_socket_by_path(const char *path)
+int
+rte_mp_channel_set_ready(void)
{
- char *filename;
- char *fullpath = strdup(path);
+ char path[PATH_MAX] = {0};
- if (!fullpath)
- return;
- filename = basename(fullpath);
- unlink_sockets(filename);
- free(fullpath);
- RTE_LOG(INFO, EAL, "Remove socket %s\n", path);
+ create_initfile_path(process_peer_name, path, PATH_MAX);
+
+ return unlink(path);
}
int
@@ -444,10 +530,9 @@ send_msg(const char *dst_path, struct rte_mp_msg *msg, int type)
if (snd < 0) {
rte_errno = errno;
/* Check if it caused by peer process exits */
- if (errno == ECONNREFUSED) {
- /* We don't unlink the primary's socket here */
- if (rte_eal_process_type() == RTE_PROC_PRIMARY)
- unlink_socket_by_path(dst_path);
+ if (errno == ECONNREFUSED &&
+ rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ unlink(dst_path);
return 0;
}
if (errno == ENOBUFS) {
@@ -490,14 +575,22 @@ mp_send(struct rte_mp_msg *msg, const char *peer, int type)
}
while ((ent = readdir(mp_dir))) {
char path[PATH_MAX];
+ const char *peer_name;
+ int ready;
if (fnmatch(mp_filter, ent->d_name, 0) != 0)
continue;
snprintf(path, sizeof(path), "%s/%s", mp_dir_path,
ent->d_name);
- if (send_msg(path, msg, type) < 0)
+ peer_name = get_peer_name(path);
+
+ /* only send if we can expect to receive a reply */
+ ready = socket_is_ready(peer_name);
+ if (ready < 0)
ret = -1;
+ else if (ready > 0)
+ ret = send_msg(path, msg, type);
}
closedir(mp_dir);
@@ -655,15 +748,26 @@ rte_mp_request(struct rte_mp_msg *req, struct rte_mp_reply *reply,
rte_errno = errno;
return -1;
}
-
while ((ent = readdir(mp_dir))) {
+ const char *peer_name;
char path[PATH_MAX];
+ int ready;
if (fnmatch(mp_filter, ent->d_name, 0) != 0)
continue;
snprintf(path, sizeof(path), "%s/%s", mp_dir_path,
ent->d_name);
+ peer_name = get_peer_name(path);
+
+ ready = socket_is_ready(peer_name);
+
+ if (ready < 0) {
+ ret = -1;
+ continue;
+ } else if (ready == 0) {
+ continue;
+ }
if (mp_request_one(path, req, reply, &end))
ret = -1;
diff --git a/lib/librte_eal/common/eal_private.h b/lib/librte_eal/common/eal_private.h
index 0b28770..a3535a0 100644
--- a/lib/librte_eal/common/eal_private.h
+++ b/lib/librte_eal/common/eal_private.h
@@ -202,7 +202,15 @@ struct rte_bus *rte_bus_find_by_device_name(const char *str);
* 0 on success;
* (<0) on failure.
*/
-
int rte_mp_channel_init(void);
+/**
+ * Set unix channel for primary/secondary communication as ready.
+ *
+ * @return
+ * 0 on success;
+ * (<0) on failure.
+ */
+int rte_mp_channel_set_ready(void);
+
#endif /* _EAL_PRIVATE_H_ */
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 2ecd07b..a9af9c6 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -961,6 +961,13 @@ rte_eal_init(int argc, char **argv)
return -1;
}
+ ret = rte_mp_channel_set_ready();
+ if (ret < 0) {
+ rte_eal_init_alert("Cannot finalize mp channel init\n");
+ rte_errno = EFAULT;
+ return -1;
+ }
+
rte_eal_mcfg_complete();
return fctret;
--
2.7.4
More information about the dev
mailing list