[dpdk-users] Attaching to an ethernet port from a secondary process

[Lazaros Koromilas]

On Wed, Feb 17, 2016 at 1:17 PM, Lazaros Koromilas <l at nofutznetworks.com> wrote:
> Hi all,
>
> I'm trying to take advantage of the multi-process support together
> with the port hotplugging framework. The idea is that my primary
> process will configure all ports I'm going to need and then secondary
> processes will come up, selectively attach to ports and perform rx/tx.
> I succeed up to the point that I attach to the pre-configured ports
> but the rx/tx queue descriptors are uninitialized, so I get a segfault
> there. This can be reproduced with the attached, slightly modified
> examples/skeleton/basicfwd.c sdk app. I run this as follows:
>
> sudo ./build/basicfwd -l0 -n2 --proc-type=primary --vdev
> eth_pcap0,rx_pcap=pcap0-in.pcap,tx_pcap=pcap0-out.pcap --vdev
> eth_pcap1,rx_pcap=pcap1-in.pcap,tx_pcap=pcap1-out.pcap
> sudo ./build/basicfwd -l2 -n2 --proc-type=secondary
> eth_pcap0,rx_pcap=pcap0-in.pcap,tx_pcap=pcap0-out.pcap
> eth_pcap1,rx_pcap=pcap1-in.pcap,tx_pcap=pcap1-out.pcap
>
> Where pcap{0,1}-in.pcap files are pcap dumps. The primary will
> initialize the ports and pause(2). The secondary will attach to the
> same ports and do work. The latter then segfaults in
> rte_eth_rx_burst() with dev->data->{rx,tx}_queues being NULL pointers.
>
> All privileged operations are done from the primary. As far as I can
> tell, these are all functions in librte_ether that need to run in a
> primary process:
> grep -B12 PROC_PRIMARY_ rte_ethdev.c | grep '^rte_eth' | sed 's/(.*//'
> rte_eth_dev_rx_queue_start
> rte_eth_dev_rx_queue_stop
> rte_eth_dev_tx_queue_start
> rte_eth_dev_tx_queue_stop
> rte_eth_dev_configure
> rte_eth_dev_start
> rte_eth_dev_stop
> rte_eth_dev_set_link_up
> rte_eth_dev_set_link_down
> rte_eth_dev_close
> rte_eth_rx_queue_setup
> rte_eth_tx_queue_setup
>
> Any hints are much appreciated.
>
> Thanks!
> Lazaros.

I'm pasting the diff here, so that it exists in the archives too.

--- basicfwd.c.orig 2016-02-16 19:31:26.279293811 +0200
+++ basicfwd.c  2016-02-17 13:28:28.717350548 +0200
@@ -33,6 +33,7 @@

 #include <stdint.h>
 #include <inttypes.h>
+#include <unistd.h>
 #include <rte_eal.h>
 #include <rte_ethdev.h>
 #include <rte_cycles.h>
@@ -174,6 +175,7 @@ main(int argc, char *argv[])
    struct rte_mempool *mbuf_pool;
    unsigned nb_ports;
    uint8_t portid;
+   int i;

    /* Initialize the Environment Abstraction Layer (EAL). */
    int ret = rte_eal_init(argc, argv);
@@ -183,29 +185,47 @@ main(int argc, char *argv[])
    argc -= ret;
    argv += ret;

+   if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+       /* Attach to ports in app command line. */
+       for (i = 1; i < argc; i++) {
+           printf("Port name %s\n", argv[i]);
+           ret = rte_eth_dev_attach(argv[i], &portid);
+           if (ret < 0)
+               rte_exit(EXIT_FAILURE, "Could not attach to %s\n", argv[i]);
+           printf("Attached to port %s with id %u\n", argv[i], portid);
+       }
+   }
+
    /* Check that there is an even number of ports to send/receive on. */
    nb_ports = rte_eth_dev_count();
    if (nb_ports < 2 || (nb_ports & 1))
        rte_exit(EXIT_FAILURE, "Error: number of ports must be even\n");

-   /* Creates a new mempool in memory to hold the mbufs. */
-   mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports,
-       MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
-
-   if (mbuf_pool == NULL)
-       rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
-
-   /* Initialize all ports. */
-   for (portid = 0; portid < nb_ports; portid++)
-       if (port_init(portid, mbuf_pool) != 0)
-           rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n",
-                   portid);
+   if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+       /* Creates a new mempool in memory to hold the mbufs. */
+       mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports,
+           MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
+
+       if (mbuf_pool == NULL)
+           rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
+
+       /* Initialize all ports. */
+       for (portid = 0; portid < nb_ports; portid++)
+           if (port_init(portid, mbuf_pool) != 0)
+               rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n",
+                       portid);
+   }

    if (rte_lcore_count() > 1)
        printf("\nWARNING: Too many lcores enabled. Only 1 used.\n");

-   /* Call lcore_main on the master core only. */
-   lcore_main();
+   if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+       /* Wait here forever. */
+       pause();
+   } else {
+       /* Call lcore_main on the master core only. */
+       lcore_main();
+   }

    return 0;
 }