[dpdk-dev,v2,5/7] examples/eventdev: update sample app to use service
Checks
Commit Message
From: Pavan Bhagavatula <pbhagavatula@caviumnetworks.com>
Update the sample app eventdev_pipeline_sw_pmd to use service cores for
event scheduling in case of sw eventdev.
Signed-off-by: Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>
---
examples/eventdev_pipeline_sw_pmd/main.c | 51 +++++++++++++++++++++-----------
1 file changed, 33 insertions(+), 18 deletions(-)
Comments
> From: Pavan Nikhilesh [mailto:pbhagavatula@caviumnetworks.com]
> Sent: Friday, October 13, 2017 5:37 PM
> To: jerin.jacob@caviumnetworks.com; hemant.agrawal@nxp.com; Van Haaren,
> Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; Pavan Bhagavatula <pbhagavatula@caviumnetworks.com>
> Subject: [dpdk-dev] [PATCH v2 5/7] examples/eventdev: update sample app to
> use service
>
> From: Pavan Bhagavatula <pbhagavatula@caviumnetworks.com>
>
> Update the sample app eventdev_pipeline_sw_pmd to use service cores for
> event scheduling in case of sw eventdev.
>
> Signed-off-by: Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>
Comments inline - I think there are some side-effect changes in the application.
> ---
> examples/eventdev_pipeline_sw_pmd/main.c | 51 +++++++++++++++++++++--------
> ---
> 1 file changed, 33 insertions(+), 18 deletions(-)
>
> diff --git a/examples/eventdev_pipeline_sw_pmd/main.c
> b/examples/eventdev_pipeline_sw_pmd/main.c
> index 09b90c3..d5068d2 100644
> --- a/examples/eventdev_pipeline_sw_pmd/main.c
> +++ b/examples/eventdev_pipeline_sw_pmd/main.c
> @@ -46,6 +46,7 @@
> #include <rte_cycles.h>
> #include <rte_ethdev.h>
> #include <rte_eventdev.h>
> +#include <rte_service.h>
>
> #define MAX_NUM_STAGES 8
> #define BATCH_SIZE 16
> @@ -233,7 +234,7 @@ producer(void)
> }
>
> static inline void
> -schedule_devices(uint8_t dev_id, unsigned int lcore_id)
> +schedule_devices(unsigned int lcore_id)
> {
> if (fdata->rx_core[lcore_id] && (fdata->rx_single ||
> rte_atomic32_cmpset(&(fdata->rx_lock), 0, 1))) {
> @@ -241,16 +242,6 @@ schedule_devices(uint8_t dev_id, unsigned int lcore_id)
> rte_atomic32_clear((rte_atomic32_t *)&(fdata->rx_lock));
> }
>
> - if (fdata->sched_core[lcore_id] && (fdata->sched_single ||
> - rte_atomic32_cmpset(&(fdata->sched_lock), 0, 1))) {
> - rte_event_schedule(dev_id);
> - if (cdata.dump_dev_signal) {
> - rte_event_dev_dump(0, stdout);
> - cdata.dump_dev_signal = 0;
> - }
> - rte_atomic32_clear((rte_atomic32_t *)&(fdata->sched_lock));
> - }
See note below, about keeping the functionality provided by
fdata->sched_core[] intact.
> if (fdata->tx_core[lcore_id] && (fdata->tx_single ||
> rte_atomic32_cmpset(&(fdata->tx_lock), 0, 1))) {
> consumer();
> @@ -294,7 +285,7 @@ worker(void *arg)
> while (!fdata->done) {
> uint16_t i;
>
> - schedule_devices(dev_id, lcore_id);
> + schedule_devices(lcore_id);
>
> if (!fdata->worker_core[lcore_id]) {
> rte_pause();
> @@ -661,6 +652,27 @@ struct port_link {
> };
>
> static int
> +setup_scheduling_service(unsigned int lcore, uint8_t dev_id)
> +{
> + int ret;
> + uint32_t service_id;
> + ret = rte_event_dev_service_id_get(dev_id, &service_id);
> + if (ret == -ESRCH) {
> + printf("Event device [%d] doesn't need scheduling service\n",
> + dev_id);
> + return 0;
> + }
> + if (!ret) {
> + rte_service_runstate_set(service_id, 1);
> + rte_service_lcore_add(lcore);
> + rte_service_map_lcore_set(service_id, lcore, 1);
> + rte_service_lcore_start(lcore);
> + }
> +
> + return ret;
> +}
> +
> +static int
> setup_eventdev(struct prod_data *prod_data,
> struct cons_data *cons_data,
> struct worker_data *worker_data)
> @@ -839,6 +851,14 @@ setup_eventdev(struct prod_data *prod_data,
> *cons_data = (struct cons_data){.dev_id = dev_id,
> .port_id = i };
>
> + for (i = 0; i < MAX_NUM_CORE; i++) {
> + if (fdata->sched_core[i]
> + && setup_scheduling_service(i, dev_id)) {
> + printf("Error setting up schedulig service on %d", i);
> + return -1;
> + }
> + }
Previously, the fdata->sched_core[] array contained a "coremask" for scheduling.
A core running the scheduling could *also* perform other work. AKA: a single core
could perform all of RX, Sched, Worker, and TX.
Due to the service-core requiring to "take" the full core, there is no option to
have a core "split" its work into schedule() and RX,TX,Worker. This is a service core
implementation limitation - however it should be resolved for this sample app too.
The solution is to enable an ordinary DPDK (non-service-core) thread to run
a service. This MUST be enabled at the service-cores library level, to keep atomics
behavior of services etc), and hence removing rte_event_schedule() is still required.
The changes should become simpler than proposed here, instead of the wait_schedule() hack,
we can just run an iteration of the SW PMD using the newly-added service core iter function.
I have (just) sent a patch for service-cores to enable running a service on an ordinary
DPDK lcore, see here: http://dpdk.org/ml/archives/dev/2017-October/080022.html
Hope you can rework patches 4/7 and 5/7 to use the newly provided functionality!
Let me know if the intended usage of the new function is unclear in any way.
Regards, -Harry
> +
> if (rte_event_dev_start(dev_id) < 0) {
> printf("Error starting eventdev\n");
> return -1;
> @@ -944,8 +964,7 @@ main(int argc, char **argv)
>
> if (!fdata->rx_core[lcore_id] &&
> !fdata->worker_core[lcore_id] &&
> - !fdata->tx_core[lcore_id] &&
> - !fdata->sched_core[lcore_id])
> + !fdata->tx_core[lcore_id])
> continue;
>
> if (fdata->rx_core[lcore_id])
> @@ -958,10 +977,6 @@ main(int argc, char **argv)
> "[%s()] lcore %d executing NIC Tx, and using eventdev
> port %u\n",
> __func__, lcore_id, cons_data.port_id);
>
> - if (fdata->sched_core[lcore_id])
> - printf("[%s()] lcore %d executing scheduler\n",
> - __func__, lcore_id);
> -
> if (fdata->worker_core[lcore_id])
> printf(
> "[%s()] lcore %d executing worker, using eventdev port
> %u\n",
> --
> 2.7.4
On Mon, Oct 23, 2017 at 05:17:48PM +0000, Van Haaren, Harry wrote:
> > From: Pavan Nikhilesh [mailto:pbhagavatula@caviumnetworks.com]
> > Sent: Friday, October 13, 2017 5:37 PM
> > To: jerin.jacob@caviumnetworks.com; hemant.agrawal@nxp.com; Van Haaren,
> > Harry <harry.van.haaren@intel.com>
> > Cc: dev@dpdk.org; Pavan Bhagavatula <pbhagavatula@caviumnetworks.com>
> > Subject: [dpdk-dev] [PATCH v2 5/7] examples/eventdev: update sample app to
> > use service
> >
> > From: Pavan Bhagavatula <pbhagavatula@caviumnetworks.com>
> >
> > Update the sample app eventdev_pipeline_sw_pmd to use service cores for
> > event scheduling in case of sw eventdev.
> >
> > Signed-off-by: Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>
>
>
> Comments inline - I think there are some side-effect changes in the application.
>
>
> > ---
> > examples/eventdev_pipeline_sw_pmd/main.c | 51 +++++++++++++++++++++--------
> > ---
> > 1 file changed, 33 insertions(+), 18 deletions(-)
> >
> > diff --git a/examples/eventdev_pipeline_sw_pmd/main.c
> > b/examples/eventdev_pipeline_sw_pmd/main.c
> > index 09b90c3..d5068d2 100644
> > --- a/examples/eventdev_pipeline_sw_pmd/main.c
> > +++ b/examples/eventdev_pipeline_sw_pmd/main.c
> > @@ -46,6 +46,7 @@
> > #include <rte_cycles.h>
> > #include <rte_ethdev.h>
> > #include <rte_eventdev.h>
> > +#include <rte_service.h>
> >
> > #define MAX_NUM_STAGES 8
> > #define BATCH_SIZE 16
> > @@ -233,7 +234,7 @@ producer(void)
> > }
> >
> > static inline void
> > -schedule_devices(uint8_t dev_id, unsigned int lcore_id)
> > +schedule_devices(unsigned int lcore_id)
> > {
> > if (fdata->rx_core[lcore_id] && (fdata->rx_single ||
> > rte_atomic32_cmpset(&(fdata->rx_lock), 0, 1))) {
> > @@ -241,16 +242,6 @@ schedule_devices(uint8_t dev_id, unsigned int lcore_id)
> > rte_atomic32_clear((rte_atomic32_t *)&(fdata->rx_lock));
> > }
> >
> > - if (fdata->sched_core[lcore_id] && (fdata->sched_single ||
> > - rte_atomic32_cmpset(&(fdata->sched_lock), 0, 1))) {
> > - rte_event_schedule(dev_id);
> > - if (cdata.dump_dev_signal) {
> > - rte_event_dev_dump(0, stdout);
> > - cdata.dump_dev_signal = 0;
> > - }
> > - rte_atomic32_clear((rte_atomic32_t *)&(fdata->sched_lock));
> > - }
>
> See note below, about keeping the functionality provided by
> fdata->sched_core[] intact.
>
>
> > if (fdata->tx_core[lcore_id] && (fdata->tx_single ||
> > rte_atomic32_cmpset(&(fdata->tx_lock), 0, 1))) {
> > consumer();
> > @@ -294,7 +285,7 @@ worker(void *arg)
> > while (!fdata->done) {
> > uint16_t i;
> >
> > - schedule_devices(dev_id, lcore_id);
> > + schedule_devices(lcore_id);
> >
> > if (!fdata->worker_core[lcore_id]) {
> > rte_pause();
> > @@ -661,6 +652,27 @@ struct port_link {
> > };
> >
> > static int
> > +setup_scheduling_service(unsigned int lcore, uint8_t dev_id)
> > +{
> > + int ret;
> > + uint32_t service_id;
> > + ret = rte_event_dev_service_id_get(dev_id, &service_id);
> > + if (ret == -ESRCH) {
> > + printf("Event device [%d] doesn't need scheduling service\n",
> > + dev_id);
> > + return 0;
> > + }
> > + if (!ret) {
> > + rte_service_runstate_set(service_id, 1);
> > + rte_service_lcore_add(lcore);
> > + rte_service_map_lcore_set(service_id, lcore, 1);
> > + rte_service_lcore_start(lcore);
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +static int
> > setup_eventdev(struct prod_data *prod_data,
> > struct cons_data *cons_data,
> > struct worker_data *worker_data)
> > @@ -839,6 +851,14 @@ setup_eventdev(struct prod_data *prod_data,
> > *cons_data = (struct cons_data){.dev_id = dev_id,
> > .port_id = i };
> >
> > + for (i = 0; i < MAX_NUM_CORE; i++) {
> > + if (fdata->sched_core[i]
> > + && setup_scheduling_service(i, dev_id)) {
> > + printf("Error setting up schedulig service on %d", i);
> > + return -1;
> > + }
> > + }
>
>
> Previously, the fdata->sched_core[] array contained a "coremask" for scheduling.
> A core running the scheduling could *also* perform other work. AKA: a single core
> could perform all of RX, Sched, Worker, and TX.
>
> Due to the service-core requiring to "take" the full core, there is no option to
> have a core "split" its work into schedule() and RX,TX,Worker. This is a service core
> implementation limitation - however it should be resolved for this sample app too.
>
> The solution is to enable an ordinary DPDK (non-service-core) thread to run
> a service. This MUST be enabled at the service-cores library level, to keep atomics
> behavior of services etc), and hence removing rte_event_schedule() is still required.
>
> The changes should become simpler than proposed here, instead of the wait_schedule() hack,
> we can just run an iteration of the SW PMD using the newly-added service core iter function.
>
> I have (just) sent a patch for service-cores to enable running a service on an ordinary
> DPDK lcore, see here: http://dpdk.org/ml/archives/dev/2017-October/080022.html
>
> Hope you can rework patches 4/7 and 5/7 to use the newly provided functionality!
> Let me know if the intended usage of the new function is unclear in any way.
>
Agreed, current solution for controlled scheduling of event_sw is bit hacky,
the added flexibility of service core API helps a lot. Will rebase my patchset
on top of service core patches and spin up a v4.
Thanks,
Pavan
>
> Regards, -Harry
>
>
> > +
> > if (rte_event_dev_start(dev_id) < 0) {
> > printf("Error starting eventdev\n");
> > return -1;
> > @@ -944,8 +964,7 @@ main(int argc, char **argv)
> >
> > if (!fdata->rx_core[lcore_id] &&
> > !fdata->worker_core[lcore_id] &&
> > - !fdata->tx_core[lcore_id] &&
> > - !fdata->sched_core[lcore_id])
> > + !fdata->tx_core[lcore_id])
> > continue;
> >
> > if (fdata->rx_core[lcore_id])
> > @@ -958,10 +977,6 @@ main(int argc, char **argv)
> > "[%s()] lcore %d executing NIC Tx, and using eventdev
> > port %u\n",
> > __func__, lcore_id, cons_data.port_id);
> >
> > - if (fdata->sched_core[lcore_id])
> > - printf("[%s()] lcore %d executing scheduler\n",
> > - __func__, lcore_id);
> > -
> > if (fdata->worker_core[lcore_id])
> > printf(
> > "[%s()] lcore %d executing worker, using eventdev port
> > %u\n",
> > --
> > 2.7.4
>
@@ -46,6 +46,7 @@
#include <rte_cycles.h>
#include <rte_ethdev.h>
#include <rte_eventdev.h>
+#include <rte_service.h>
#define MAX_NUM_STAGES 8
#define BATCH_SIZE 16
@@ -233,7 +234,7 @@ producer(void)
}
static inline void
-schedule_devices(uint8_t dev_id, unsigned int lcore_id)
+schedule_devices(unsigned int lcore_id)
{
if (fdata->rx_core[lcore_id] && (fdata->rx_single ||
rte_atomic32_cmpset(&(fdata->rx_lock), 0, 1))) {
@@ -241,16 +242,6 @@ schedule_devices(uint8_t dev_id, unsigned int lcore_id)
rte_atomic32_clear((rte_atomic32_t *)&(fdata->rx_lock));
}
- if (fdata->sched_core[lcore_id] && (fdata->sched_single ||
- rte_atomic32_cmpset(&(fdata->sched_lock), 0, 1))) {
- rte_event_schedule(dev_id);
- if (cdata.dump_dev_signal) {
- rte_event_dev_dump(0, stdout);
- cdata.dump_dev_signal = 0;
- }
- rte_atomic32_clear((rte_atomic32_t *)&(fdata->sched_lock));
- }
-
if (fdata->tx_core[lcore_id] && (fdata->tx_single ||
rte_atomic32_cmpset(&(fdata->tx_lock), 0, 1))) {
consumer();
@@ -294,7 +285,7 @@ worker(void *arg)
while (!fdata->done) {
uint16_t i;
- schedule_devices(dev_id, lcore_id);
+ schedule_devices(lcore_id);
if (!fdata->worker_core[lcore_id]) {
rte_pause();
@@ -661,6 +652,27 @@ struct port_link {
};
static int
+setup_scheduling_service(unsigned int lcore, uint8_t dev_id)
+{
+ int ret;
+ uint32_t service_id;
+ ret = rte_event_dev_service_id_get(dev_id, &service_id);
+ if (ret == -ESRCH) {
+ printf("Event device [%d] doesn't need scheduling service\n",
+ dev_id);
+ return 0;
+ }
+ if (!ret) {
+ rte_service_runstate_set(service_id, 1);
+ rte_service_lcore_add(lcore);
+ rte_service_map_lcore_set(service_id, lcore, 1);
+ rte_service_lcore_start(lcore);
+ }
+
+ return ret;
+}
+
+static int
setup_eventdev(struct prod_data *prod_data,
struct cons_data *cons_data,
struct worker_data *worker_data)
@@ -839,6 +851,14 @@ setup_eventdev(struct prod_data *prod_data,
*cons_data = (struct cons_data){.dev_id = dev_id,
.port_id = i };
+ for (i = 0; i < MAX_NUM_CORE; i++) {
+ if (fdata->sched_core[i]
+ && setup_scheduling_service(i, dev_id)) {
+ printf("Error setting up schedulig service on %d", i);
+ return -1;
+ }
+ }
+
if (rte_event_dev_start(dev_id) < 0) {
printf("Error starting eventdev\n");
return -1;
@@ -944,8 +964,7 @@ main(int argc, char **argv)
if (!fdata->rx_core[lcore_id] &&
!fdata->worker_core[lcore_id] &&
- !fdata->tx_core[lcore_id] &&
- !fdata->sched_core[lcore_id])
+ !fdata->tx_core[lcore_id])
continue;
if (fdata->rx_core[lcore_id])
@@ -958,10 +977,6 @@ main(int argc, char **argv)
"[%s()] lcore %d executing NIC Tx, and using eventdev port %u\n",
__func__, lcore_id, cons_data.port_id);
- if (fdata->sched_core[lcore_id])
- printf("[%s()] lcore %d executing scheduler\n",
- __func__, lcore_id);
-
if (fdata->worker_core[lcore_id])
printf(
"[%s()] lcore %d executing worker, using eventdev port %u\n",