[v3] app/testpmd: make txonly mode generate multiple flows

  Testpmd can generate multiple flows without taking much cost and this could
be a simple traffic generator for developer's quick tests. If
"--txonly-multi-flow" is specified in the command line, IP source address
is varied to gnerate multiple flows.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---

v3:
* Add "--txonly-multi-flow" param to enable the feature.
* Replace __thread with RTE_PER_LCORE.

v2:
* Add detailed explanation in a comment.

 app/test-pmd/parameters.c             |  4 ++++
 app/test-pmd/testpmd.c                |  3 +++
 app/test-pmd/testpmd.h                |  2 ++
 app/test-pmd/txonly.c                 | 22 ++++++++++++++++++++++
 doc/guides/testpmd_app_ug/run_app.rst |  4 ++++
 5 files changed, 35 insertions(+)
  

Hi Yongseok,

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Yongseok Koh
> Sent: Thursday, March 28, 2019 1:51 AM
> To: Lu, Wenzhuo <wenzhuo.lu@intel.com>; Wu, Jingjing
> <jingjing.wu@intel.com>
> Cc: dev@dpdk.org; jerinj@marvell.com; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>
> Subject: [dpdk-dev] [PATCH v3] app/testpmd: make txonly mode generate
> multiple flows
> 
> Testpmd can generate multiple flows without taking much cost and this could
> be a simple traffic generator for developer's quick tests. If "--txonly-multi-
> flow" is specified in the command line, IP source address is varied to gnerate
> multiple flows.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
> 
> v3:
> * Add "--txonly-multi-flow" param to enable the feature.
> * Replace __thread with RTE_PER_LCORE.
> 
> v2:
> * Add detailed explanation in a comment.
> 
>  app/test-pmd/parameters.c             |  4 ++++
>  app/test-pmd/testpmd.c                |  3 +++
>  app/test-pmd/testpmd.h                |  2 ++
>  app/test-pmd/txonly.c                 | 22 ++++++++++++++++++++++
>  doc/guides/testpmd_app_ug/run_app.rst |  4 ++++
>  5 files changed, 35 insertions(+)
> 
> diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c index
> 38b419767b..7b6b60905d 100644
> --- a/app/test-pmd/parameters.c
> +++ b/app/test-pmd/parameters.c
> @@ -173,6 +173,7 @@ usage(char* progname)
>  	       " Used mainly with PCAP drivers.\n");
>  	printf("  --txpkts=X[,Y]*: set TX segment sizes"
>  		" or total packet length.\n");
> +	printf("  --txonly-multi-flow: generate multiple flows in txonly
> +mode\n");
>  	printf("  --disable-link-check: disable check on link status when "
>  	       "starting/stopping ports.\n");
>  	printf("  --no-lsc-interrupt: disable link status change interrupt.\n");
> @@ -632,6 +633,7 @@ launch_args_parse(int argc, char** argv)
>  		{ "no-flush-rx",	0, 0, 0 },
>  		{ "flow-isolate-all",	        0, 0, 0 },
>  		{ "txpkts",			1, 0, 0 },
> +		{ "txonly-multi-flow",		0, 0, 0 },
>  		{ "disable-link-check",		0, 0, 0 },
>  		{ "no-lsc-interrupt",		0, 0, 0 },
>  		{ "no-rmv-interrupt",		0, 0, 0 },
> @@ -1141,6 +1143,8 @@ launch_args_parse(int argc, char** argv)
>  				else
>  					rte_exit(EXIT_FAILURE, "bad
> txpkts\n");
>  			}
> +			if (!strcmp(lgopts[opt_idx].name, "txonly-multi-
> flow"))
> +				txonly_multi_flow = 1;
>  			if (!strcmp(lgopts[opt_idx].name, "no-flush-rx"))
>  				no_flush_rx = 1;
>  			if (!strcmp(lgopts[opt_idx].name, "disable-link-
> check")) diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
> index 216be47f92..b950e8cc55 100644
> --- a/app/test-pmd/testpmd.c
> +++ b/app/test-pmd/testpmd.c
> @@ -217,6 +217,9 @@ uint8_t  tx_pkt_nb_segs = 1; /**< Number of
> segments in TXONLY packets */  enum tx_pkt_split tx_pkt_split =
> TX_PKT_SPLIT_OFF;  /**< Split policy for packets to TX. */
> 
> +int txonly_multi_flow;

This should be "uint8_t  txonly_multi_flow;" similar to other variables in use,
 for example "uint8_t no_flush_rx = 0;"

> +/**< Whether multiple flows are generated in TXONLY mode. */
> +
>  uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per
> burst. */  uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of
> mbuf mempool cache. */
> 
> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h index
> 85b791b6bb..668227bc55 100644
> --- a/app/test-pmd/testpmd.h
> +++ b/app/test-pmd/testpmd.h
> @@ -439,6 +439,8 @@ enum tx_pkt_split {
> 
>  extern enum tx_pkt_split tx_pkt_split;
> 
> +extern int txonly_multi_flow;

This should be "extern uint8_t txonly_multi_flow;" in line with the similar variables in use,
For example  "extern uint8_t no_flush_rx;"
 
> +
>  extern uint16_t nb_pkt_per_burst;
>  extern uint16_t mb_mempool_cache;
>  extern int8_t rx_pthresh;
> diff --git a/app/test-pmd/txonly.c b/app/test-pmd/txonly.c index
> 1f08b6ed37..def52a0487 100644
> --- a/app/test-pmd/txonly.c
> +++ b/app/test-pmd/txonly.c
> @@ -52,6 +52,7 @@
>  #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)
> 
>  static struct ipv4_hdr  pkt_ip_hdr;  /**< IP header of transmitted packets. */
> +RTE_DEFINE_PER_LCORE(uint8_t, _ip_var); /**< IP address variation */
>  static struct udp_hdr pkt_udp_hdr; /**< UDP header of transmitted
> packets. */
> 
>  static void
> @@ -164,6 +165,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
>  	uint16_t vlan_tci, vlan_tci_outer;
>  	uint32_t retry;
>  	uint64_t ol_flags = 0;
> +	uint8_t  ip_var = RTE_PER_LCORE(_ip_var);
>  	uint8_t  i;
>  	uint64_t tx_offloads;
>  #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
> @@ -237,6 +239,23 @@ pkt_burst_transmit(struct fwd_stream *fs)
>  		copy_buf_to_pkt(&eth_hdr, sizeof(eth_hdr), pkt, 0);
>  		copy_buf_to_pkt(&pkt_ip_hdr, sizeof(pkt_ip_hdr), pkt,
>  				sizeof(struct ether_hdr));
> +		if (txonly_multi_flow) {
> +			struct ipv4_hdr *ip_hdr;
> +			uint32_t addr;
> +
> +			ip_hdr = rte_pktmbuf_mtod_offset(pkt,
> +					struct ipv4_hdr *,
> +					sizeof(struct ether_hdr));
> +			/*
> +			 * Generate multiple flows by varying IP src addr. This
> +			 * enables packets are well distributed by RSS in
> +			 * receiver side if any and txonly mode can be a
> decent
> +			 * packet generator for developer's quick
> performance
> +			 * regression test.
> +			 */
> +			addr = (IP_DST_ADDR | (ip_var++ << 8)) +
> rte_lcore_id();
> +			ip_hdr->src_addr = rte_cpu_to_be_32(addr);
> +		}
>  		copy_buf_to_pkt(&pkt_udp_hdr, sizeof(pkt_udp_hdr), pkt,
>  				sizeof(struct ether_hdr) +
>  				sizeof(struct ipv4_hdr));
> @@ -268,6 +287,9 @@ pkt_burst_transmit(struct fwd_stream *fs)
>  	}
>  	fs->tx_packets += nb_tx;
> 
> +	if (txonly_multi_flow)
> +		RTE_PER_LCORE(_ip_var) += nb_tx;
> +
>  #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
>  	fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
>  #endif
> diff --git a/doc/guides/testpmd_app_ug/run_app.rst
> b/doc/guides/testpmd_app_ug/run_app.rst
> index 4495ed0382..b717b8c7b7 100644
> --- a/doc/guides/testpmd_app_ug/run_app.rst
> +++ b/doc/guides/testpmd_app_ug/run_app.rst
> @@ -324,6 +324,10 @@ The commandline options are:
>      Set TX segment sizes or total packet length. Valid for ``tx-only``
>      and ``flowgen`` forwarding modes.
> 
> +*   ``--txonly-multi-flow``
> +
> +    Generate multiple flows in txonly mode.
> +
>  *   ``--disable-link-check``
> 
>      Disable check on link status when starting/stopping ports.
> --
> 2.11.0
  

> On Mar 28, 2019, at 3:24 AM, Iremonger, Bernard <bernard.iremonger@intel.com> wrote:
> 
> Hi Yongseok,
> 
>> -----Original Message-----
>> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Yongseok Koh
>> Sent: Thursday, March 28, 2019 1:51 AM
>> To: Lu, Wenzhuo <wenzhuo.lu@intel.com>; Wu, Jingjing
>> <jingjing.wu@intel.com>
>> Cc: dev@dpdk.org; jerinj@marvell.com; Ananyev, Konstantin
>> <konstantin.ananyev@intel.com>
>> Subject: [dpdk-dev] [PATCH v3] app/testpmd: make txonly mode generate
>> multiple flows
>> 
>> Testpmd can generate multiple flows without taking much cost and this could
>> be a simple traffic generator for developer's quick tests. If "--txonly-multi-
>> flow" is specified in the command line, IP source address is varied to gnerate
>> multiple flows.
>> 
>> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
>> ---
>> 
>> v3:
>> * Add "--txonly-multi-flow" param to enable the feature.
>> * Replace __thread with RTE_PER_LCORE.
>> 
>> v2:
>> * Add detailed explanation in a comment.
>> 
>> app/test-pmd/parameters.c             |  4 ++++
>> app/test-pmd/testpmd.c                |  3 +++
>> app/test-pmd/testpmd.h                |  2 ++
>> app/test-pmd/txonly.c                 | 22 ++++++++++++++++++++++
>> doc/guides/testpmd_app_ug/run_app.rst |  4 ++++
>> 5 files changed, 35 insertions(+)
>> 
>> diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c index
>> 38b419767b..7b6b60905d 100644
>> --- a/app/test-pmd/parameters.c
>> +++ b/app/test-pmd/parameters.c
>> @@ -173,6 +173,7 @@ usage(char* progname)
>> 	       " Used mainly with PCAP drivers.\n");
>> 	printf("  --txpkts=X[,Y]*: set TX segment sizes"
>> 		" or total packet length.\n");
>> +	printf("  --txonly-multi-flow: generate multiple flows in txonly
>> +mode\n");
>> 	printf("  --disable-link-check: disable check on link status when "
>> 	       "starting/stopping ports.\n");
>> 	printf("  --no-lsc-interrupt: disable link status change interrupt.\n");
>> @@ -632,6 +633,7 @@ launch_args_parse(int argc, char** argv)
>> 		{ "no-flush-rx",	0, 0, 0 },
>> 		{ "flow-isolate-all",	        0, 0, 0 },
>> 		{ "txpkts",			1, 0, 0 },
>> +		{ "txonly-multi-flow",		0, 0, 0 },
>> 		{ "disable-link-check",		0, 0, 0 },
>> 		{ "no-lsc-interrupt",		0, 0, 0 },
>> 		{ "no-rmv-interrupt",		0, 0, 0 },
>> @@ -1141,6 +1143,8 @@ launch_args_parse(int argc, char** argv)
>> 				else
>> 					rte_exit(EXIT_FAILURE, "bad
>> txpkts\n");
>> 			}
>> +			if (!strcmp(lgopts[opt_idx].name, "txonly-multi-
>> flow"))
>> +				txonly_multi_flow = 1;
>> 			if (!strcmp(lgopts[opt_idx].name, "no-flush-rx"))
>> 				no_flush_rx = 1;
>> 			if (!strcmp(lgopts[opt_idx].name, "disable-link-
>> check")) diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
>> index 216be47f92..b950e8cc55 100644
>> --- a/app/test-pmd/testpmd.c
>> +++ b/app/test-pmd/testpmd.c
>> @@ -217,6 +217,9 @@ uint8_t  tx_pkt_nb_segs = 1; /**< Number of
>> segments in TXONLY packets */  enum tx_pkt_split tx_pkt_split =
>> TX_PKT_SPLIT_OFF;  /**< Split policy for packets to TX. */
>> 
>> +int txonly_multi_flow;
> 
> This should be "uint8_t  txonly_multi_flow;" similar to other variables in use,
> for example "uint8_t no_flush_rx = 0;"
> 
>> +/**< Whether multiple flows are generated in TXONLY mode. */
>> +
>> uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per
>> burst. */  uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of
>> mbuf mempool cache. */
>> 
>> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h index
>> 85b791b6bb..668227bc55 100644
>> --- a/app/test-pmd/testpmd.h
>> +++ b/app/test-pmd/testpmd.h
>> @@ -439,6 +439,8 @@ enum tx_pkt_split {
>> 
>> extern enum tx_pkt_split tx_pkt_split;
>> 
>> +extern int txonly_multi_flow;
> 
> This should be "extern uint8_t txonly_multi_flow;" in line with the similar variables in use,
> For example  "extern uint8_t no_flush_rx;"

Okay, will change it to uint8_t but what about the error by checkpatches.sh?

	+ ./devtools/checkpatches.sh -n1

	### app/testpmd: make txonly mode generate multiple flows

	ERROR:GLOBAL_INITIALISERS: do not initialise globals to 0
	#50: FILE: app/test-pmd/testpmd.c:220:
	+uint8_t txonly_multi_flow = 0;

	total: 1 errors, 0 warnings, 95 lines checked

	0/1 valid patch

I'll submit without the init but if you still insist that way for consistency,
let me know then I'll submit another version.

Thanks,
Yongseok

> 
>> +
>> extern uint16_t nb_pkt_per_burst;
>> extern uint16_t mb_mempool_cache;
>> extern int8_t rx_pthresh;
>> diff --git a/app/test-pmd/txonly.c b/app/test-pmd/txonly.c index
>> 1f08b6ed37..def52a0487 100644
>> --- a/app/test-pmd/txonly.c
>> +++ b/app/test-pmd/txonly.c
>> @@ -52,6 +52,7 @@
>> #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)
>> 
>> static struct ipv4_hdr  pkt_ip_hdr;  /**< IP header of transmitted packets. */
>> +RTE_DEFINE_PER_LCORE(uint8_t, _ip_var); /**< IP address variation */
>> static struct udp_hdr pkt_udp_hdr; /**< UDP header of transmitted
>> packets. */
>> 
>> static void
>> @@ -164,6 +165,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
>> 	uint16_t vlan_tci, vlan_tci_outer;
>> 	uint32_t retry;
>> 	uint64_t ol_flags = 0;
>> +	uint8_t  ip_var = RTE_PER_LCORE(_ip_var);
>> 	uint8_t  i;
>> 	uint64_t tx_offloads;
>> #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
>> @@ -237,6 +239,23 @@ pkt_burst_transmit(struct fwd_stream *fs)
>> 		copy_buf_to_pkt(&eth_hdr, sizeof(eth_hdr), pkt, 0);
>> 		copy_buf_to_pkt(&pkt_ip_hdr, sizeof(pkt_ip_hdr), pkt,
>> 				sizeof(struct ether_hdr));
>> +		if (txonly_multi_flow) {
>> +			struct ipv4_hdr *ip_hdr;
>> +			uint32_t addr;
>> +
>> +			ip_hdr = rte_pktmbuf_mtod_offset(pkt,
>> +					struct ipv4_hdr *,
>> +					sizeof(struct ether_hdr));
>> +			/*
>> +			 * Generate multiple flows by varying IP src addr. This
>> +			 * enables packets are well distributed by RSS in
>> +			 * receiver side if any and txonly mode can be a
>> decent
>> +			 * packet generator for developer's quick
>> performance
>> +			 * regression test.
>> +			 */
>> +			addr = (IP_DST_ADDR | (ip_var++ << 8)) +
>> rte_lcore_id();
>> +			ip_hdr->src_addr = rte_cpu_to_be_32(addr);
>> +		}
>> 		copy_buf_to_pkt(&pkt_udp_hdr, sizeof(pkt_udp_hdr), pkt,
>> 				sizeof(struct ether_hdr) +
>> 				sizeof(struct ipv4_hdr));
>> @@ -268,6 +287,9 @@ pkt_burst_transmit(struct fwd_stream *fs)
>> 	}
>> 	fs->tx_packets += nb_tx;
>> 
>> +	if (txonly_multi_flow)
>> +		RTE_PER_LCORE(_ip_var) += nb_tx;
>> +
>> #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
>> 	fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
>> #endif
>> diff --git a/doc/guides/testpmd_app_ug/run_app.rst
>> b/doc/guides/testpmd_app_ug/run_app.rst
>> index 4495ed0382..b717b8c7b7 100644
>> --- a/doc/guides/testpmd_app_ug/run_app.rst
>> +++ b/doc/guides/testpmd_app_ug/run_app.rst
>> @@ -324,6 +324,10 @@ The commandline options are:
>>     Set TX segment sizes or total packet length. Valid for ``tx-only``
>>     and ``flowgen`` forwarding modes.
>> 
>> +*   ``--txonly-multi-flow``
>> +
>> +    Generate multiple flows in txonly mode.
>> +
>> *   ``--disable-link-check``
>> 
>>     Disable check on link status when starting/stopping ports.
>> --
>> 2.11.0
  

Hi Yongseok,

<snip>

> >> Subject: [dpdk-dev] [PATCH v3] app/testpmd: make txonly mode generate
> >> multiple flows
> >>
> >> Testpmd can generate multiple flows without taking much cost and this
> >> could be a simple traffic generator for developer's quick tests. If
> >> "--txonly-multi- flow" is specified in the command line, IP source
> >> address is varied to gnerate multiple flows.
> >>
> >> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> >> ---
> >>
> >> v3:
> >> * Add "--txonly-multi-flow" param to enable the feature.
> >> * Replace __thread with RTE_PER_LCORE.
> >>
> >> v2:
> >> * Add detailed explanation in a comment.
> >>
> >> app/test-pmd/parameters.c             |  4 ++++
> >> app/test-pmd/testpmd.c                |  3 +++
> >> app/test-pmd/testpmd.h                |  2 ++
> >> app/test-pmd/txonly.c                 | 22 ++++++++++++++++++++++
> >> doc/guides/testpmd_app_ug/run_app.rst |  4 ++++
> >> 5 files changed, 35 insertions(+)
> >>
> >> diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
> >> index 38b419767b..7b6b60905d 100644
> >> --- a/app/test-pmd/parameters.c
> >> +++ b/app/test-pmd/parameters.c
> >> @@ -173,6 +173,7 @@ usage(char* progname)
> >> 	       " Used mainly with PCAP drivers.\n");
> >> 	printf("  --txpkts=X[,Y]*: set TX segment sizes"
> >> 		" or total packet length.\n");
> >> +	printf("  --txonly-multi-flow: generate multiple flows in txonly
> >> +mode\n");
> >> 	printf("  --disable-link-check: disable check on link status when "
> >> 	       "starting/stopping ports.\n");
> >> 	printf("  --no-lsc-interrupt: disable link status change
> >> interrupt.\n"); @@ -632,6 +633,7 @@ launch_args_parse(int argc, char**
> argv)
> >> 		{ "no-flush-rx",	0, 0, 0 },
> >> 		{ "flow-isolate-all",	        0, 0, 0 },
> >> 		{ "txpkts",			1, 0, 0 },
> >> +		{ "txonly-multi-flow",		0, 0, 0 },
> >> 		{ "disable-link-check",		0, 0, 0 },
> >> 		{ "no-lsc-interrupt",		0, 0, 0 },
> >> 		{ "no-rmv-interrupt",		0, 0, 0 },
> >> @@ -1141,6 +1143,8 @@ launch_args_parse(int argc, char** argv)
> >> 				else
> >> 					rte_exit(EXIT_FAILURE, "bad
> >> txpkts\n");
> >> 			}
> >> +			if (!strcmp(lgopts[opt_idx].name, "txonly-multi-
> >> flow"))
> >> +				txonly_multi_flow = 1;
> >> 			if (!strcmp(lgopts[opt_idx].name, "no-flush-rx"))
> >> 				no_flush_rx = 1;
> >> 			if (!strcmp(lgopts[opt_idx].name, "disable-link-
> >> check")) diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
> >> index 216be47f92..b950e8cc55 100644
> >> --- a/app/test-pmd/testpmd.c
> >> +++ b/app/test-pmd/testpmd.c
> >> @@ -217,6 +217,9 @@ uint8_t  tx_pkt_nb_segs = 1; /**< Number of
> >> segments in TXONLY packets */  enum tx_pkt_split tx_pkt_split =
> >> TX_PKT_SPLIT_OFF;  /**< Split policy for packets to TX. */
> >>
> >> +int txonly_multi_flow;
> >
> > This should be "uint8_t  txonly_multi_flow;" similar to other
> > variables in use, for example "uint8_t no_flush_rx = 0;"
> >
> >> +/**< Whether multiple flows are generated in TXONLY mode. */
> >> +
> >> uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per
> >> burst. */  uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of
> >> mbuf mempool cache. */
> >>
> >> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h index
> >> 85b791b6bb..668227bc55 100644
> >> --- a/app/test-pmd/testpmd.h
> >> +++ b/app/test-pmd/testpmd.h
> >> @@ -439,6 +439,8 @@ enum tx_pkt_split {
> >>
> >> extern enum tx_pkt_split tx_pkt_split;
> >>
> >> +extern int txonly_multi_flow;
> >
> > This should be "extern uint8_t txonly_multi_flow;" in line with the
> > similar variables in use, For example  "extern uint8_t no_flush_rx;"
> 
> Okay, will change it to uint8_t but what about the error by checkpatches.sh?
> 
> 	+ ./devtools/checkpatches.sh -n1
> 
> 	### app/testpmd: make txonly mode generate multiple flows
> 
> 	ERROR:GLOBAL_INITIALISERS: do not initialise globals to 0
> 	#50: FILE: app/test-pmd/testpmd.c:220:
> 	+uint8_t txonly_multi_flow = 0;
> 
> 	total: 1 errors, 0 warnings, 95 lines checked
> 
> 	0/1 valid patch
> 
> I'll submit without the init but if you still insist that way for consistency, let me
> know then I'll submit another version.
> 
> Thanks,
> Yongseok

Please submit without the init to avoid checkpatch error (example probably predates checkpatch checks).

Regards,

Bernard.