[dpdk-dev,v4] net/mlx5: load libmlx5 and libibverbs in run-time
Checks
Commit Message
MLX5 PMD loads libraries: libibverbs and libmlx5.
MLX5 PMD is not linked to external libraries.
Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
v1:
load external libraries in run-time
v2:
* fix checkpatch warnings
v3:
* fix checkpatch warnings
v4:
New MACROs in order to reuse code
---
config/common_base | 1 +
drivers/net/mlx5/Makefile | 22 ++-
drivers/net/mlx5/lib/mlx5_dll.c | 294 +++++++++++++++++++++++++++++++++++++++
drivers/net/mlx5/lib/mlx5_dll.h | 103 ++++++++++++++
drivers/net/mlx5/mlx5.c | 17 ++-
drivers/net/mlx5/mlx5.h | 4 +
drivers/net/mlx5/mlx5_flow.c | 4 +
drivers/net/mlx5/mlx5_mac.c | 4 +
drivers/net/mlx5/mlx5_mr.c | 4 +
drivers/net/mlx5/mlx5_rss.c | 4 +
drivers/net/mlx5/mlx5_rxmode.c | 4 +
drivers/net/mlx5/mlx5_rxq.c | 4 +
drivers/net/mlx5/mlx5_rxtx.c | 4 +
drivers/net/mlx5/mlx5_rxtx.h | 6 +-
drivers/net/mlx5/mlx5_rxtx_vec.c | 4 +
drivers/net/mlx5/mlx5_txq.c | 4 +
mk/rte.app.mk | 8 +-
17 files changed, 479 insertions(+), 12 deletions(-)
create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h
Comments
Hi Shachar,
Please see small comment bellow,
On Sun, Dec 31, 2017 at 07:52:51AM +0000, Shachar Beiser wrote:
> MLX5 PMD loads libraries: libibverbs and libmlx5.
> MLX5 PMD is not linked to external libraries.
>
> Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
> ---
> v1:
> load external libraries in run-time
> v2:
> * fix checkpatch warnings
> v3:
> * fix checkpatch warnings
> v4:
> New MACROs in order to reuse code
> ---
> config/common_base | 1 +
> drivers/net/mlx5/Makefile | 22 ++-
> drivers/net/mlx5/lib/mlx5_dll.c | 294 +++++++++++++++++++++++++++++++++++++++
> drivers/net/mlx5/lib/mlx5_dll.h | 103 ++++++++++++++
> drivers/net/mlx5/mlx5.c | 17 ++-
> drivers/net/mlx5/mlx5.h | 4 +
> drivers/net/mlx5/mlx5_flow.c | 4 +
> drivers/net/mlx5/mlx5_mac.c | 4 +
> drivers/net/mlx5/mlx5_mr.c | 4 +
> drivers/net/mlx5/mlx5_rss.c | 4 +
> drivers/net/mlx5/mlx5_rxmode.c | 4 +
> drivers/net/mlx5/mlx5_rxq.c | 4 +
> drivers/net/mlx5/mlx5_rxtx.c | 4 +
> drivers/net/mlx5/mlx5_rxtx.h | 6 +-
> drivers/net/mlx5/mlx5_rxtx_vec.c | 4 +
> drivers/net/mlx5/mlx5_txq.c | 4 +
> mk/rte.app.mk | 8 +-
> 17 files changed, 479 insertions(+), 12 deletions(-)
> create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
> create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h
>
> diff --git a/config/common_base b/config/common_base
> index b8ee8f9..30c8fcf 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> #
> CONFIG_RTE_LIBRTE_MLX5_PMD=n
> +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
Not sure a new configuration item is allowed. If it is, the
documentation of such variable is missing.
> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> index a3984eb..24fa127 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
> SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
> SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
> SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> -
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
> +endif
> # Basic CFLAGS.
> CFLAGS += -O3
> CFLAGS += -std=c11 -Wall -Wextra
> CFLAGS += -g
> CFLAGS += -I.
> +CFLAGS += -I$(SRCDIR)
> CFLAGS += -D_BSD_SOURCE
> CFLAGS += -D_DEFAULT_SOURCE
> CFLAGS += -D_XOPEN_SOURCE=600
> CFLAGS += $(WERROR_FLAGS)
> CFLAGS += -Wno-strict-prototypes
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +LDLIBS += -ldl
> +else
> LDLIBS += -libverbs -lmlx5
> +endif
> LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
> LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
> LDLIBS += -lrte_bus_pci
> @@ -105,26 +112,28 @@ endif
>
> mlx5_autoconf.h.new: FORCE
>
> +VERBS_H := infiniband/verbs.h
> +MLX5DV_H := infiniband/mlx5dv.h
> mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> $Q $(RM) -f -- '$@'
> $Q sh -- '$<' '$@' \
> HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> - infiniband/verbs.h \
> + $(VERBS_H) \
> enum IBV_DEVICE_VXLAN_SUPPORT \
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> - infiniband/verbs.h \
> + $(VERBS_H) \
> enum IBV_WQ_FLAG_RX_END_PADDING \
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> HAVE_IBV_MLX5_MOD_MPW \
> - infiniband/mlx5dv.h \
> + $(MLX5DV_H) \
> enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> - infiniband/mlx5dv.h \
> + $(MLX5DV_H) \
> enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> @@ -144,10 +153,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> - infiniband/verbs.h \
> + $(VERBS_H) \
> enum IBV_FLOW_SPEC_ACTION_COUNT \
> $(AUTOCONF_OUTPUT)
This modification should be inside its own patch, it is not directly
related to the this patch itself.
> -
> # Create mlx5_autoconf.h or update it in case it differs from the new one.
>
> mlx5_autoconf.h: mlx5_autoconf.h.new
<snip/>
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index cd66fe1..eeef782 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -30,7 +30,8 @@
> * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> */
> -
> +#define _GNU_SOURCE
> +#include <stdio.h>
> #include <stddef.h>
> #include <unistd.h>
> #include <string.h>
> @@ -39,13 +40,17 @@
> #include <stdlib.h>
> #include <errno.h>
> #include <net/if.h>
> -
> +#include <dlfcn.h>
The empty line should remain.
> /* Verbs header. */
> /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> +#endif
This could be done by the mlx5_dll.h file which could include the
correct header according to the configuration.
<snip/>
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk
> index 6a6a745..2dd2f6b 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -51,7 +51,9 @@ endif
>
> # Link only the libraries used in the application
> LDFLAGS += --as-needed
> -
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +LDFLAGS += -ldl
> +endif
> # default path for libs
> _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
>
> @@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI) += -lrte_pmd_kni
> endif
> _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += -lrte_pmd_lio
> _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs -lmlx4
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -ldl
> +else
> _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs -lmlx5
> +endif
> _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
> _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp
> _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += -lrte_pmd_null
> --
> 1.8.3.1
>
Thanks,
> -----Original Message-----
> From: Nelio Laranjeiro [mailto:nelio.laranjeiro@6wind.com]
> Sent: Tuesday, January 2, 2018 4:07 PM
> To: Shachar Beiser <shacharbe@mellanox.com>
> Cc: dev@dpdk.org; Adrien Mazarguil <adrien.mazarguil@6wind.com>
> Subject: Re: [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time
>
> Hi Shachar,
>
> Please see small comment bellow,
>
> On Sun, Dec 31, 2017 at 07:52:51AM +0000, Shachar Beiser wrote:
> > MLX5 PMD loads libraries: libibverbs and libmlx5.
> > MLX5 PMD is not linked to external libraries.
> >
> > Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
> > ---
> > v1:
> > load external libraries in run-time
> > v2:
> > * fix checkpatch warnings
> > v3:
> > * fix checkpatch warnings
> > v4:
> > New MACROs in order to reuse code
> > ---
> > config/common_base | 1 +
> > drivers/net/mlx5/Makefile | 22 ++-
> > drivers/net/mlx5/lib/mlx5_dll.c | 294
> > +++++++++++++++++++++++++++++++++++++++
> > drivers/net/mlx5/lib/mlx5_dll.h | 103 ++++++++++++++
> > drivers/net/mlx5/mlx5.c | 17 ++-
> > drivers/net/mlx5/mlx5.h | 4 +
> > drivers/net/mlx5/mlx5_flow.c | 4 +
> > drivers/net/mlx5/mlx5_mac.c | 4 +
> > drivers/net/mlx5/mlx5_mr.c | 4 +
> > drivers/net/mlx5/mlx5_rss.c | 4 +
> > drivers/net/mlx5/mlx5_rxmode.c | 4 +
> > drivers/net/mlx5/mlx5_rxq.c | 4 +
> > drivers/net/mlx5/mlx5_rxtx.c | 4 +
> > drivers/net/mlx5/mlx5_rxtx.h | 6 +-
> > drivers/net/mlx5/mlx5_rxtx_vec.c | 4 +
> > drivers/net/mlx5/mlx5_txq.c | 4 +
> > mk/rte.app.mk | 8 +-
> > 17 files changed, 479 insertions(+), 12 deletions(-) create mode
> > 100644 drivers/net/mlx5/lib/mlx5_dll.c create mode 100644
> > drivers/net/mlx5/lib/mlx5_dll.h
> >
> > diff --git a/config/common_base b/config/common_base index
> > b8ee8f9..30c8fcf 100644
> > --- a/config/common_base
> > +++ b/config/common_base
> > @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> > # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> > # CONFIG_RTE_LIBRTE_MLX5_PMD=n
> > +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> > CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> > CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
>
> Not sure a new configuration item is allowed. If it is, the documentation of
> such variable is missing.
[S.B] The patch is based on this CONFIG_RTE_LIBRTE_MLX5_DLL , it was required by Adrian in the design phase to enable/disable this linkage mode.
I will update the documentation .
>
> > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> > index a3984eb..24fa127 100644
> > --- a/drivers/net/mlx5/Makefile
> > +++ b/drivers/net/mlx5/Makefile
> > @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) +=
> mlx5_rss.c
> > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
> > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
> > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> > -
> > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> > +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c endif
> > # Basic CFLAGS.
> > CFLAGS += -O3
> > CFLAGS += -std=c11 -Wall -Wextra
> > CFLAGS += -g
> > CFLAGS += -I.
> > +CFLAGS += -I$(SRCDIR)
> > CFLAGS += -D_BSD_SOURCE
> > CFLAGS += -D_DEFAULT_SOURCE
> > CFLAGS += -D_XOPEN_SOURCE=600
> > CFLAGS += $(WERROR_FLAGS)
> > CFLAGS += -Wno-strict-prototypes
> > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) LDLIBS += -ldl else
> > LDLIBS += -libverbs -lmlx5
> > +endif
> > LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS +=
> > -lrte_ethdev -lrte_net -lrte_kvargs LDLIBS += -lrte_bus_pci @@
> > -105,26 +112,28 @@ endif
> >
> > mlx5_autoconf.h.new: FORCE
> >
> > +VERBS_H := infiniband/verbs.h
> > +MLX5DV_H := infiniband/mlx5dv.h
> > mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> > $Q $(RM) -f -- '$@'
> > $Q sh -- '$<' '$@' \
> > HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> > - infiniband/verbs.h \
> > + $(VERBS_H) \
> > enum IBV_DEVICE_VXLAN_SUPPORT \
> > $(AUTOCONF_OUTPUT)
> > $Q sh -- '$<' '$@' \
> > HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> > - infiniband/verbs.h \
> > + $(VERBS_H) \
> > enum IBV_WQ_FLAG_RX_END_PADDING \
> > $(AUTOCONF_OUTPUT)
> > $Q sh -- '$<' '$@' \
> > HAVE_IBV_MLX5_MOD_MPW \
> > - infiniband/mlx5dv.h \
> > + $(MLX5DV_H) \
> > enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
> > $(AUTOCONF_OUTPUT)
> > $Q sh -- '$<' '$@' \
> > HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> > - infiniband/mlx5dv.h \
> > + $(MLX5DV_H) \
> > enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
> > $(AUTOCONF_OUTPUT)
> > $Q sh -- '$<' '$@' \
> > @@ -144,10 +153,9 @@ mlx5_autoconf.h.new:
> $(RTE_SDK)/buildtools/auto-config-h.sh
> > $(AUTOCONF_OUTPUT)
> > $Q sh -- '$<' '$@' \
> > HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> > - infiniband/verbs.h \
> > + $(VERBS_H) \
> > enum IBV_FLOW_SPEC_ACTION_COUNT \
> > $(AUTOCONF_OUTPUT)
>
> This modification should be inside its own patch, it is not directly related to
> the this patch itself.
[S.B] I can revert the VERBS_H , MLX5DV_H and not change it all .
There is no need to change in a different patch.
>
> > -
> > # Create mlx5_autoconf.h or update it in case it differs from the new one.
> >
> > mlx5_autoconf.h: mlx5_autoconf.h.new
> <snip/>
> > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > cd66fe1..eeef782 100644
> > --- a/drivers/net/mlx5/mlx5.c
> > +++ b/drivers/net/mlx5/mlx5.c
> > @@ -30,7 +30,8 @@
> > * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
> OF THE USE
> > * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> DAMAGE.
> > */
> > -
> > +#define _GNU_SOURCE
> > +#include <stdio.h>
> > #include <stddef.h>
> > #include <unistd.h>
> > #include <string.h>
> > @@ -39,13 +40,17 @@
> > #include <stdlib.h>
> > #include <errno.h>
> > #include <net/if.h>
> > -
> > +#include <dlfcn.h>
>
> The empty line should remain.
[S.B] OK.
>
> > /* Verbs header. */
> > /* ISO C doesn't support unnamed structs/unions, disabling -pedantic.
> > */ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic"
> > #endif
> > +#ifdef RTE_LIBRTE_MLX5_DLL
> > +#include "lib/mlx5_dll.h"
> > +#else
> > #include <infiniband/verbs.h>
> > +#endif
>
> This could be done by the mlx5_dll.h file which could include the correct
> header according to the configuration.
[S.B] I guess you refer to all *.c files that includes the verbs.h .
I will change them all ?
>
> <snip/>
> > diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 6a6a745..2dd2f6b
> > 100644
> > --- a/mk/rte.app.mk
> > +++ b/mk/rte.app.mk
> > @@ -51,7 +51,9 @@ endif
> >
> > # Link only the libraries used in the application LDFLAGS +=
> > --as-needed
> > -
> > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) LDFLAGS += -ldl endif
> > # default path for libs
> > _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
> >
> > @@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI) +=
> -lrte_pmd_kni
> > endif
> > _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += -lrte_pmd_lio
> > _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -
> libverbs -lmlx4
> > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> > +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -ldl
> > +else
> > _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -
> libverbs -lmlx5
> > +endif
> > _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += -lrte_pmd_mrvl -
> L$(LIBMUSDK_PATH)/lib -lmusdk
> > _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp
> > _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += -lrte_pmd_null
> > --
> > 1.8.3.1
> >
>
> Thanks,
>
> --
> Nélio Laranjeiro
> 6WIND
Hi Shachar,
On Wed, Jan 03, 2018 at 03:00:46PM +0000, Shachar Beiser wrote:
<snip/>
> > > --- a/config/common_base
> > > +++ b/config/common_base
> > > @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> > > # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> > > # CONFIG_RTE_LIBRTE_MLX5_PMD=n
> > > +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> > > CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> > > CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
> >
> > Not sure a new configuration item is allowed. If it is, the documentation of
> > such variable is missing.
>
> [S.B] The patch is based on this CONFIG_RTE_LIBRTE_MLX5_DLL , it was
> required by Adrian in the design phase to enable/disable this linkage
> mode.
> I will update the documentation .
Before updating the documentation you should speak with Thomas or
Ferruh, not sure new items are allowed anymore.
> > > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> > > index a3984eb..24fa127 100644
> > > --- a/drivers/net/mlx5/Makefile
> > > +++ b/drivers/net/mlx5/Makefile
> > > @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) +=
> > mlx5_rss.c
> > > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
> > > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
> > > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> > > -
> > > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> > > +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c endif
> > > # Basic CFLAGS.
> > > CFLAGS += -O3
> > > CFLAGS += -std=c11 -Wall -Wextra
> > > CFLAGS += -g
> > > CFLAGS += -I.
> > > +CFLAGS += -I$(SRCDIR)
> > > CFLAGS += -D_BSD_SOURCE
> > > CFLAGS += -D_DEFAULT_SOURCE
> > > CFLAGS += -D_XOPEN_SOURCE=600
> > > CFLAGS += $(WERROR_FLAGS)
> > > CFLAGS += -Wno-strict-prototypes
> > > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) LDLIBS += -ldl else
> > > LDLIBS += -libverbs -lmlx5
> > > +endif
> > > LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS +=
> > > -lrte_ethdev -lrte_net -lrte_kvargs LDLIBS += -lrte_bus_pci @@
> > > -105,26 +112,28 @@ endif
> > >
> > > mlx5_autoconf.h.new: FORCE
> > >
> > > +VERBS_H := infiniband/verbs.h
> > > +MLX5DV_H := infiniband/mlx5dv.h
> > > mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> > > $Q $(RM) -f -- '$@'
> > > $Q sh -- '$<' '$@' \
> > > HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> > > - infiniband/verbs.h \
> > > + $(VERBS_H) \
> > > enum IBV_DEVICE_VXLAN_SUPPORT \
> > > $(AUTOCONF_OUTPUT)
> > > $Q sh -- '$<' '$@' \
> > > HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> > > - infiniband/verbs.h \
> > > + $(VERBS_H) \
> > > enum IBV_WQ_FLAG_RX_END_PADDING \
> > > $(AUTOCONF_OUTPUT)
> > > $Q sh -- '$<' '$@' \
> > > HAVE_IBV_MLX5_MOD_MPW \
> > > - infiniband/mlx5dv.h \
> > > + $(MLX5DV_H) \
> > > enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
> > > $(AUTOCONF_OUTPUT)
> > > $Q sh -- '$<' '$@' \
> > > HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> > > - infiniband/mlx5dv.h \
> > > + $(MLX5DV_H) \
> > > enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
> > > $(AUTOCONF_OUTPUT)
> > > $Q sh -- '$<' '$@' \
> > > @@ -144,10 +153,9 @@ mlx5_autoconf.h.new:
> > $(RTE_SDK)/buildtools/auto-config-h.sh
> > > $(AUTOCONF_OUTPUT)
> > > $Q sh -- '$<' '$@' \
> > > HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> > > - infiniband/verbs.h \
> > > + $(VERBS_H) \
> > > enum IBV_FLOW_SPEC_ACTION_COUNT \
> > > $(AUTOCONF_OUTPUT)
> >
> > This modification should be inside its own patch, it is not directly related to
> > the this patch itself.
> [S.B] I can revert the VERBS_H , MLX5DV_H and not change it all .
> There is no need to change in a different patch.
As you wish.
> > > -
> > > # Create mlx5_autoconf.h or update it in case it differs from the new one.
> > >
> > > mlx5_autoconf.h: mlx5_autoconf.h.new
> > <snip/>
> > > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > > cd66fe1..eeef782 100644
> > > --- a/drivers/net/mlx5/mlx5.c
> > > +++ b/drivers/net/mlx5/mlx5.c
> > > @@ -30,7 +30,8 @@
> > > * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
> > OF THE USE
> > > * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> > DAMAGE.
> > > */
> > > -
> > > +#define _GNU_SOURCE
> > > +#include <stdio.h>
> > > #include <stddef.h>
> > > #include <unistd.h>
> > > #include <string.h>
> > > @@ -39,13 +40,17 @@
> > > #include <stdlib.h>
> > > #include <errno.h>
> > > #include <net/if.h>
> > > -
> > > +#include <dlfcn.h>
> >
> > The empty line should remain.
> [S.B] OK.
To be sure, the empty line should be between the system include and
verbs ones.
> >
> > > /* Verbs header. */
> > > /* ISO C doesn't support unnamed structs/unions, disabling -pedantic.
> > > */ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic"
> > > #endif
> > > +#ifdef RTE_LIBRTE_MLX5_DLL
> > > +#include "lib/mlx5_dll.h"
> > > +#else
> > > #include <infiniband/verbs.h>
> > > +#endif
> >
> > This could be done by the mlx5_dll.h file which could include the correct
> > header according to the configuration.
> [S.B] I guess you refer to all *.c files that includes the verbs.h .
> I will change them all ?
Yes
<snip/>
Thanks,
Hi,
31/12/2017 08:52, Shachar Beiser:
> MLX5 PMD loads libraries: libibverbs and libmlx5.
> MLX5 PMD is not linked to external libraries.
>
> Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
Please better describe the change, even if it is obvious:
how it is before, after and why making this change.
> --- /dev/null
> +++ b/drivers/net/mlx5/lib/mlx5_dll.c
What means "dll"? Isn't it a Windows wording?
Maybe this file is better described as "verbs glue"?
> @@ -0,0 +1,294 @@
> +/*-
> + * BSD LICENSE
> + *
> + * Copyright 2017 6WIND S.A.
> + * Copyright 2017 Mellanox.
I think it should be
Copyright 2017 Mellanox Technologies, Ltd.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + *
> + * * Redistributions of source code must retain the above copyright
> + * notice, this list of conditions and the following disclaimer.
> + * * Redistributions in binary form must reproduce the above copyright
> + * notice, this list of conditions and the following disclaimer in
> + * the documentation and/or other materials provided with the
> + * distribution.
> + * * Neither the name of 6WIND S.A. nor the names of its
> + * contributors may be used to endorse or promote products derived
> + * from this software without specific prior written permission.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
Please replace the license text by SPDX identifier.
Example: https://dpdk.org/patch/32809/
04/01/2018 08:36, Nélio Laranjeiro:
> Hi Shachar,
>
> On Wed, Jan 03, 2018 at 03:00:46PM +0000, Shachar Beiser wrote:
> <snip/>
> > > > --- a/config/common_base
> > > > +++ b/config/common_base
> > > > @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> > > > # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> > > > # CONFIG_RTE_LIBRTE_MLX5_PMD=n
> > > > +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> > > > CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> > > > CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
> > >
> > > Not sure a new configuration item is allowed. If it is, the documentation of
> > > such variable is missing.
> >
> > [S.B] The patch is based on this CONFIG_RTE_LIBRTE_MLX5_DLL , it was
> > required by Adrian in the design phase to enable/disable this linkage
> > mode.
> > I will update the documentation .
>
> Before updating the documentation you should speak with Thomas or
> Ferruh, not sure new items are allowed anymore.
It is OK to add a compilation option to configure the compilation.
Hi,
On Sun, Dec 31, 2017 at 07:52:51AM +0000, Shachar Beiser wrote:
> MLX5 PMD loads libraries: libibverbs and libmlx5.
> MLX5 PMD is not linked to external libraries.
>
> Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
> ---
> v1:
> load external libraries in run-time
> v2:
> * fix checkpatch warnings
> v3:
> * fix checkpatch warnings
> v4:
> New MACROs in order to reuse code
> ---
> config/common_base | 1 +
> drivers/net/mlx5/Makefile | 22 ++-
> drivers/net/mlx5/lib/mlx5_dll.c | 294 +++++++++++++++++++++++++++++++++++++++
> drivers/net/mlx5/lib/mlx5_dll.h | 103 ++++++++++++++
> drivers/net/mlx5/mlx5.c | 17 ++-
> drivers/net/mlx5/mlx5.h | 4 +
> drivers/net/mlx5/mlx5_flow.c | 4 +
> drivers/net/mlx5/mlx5_mac.c | 4 +
> drivers/net/mlx5/mlx5_mr.c | 4 +
> drivers/net/mlx5/mlx5_rss.c | 4 +
> drivers/net/mlx5/mlx5_rxmode.c | 4 +
> drivers/net/mlx5/mlx5_rxq.c | 4 +
> drivers/net/mlx5/mlx5_rxtx.c | 4 +
> drivers/net/mlx5/mlx5_rxtx.h | 6 +-
> drivers/net/mlx5/mlx5_rxtx_vec.c | 4 +
> drivers/net/mlx5/mlx5_txq.c | 4 +
> mk/rte.app.mk | 8 +-
> 17 files changed, 479 insertions(+), 12 deletions(-)
> create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
> create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h
>
> diff --git a/config/common_base b/config/common_base
> index b8ee8f9..30c8fcf 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> #
> CONFIG_RTE_LIBRTE_MLX5_PMD=n
> +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
>
> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> index a3984eb..24fa127 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
> SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
> SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
> SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> -
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
> +endif
> # Basic CFLAGS.
> CFLAGS += -O3
> CFLAGS += -std=c11 -Wall -Wextra
> CFLAGS += -g
> CFLAGS += -I.
> +CFLAGS += -I$(SRCDIR)
> CFLAGS += -D_BSD_SOURCE
> CFLAGS += -D_DEFAULT_SOURCE
> CFLAGS += -D_XOPEN_SOURCE=600
> CFLAGS += $(WERROR_FLAGS)
> CFLAGS += -Wno-strict-prototypes
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +LDLIBS += -ldl
> +else
> LDLIBS += -libverbs -lmlx5
> +endif
> LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
> LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
> LDLIBS += -lrte_bus_pci
> @@ -105,26 +112,28 @@ endif
>
> mlx5_autoconf.h.new: FORCE
>
> +VERBS_H := infiniband/verbs.h
> +MLX5DV_H := infiniband/mlx5dv.h
> mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> $Q $(RM) -f -- '$@'
> $Q sh -- '$<' '$@' \
> HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> - infiniband/verbs.h \
> + $(VERBS_H) \
> enum IBV_DEVICE_VXLAN_SUPPORT \
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> - infiniband/verbs.h \
> + $(VERBS_H) \
> enum IBV_WQ_FLAG_RX_END_PADDING \
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> HAVE_IBV_MLX5_MOD_MPW \
> - infiniband/mlx5dv.h \
> + $(MLX5DV_H) \
> enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> - infiniband/mlx5dv.h \
> + $(MLX5DV_H) \
> enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> @@ -144,10 +153,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> $(AUTOCONF_OUTPUT)
> $Q sh -- '$<' '$@' \
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> - infiniband/verbs.h \
> + $(VERBS_H) \
> enum IBV_FLOW_SPEC_ACTION_COUNT \
> $(AUTOCONF_OUTPUT)
> -
> # Create mlx5_autoconf.h or update it in case it differs from the new one.
>
> mlx5_autoconf.h: mlx5_autoconf.h.new
> diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c
> new file mode 100644
> index 0000000..cafc78a
> --- /dev/null
> +++ b/drivers/net/mlx5/lib/mlx5_dll.c
> @@ -0,0 +1,294 @@
> +/*-
> + * BSD LICENSE
> + *
> + * Copyright 2017 6WIND S.A.
> + * Copyright 2017 Mellanox.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + *
> + * * Redistributions of source code must retain the above copyright
> + * notice, this list of conditions and the following disclaimer.
> + * * Redistributions in binary form must reproduce the above copyright
> + * notice, this list of conditions and the following disclaimer in
> + * the documentation and/or other materials provided with the
> + * distribution.
> + * * Neither the name of 6WIND S.A. nor the names of its
> + * contributors may be used to endorse or promote products derived
> + * from this software without specific prior written permission.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#define _GNU_SOURCE
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <dlfcn.h>
> +
> +#include <rte_log.h>
> +
> +#include "mlx5_utils.h"
> +#include "mlx5_dll.h"
> +
> +MLX5_MLX5DV_FUNC(create_cq, struct ibv_cq_ex *,
> + struct ibv_context *context,
> + struct ibv_cq_init_attr_ex *cq_attr,
> + struct mlx5dv_cq_init_attr *mlx5_cq_attr)
> +{
> + return __MLX5_MLX5DV_FUNC(create_cq, context, cq_attr, mlx5_cq_attr);
> +}
> +
> +MLX5_MLX5DV_FUNC(init_obj, int, struct mlx5dv_obj *obj, uint64_t obj_type)
> +{
> + return __MLX5_MLX5DV_FUNC(init_obj, obj, obj_type);
> +}
> +
> +MLX5_MLX5DV_FUNC(query_device, int, struct ibv_context *ctx_in,
> + struct mlx5dv_context *attrs_out)
> +{
> + return __MLX5_MLX5DV_FUNC(query_device, ctx_in, attrs_out);
> +}
> +
> +MLX5_MLX5DV_FUNC(set_context_attr, int, struct ibv_context *context,
> + enum mlx5dv_set_ctx_attr_type type, void *attr)
> +{
> + return __MLX5_MLX5DV_FUNC(set_context_attr, context, type, attr);
> +}
> +
> +MLX5_IBVERBS_FUNC(destroy_qp, int, struct ibv_qp *qp)
> +{
> + return __MLX5_IBVERBS_FUNC(destroy_qp, qp);
> +}
> +
> +MLX5_IBVERBS_FUNC(destroy_cq, int, struct ibv_cq *cq)
> +{
> + return __MLX5_IBVERBS_FUNC(destroy_cq, cq);
> +}
> +
> +MLX5_IBVERBS_FUNC(dealloc_pd, int, struct ibv_pd *pd)
> +{
> + return __MLX5_IBVERBS_FUNC(dealloc_pd, pd);
> +}
> +
> +MLX5_IBVERBS_FUNC(get_device_list, struct ibv_device **, int *num_devices)
> +{
> + return __MLX5_IBVERBS_FUNC(get_device_list, num_devices);
> +}
> +
> +MLX5_IBVERBS_FUNC(query_device, int, struct ibv_context *context,
> + struct ibv_device_attr *device_attr)
> +{
> + return __MLX5_IBVERBS_FUNC(query_device, context, device_attr);
> +}
> +
> +MLX5_IBVERBS_FUNC(close_device, int, struct ibv_context *context)
> +{
> + return __MLX5_IBVERBS_FUNC(close_device, context);
> +}
> +
> +MLX5_IBVERBS_FUNC(free_device_list, void, struct ibv_device **list)
> +{
> + __MLX5_IBVERBS_FUNC(free_device_list, list);
> +}
> +
> +MLX5_IBVERBS_FUNC(open_device, struct ibv_context *, struct ibv_device *device)
> +{
> + return __MLX5_IBVERBS_FUNC(open_device, device);
> +}
> +
> +MLX5_IBVERBS_FUNC(alloc_pd, struct ibv_pd *, struct ibv_context *context)
> +{
> + return __MLX5_IBVERBS_FUNC(alloc_pd, context);
> +}
> +
> +MLX5_IBVERBS_FUNC(get_device_name, const char *, struct ibv_device *device)
> +{
> + return __MLX5_IBVERBS_FUNC(get_device_name, device);
> +}
> +
> +MLX5_IBVERBS_FUNC(fork_init, int, void)
> +{
> + return __MLX5_IBVERBS_FUNC(fork_init,);
> +}
> +
> +MLX5_IBVERBS_FUNC(destroy_comp_channel, int, struct ibv_comp_channel *channel)
> +{
> + return __MLX5_IBVERBS_FUNC(destroy_comp_channel, channel);
> +}
> +
> +MLX5_IBVERBS_FUNC(dereg_mr, int, struct ibv_mr *mr)
> +{
> + return __MLX5_IBVERBS_FUNC(dereg_mr, mr);
> +}
> +
> +MLX5_IBVERBS_FUNC(create_comp_channel, struct ibv_comp_channel *,
> + struct ibv_context *context)
> +{
> + return __MLX5_IBVERBS_FUNC(create_comp_channel, context);
> +}
> +
> +MLX5_IBVERBS_FUNC(get_cq_event, int, struct ibv_comp_channel *channel,
> + struct ibv_cq **cq, void **cq_context)
> +{
> + return __MLX5_IBVERBS_FUNC(get_cq_event, channel, cq, cq_context);
> +}
> +
> +MLX5_IBVERBS_FUNC(ack_cq_events, void, struct ibv_cq *cq, unsigned int nevents)
> +{
> + __MLX5_IBVERBS_FUNC(ack_cq_events, cq, nevents);
> +}
> +
> +MLX5_IBVERBS_FUNC(create_qp, struct ibv_qp *, struct ibv_pd *pd,
> + struct ibv_qp_init_attr *qp_init_attr)
> +{
> + return __MLX5_IBVERBS_FUNC(create_qp, pd, qp_init_attr);
> +}
> +
> +MLX5_IBVERBS_FUNC(ack_async_event, void, struct ibv_async_event *event)
> +{
> + __MLX5_IBVERBS_FUNC(ack_async_event, event);
> +}
> +
> +MLX5_IBVERBS_FUNC(get_async_event, int, struct ibv_context *context,
> + struct ibv_async_event *event)
> +{
> + return __MLX5_IBVERBS_FUNC(get_async_event, context, event);
> +}
> +
> +MLX5_IBVERBS_FUNC(reg_mr, struct ibv_mr *, struct ibv_pd *pd, void *addr,
> + size_t length, int access)
> +{
> + return __MLX5_IBVERBS_FUNC(reg_mr, pd, addr, length, access);
> +}
> +
> +MLX5_IBVERBS_FUNC(create_cq, struct ibv_cq*, struct ibv_context *context,
> + int cqe, void *cq_context, struct ibv_comp_channel *channel,
> + int comp_vector)
> +{
> + return __MLX5_IBVERBS_FUNC(create_cq, context, cqe, cq_context, channel,
> + comp_vector);
> +}
> +
> +MLX5_IBVERBS_FUNC(modify_qp, int, struct ibv_qp *qp, struct ibv_qp_attr *attr,
> + int attr_mask)
> +{
> + return __MLX5_IBVERBS_FUNC(modify_qp, qp, attr, attr_mask);
> +}
> +
> +MLX5_IBVERBS_FUNC(query_port, int, struct ibv_context *context,
> + uint8_t port_num, struct ibv_port_attr *port_attr)
> +{
> + port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
> + port_attr->reserved = 0;
This was supposed to be just a wrapper around the function pointer.
Why is this extra code needed? Isn't it also needed when using the
function directly?
> + return __MLX5_IBVERBS_FUNC(query_port, context, port_num, port_attr);
> +}
> +
> +MLX5_IBVERBS_FUNC(port_state_str, const char *, enum ibv_port_state port_state)
> +{
> + return __MLX5_IBVERBS_FUNC(port_state_str, port_state);
> +}
> +
> +/**
> + * Register libibverbs APIs.
> + *
> + * @param[in] handle
> + * Library handle.
> + *
> + * @return
> + * 0 on success, -1 on failure.
> + */
> +int mlx5_libverbs_function_register(void *handle)
> +{
> + MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_qp);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_cq);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, dealloc_pd);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, alloc_pd);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_list);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, open_device);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, query_device);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, close_device);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, free_device_list);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_name);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, fork_init);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_comp_channel);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, create_comp_channel);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, dereg_mr);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, get_cq_event);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, create_qp);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, ack_async_event);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, get_async_event);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, reg_mr);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, create_cq);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, query_port);
> + MLX5_DLSYM_IBVERBS_FUNC(handle, port_state_str);
> + return 0;
> +}
> +
> +/**
> + * Register libmlx5 APIs.
> + *
> + * @param[in] handle
> + * Library handle.
> + *
> + * @return
> + * 0 on success, -1 on failure.
> + */
> +int mlx5_lmlx5_function_register(void *handle)
> +{
> + MLX5_DLSYM_MLX5DV_FUNC(handle, create_cq);
> + MLX5_DLSYM_MLX5DV_FUNC(handle, init_obj);
> + MLX5_DLSYM_MLX5DV_FUNC(handle, query_device);
> + MLX5_DLSYM_MLX5DV_FUNC(handle, set_context_attr);
> + return 0;
> +}
> +
> +/**
> + * Load symbols of libibverbs and libmlx5.
> + *
> + * @return
> + * 0 on success, -1 on failure.
> + */
> +int mlx5_load_libs(void)
> +{
> + void *dlhandle;
> + int ret;
> +
> + dlhandle = dlopen(MLX5_LIB_IBVERBS_FILENAME, RTLD_LAZY);
> + if (dlhandle == NULL) {
> + ERROR("failed to open %s: %s\n",
> + MLX5_LIB_IBVERBS_FILENAME, dlerror());
> + return -1;
> + }
> + ret = mlx5_libverbs_function_register(dlhandle);
> + if (ret) {
> + ERROR("failed to register a function in %s",
> + MLX5_LIB_IBVERBS_FILENAME);
> + return -1;
> + }
> + dlhandle = dlopen(MLX5_LIB_MLX5_FILENAME, RTLD_LAZY);
> + if (dlhandle == NULL) {
> + ERROR("failed to open %s: %s\n",
> + MLX5_LIB_MLX5_FILENAME, dlerror());
> + return -1;
> + }
> + ret = mlx5_lmlx5_function_register(dlhandle);
> + if (ret) {
> + ERROR("failed to register a function in %s",
> + MLX5_LIB_MLX5_FILENAME);
> + return -1;
> + }
> + return 0;
> +}
> diff --git a/drivers/net/mlx5/lib/mlx5_dll.h b/drivers/net/mlx5/lib/mlx5_dll.h
> new file mode 100644
> index 0000000..9c3d604
> --- /dev/null
> +++ b/drivers/net/mlx5/lib/mlx5_dll.h
> @@ -0,0 +1,103 @@
> +/*-
> + * BSD LICENSE
> + *
> + * Copyright 2017 6WIND S.A.
> + * Copyright 2017 Mellanox.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + *
> + * * Redistributions of source code must retain the above copyright
> + * notice, this list of conditions and the following disclaimer.
> + * * Redistributions in binary form must reproduce the above copyright
> + * notice, this list of conditions and the following disclaimer in
> + * the documentation and/or other materials provided with the
> + * distribution.
> + * * Neither the name of 6WIND S.A. nor the names of its
> + * contributors may be used to endorse or promote products derived
> + * from this software without specific prior written permission.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef MLX5_DLL_H_
> +#define MLX5_DLL_H_
> +
> +#include <infiniband/verbs.h>
> +#include <infiniband/mlx5dv.h>
> +
> +#define MLX5_LIB_IBVERBS_FILENAME "libibverbs.so"
> +#define MLX5_LIB_MLX5_FILENAME "libmlx5.so"
These are only packaged in -devel packages. You probably want to
specify the major version here. As in, libmlx5.so.1.
> +
> +#ifndef RTE_LIBRTE_MLX4_PMD
This may be project specific, thus I'm sorry if I'm violating some
code style policy but most of the macros below don't need/shouldn't be
here in the header as they are only used by the dll wrapper and
shouldn't be used outside of it.
Perhaps the only thing really needed here is the
+int mlx5_load_libs(void);
declaration and the includes above.
> +
> +#define MLX5_IBVERBS_FUNC(name, ret, ...) \
> + ret(*__mlx5_pmd_ibv_##name)(__VA_ARGS__); \
> + ret(ibv_##name)(__VA_ARGS__)
> +
> +#define __MLX5_IBVERBS_FUNC(name, ...) \
> + (*__mlx5_pmd_ibv_##name)(__VA_ARGS__)
> +
> +#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) \
> + do { \
> + char *err; \
> + *(void **)(&__mlx5_pmd_ibv_##name) = \
> + dlsym(handle, "ibv_"#name); \
> + err = dlerror(); \
> + if (err != NULL) { \
> + ERROR("failed to link a symbol: %s", err); \
> + return -1; \
> + } \
> + } while (0)
> +
> +#else /* RTE_LIBRTE_MLX4_PMD */
> +
> +#define MLX5_IBVERBS_FUNC(name, ret, ...) \
> + ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__); \
> + ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__)
> +
> +#define __MLX5_IBVERBS_FUNC(name, ...) \
> + (ibv_##name)(__VA_ARGS__)
> +
> +#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) do { (void)handle; } while (0)
> +
> +#endif /* RTE_LIBRTE_MLX4_PMD */
> +
> +#define MLX5_MLX5DV_FUNC(name, ret, ...) \
> + ret(*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__); \
> + ret(mlx5dv_##name)(__VA_ARGS__)
> +
> +#define __MLX5_MLX5DV_FUNC(name, ...) \
> + (*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__)
> +
> +#define MLX5_DLSYM_MLX5DV_FUNC(handle, name) \
> + do { \
> + char *err; \
> + *(void **)(&__mlx5_pmd_mlx5dv_##name) = \
> + dlsym(handle, "mlx5dv_"#name); \
> + err = dlerror(); \
> + if (err != NULL) { \
> + ERROR("failed to link a symbol: %s", err); \
> + return -1; \
> + } \
> + } while (0)
> +
> +#undef ibv_query_port
> +
> +void *mlx5_lib_load(const char *name);
> +int mlx5_libverbs_function_register(void *handle);
> +int mlx5_lmlx5_function_register(void *handle);
> +int mlx5_load_libs(void);
> +
> +#endif /* RTE_PMD_MLX5_DLL_H_ */
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index cd66fe1..eeef782 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -30,7 +30,8 @@
> * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> */
> -
> +#define _GNU_SOURCE
> +#include <stdio.h>
this and
> #include <stddef.h>
> #include <unistd.h>
> #include <string.h>
> @@ -39,13 +40,17 @@
> #include <stdlib.h>
> #include <errno.h>
> #include <net/if.h>
> -
> +#include <dlfcn.h>
this change shouldn't be needed in this file, as all it does is call
an initialization function below.
> /* Verbs header. */
> /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> @@ -1027,6 +1032,9 @@ struct mlx5_args {
> rte_mlx5_pmd_init(void)
> {
> /* Build the static table for ptype conversion. */
> +#ifdef RTE_LIBRTE_MLX5_DLL
> + int ret;
> +#endif
> mlx5_set_ptype_table();
> /*
> * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
> @@ -1038,6 +1046,11 @@ struct mlx5_args {
> /* Match the size of Rx completion entry to the size of a cacheline. */
> if (RTE_CACHE_LINE_SIZE == 128)
> setenv("MLX5_CQE_SIZE", "128", 0);
> +#ifdef RTE_LIBRTE_MLX5_DLL
> + ret = mlx5_load_libs();
> + if (ret != 0)
> + return;
> +#endif
Also, depends on project code style, but defining an empty
mlx5_load_libs() in case RTE_LIBRTE_MLX5_DLL is NOT defined would
avoid the two ifdef above.
Marcelo
> ibv_fork_init();
> rte_pci_register(&mlx5_driver);
> }
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
> index e6a69b8..9db7024 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -46,7 +46,11 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 6605cfd..49f8972 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -39,7 +39,11 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
> index 9fb5ba5..c9f4bfd 100644
> --- a/drivers/net/mlx5/mlx5_mac.c
> +++ b/drivers/net/mlx5/mlx5_mac.c
> @@ -46,7 +46,11 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> index 6b29eed..f27a428 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -36,7 +36,11 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
> index f491135..63cbfa0 100644
> --- a/drivers/net/mlx5/mlx5_rss.c
> +++ b/drivers/net/mlx5/mlx5_rss.c
> @@ -42,7 +42,11 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c
> index 6fb245b..1ddad70 100644
> --- a/drivers/net/mlx5/mlx5_rxmode.c
> +++ b/drivers/net/mlx5/mlx5_rxmode.c
> @@ -40,7 +40,11 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index a4cdd37..aaf4ce1 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -44,8 +44,12 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> #include <infiniband/mlx5dv.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
> index 67e3db1..4faf302 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.c
> +++ b/drivers/net/mlx5/mlx5_rxtx.c
> @@ -41,8 +41,12 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> #include <infiniband/mlx5dv.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index e70d523..e637555 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -43,12 +43,14 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> -#include <infiniband/mlx5dv.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> -
> #include <rte_mbuf.h>
> #include <rte_mempool.h>
> #include <rte_common.h>
> diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
> index 8d23dae..679bee7 100644
> --- a/drivers/net/mlx5/mlx5_rxtx_vec.c
> +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
> @@ -41,8 +41,12 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> #include <infiniband/mlx5dv.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
> index 89b16fd..8b0cdf4 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -44,7 +44,11 @@
> #ifdef PEDANTIC
> #pragma GCC diagnostic ignored "-Wpedantic"
> #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
> #include <infiniband/verbs.h>
> +#endif
> #ifdef PEDANTIC
> #pragma GCC diagnostic error "-Wpedantic"
> #endif
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk
> index 6a6a745..2dd2f6b 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -51,7 +51,9 @@ endif
>
> # Link only the libraries used in the application
> LDFLAGS += --as-needed
> -
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +LDFLAGS += -ldl
> +endif
> # default path for libs
> _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
>
> @@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI) += -lrte_pmd_kni
> endif
> _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += -lrte_pmd_lio
> _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs -lmlx4
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -ldl
> +else
> _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs -lmlx5
> +endif
> _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
> _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp
> _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += -lrte_pmd_null
> --
> 1.8.3.1
>
Hello,
On Thu, Jan 04, 2018 at 08:36:08AM +0100, Nélio Laranjeiro wrote:
> Hi Shachar,
>
> On Wed, Jan 03, 2018 at 03:00:46PM +0000, Shachar Beiser wrote:
> <snip/>
> > > > --- a/config/common_base
> > > > +++ b/config/common_base
> > > > @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> > > > # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> > > > # CONFIG_RTE_LIBRTE_MLX5_PMD=n
> > > > +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> > > > CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> > > > CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
> > >
> > > Not sure a new configuration item is allowed. If it is, the documentation of
> > > such variable is missing.
> >
> > [S.B] The patch is based on this CONFIG_RTE_LIBRTE_MLX5_DLL , it was
> > required by Adrian in the design phase to enable/disable this linkage
> > mode.
> > I will update the documentation .
>
> Before updating the documentation you should speak with Thomas or
> Ferruh, not sure new items are allowed anymore.
>
> > > > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> > > > index a3984eb..24fa127 100644
> > > > --- a/drivers/net/mlx5/Makefile
> > > > +++ b/drivers/net/mlx5/Makefile
> > > > @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) +=
> > > mlx5_rss.c
> > > > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
> > > > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
> > > > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> > > > -
> > > > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> > > > +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c endif
> > > > # Basic CFLAGS.
> > > > CFLAGS += -O3
> > > > CFLAGS += -std=c11 -Wall -Wextra
> > > > CFLAGS += -g
> > > > CFLAGS += -I.
> > > > +CFLAGS += -I$(SRCDIR)
> > > > CFLAGS += -D_BSD_SOURCE
> > > > CFLAGS += -D_DEFAULT_SOURCE
> > > > CFLAGS += -D_XOPEN_SOURCE=600
> > > > CFLAGS += $(WERROR_FLAGS)
> > > > CFLAGS += -Wno-strict-prototypes
> > > > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) LDLIBS += -ldl else
> > > > LDLIBS += -libverbs -lmlx5
> > > > +endif
> > > > LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS +=
> > > > -lrte_ethdev -lrte_net -lrte_kvargs LDLIBS += -lrte_bus_pci @@
> > > > -105,26 +112,28 @@ endif
> > > >
> > > > mlx5_autoconf.h.new: FORCE
> > > >
> > > > +VERBS_H := infiniband/verbs.h
> > > > +MLX5DV_H := infiniband/mlx5dv.h
> > > > mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> > > > $Q $(RM) -f -- '$@'
> > > > $Q sh -- '$<' '$@' \
> > > > HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> > > > - infiniband/verbs.h \
> > > > + $(VERBS_H) \
> > > > enum IBV_DEVICE_VXLAN_SUPPORT \
> > > > $(AUTOCONF_OUTPUT)
> > > > $Q sh -- '$<' '$@' \
> > > > HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> > > > - infiniband/verbs.h \
> > > > + $(VERBS_H) \
> > > > enum IBV_WQ_FLAG_RX_END_PADDING \
> > > > $(AUTOCONF_OUTPUT)
> > > > $Q sh -- '$<' '$@' \
> > > > HAVE_IBV_MLX5_MOD_MPW \
> > > > - infiniband/mlx5dv.h \
> > > > + $(MLX5DV_H) \
> > > > enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
> > > > $(AUTOCONF_OUTPUT)
> > > > $Q sh -- '$<' '$@' \
> > > > HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> > > > - infiniband/mlx5dv.h \
> > > > + $(MLX5DV_H) \
> > > > enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
> > > > $(AUTOCONF_OUTPUT)
> > > > $Q sh -- '$<' '$@' \
> > > > @@ -144,10 +153,9 @@ mlx5_autoconf.h.new:
> > > $(RTE_SDK)/buildtools/auto-config-h.sh
> > > > $(AUTOCONF_OUTPUT)
> > > > $Q sh -- '$<' '$@' \
> > > > HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> > > > - infiniband/verbs.h \
> > > > + $(VERBS_H) \
> > > > enum IBV_FLOW_SPEC_ACTION_COUNT \
> > > > $(AUTOCONF_OUTPUT)
> > >
> > > This modification should be inside its own patch, it is not directly related to
> > > the this patch itself.
> > [S.B] I can revert the VERBS_H , MLX5DV_H and not change it all .
> > There is no need to change in a different patch.
>
> As you wish.
>
> > > > -
> > > > # Create mlx5_autoconf.h or update it in case it differs from the new one.
> > > >
> > > > mlx5_autoconf.h: mlx5_autoconf.h.new
> > > <snip/>
> > > > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > > > cd66fe1..eeef782 100644
> > > > --- a/drivers/net/mlx5/mlx5.c
> > > > +++ b/drivers/net/mlx5/mlx5.c
> > > > @@ -30,7 +30,8 @@
> > > > * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
> > > OF THE USE
> > > > * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> > > DAMAGE.
> > > > */
> > > > -
> > > > +#define _GNU_SOURCE
> > > > +#include <stdio.h>
> > > > #include <stddef.h>
> > > > #include <unistd.h>
> > > > #include <string.h>
> > > > @@ -39,13 +40,17 @@
> > > > #include <stdlib.h>
> > > > #include <errno.h>
> > > > #include <net/if.h>
> > > > -
> > > > +#include <dlfcn.h>
> > >
> > > The empty line should remain.
> > [S.B] OK.
>
> To be sure, the empty line should be between the system include and
> verbs ones.
>
> > >
> > > > /* Verbs header. */
> > > > /* ISO C doesn't support unnamed structs/unions, disabling -pedantic.
> > > > */ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic"
> > > > #endif
> > > > +#ifdef RTE_LIBRTE_MLX5_DLL
> > > > +#include "lib/mlx5_dll.h"
> > > > +#else
> > > > #include <infiniband/verbs.h>
> > > > +#endif
> > >
> > > This could be done by the mlx5_dll.h file which could include the correct
> > > header according to the configuration.
> > [S.B] I guess you refer to all *.c files that includes the verbs.h .
> > I will change them all ?
>
> Yes
Actually, why this change? It shouldn't be necessary. As the patch is
defining functions with the very same prototype, it shouldn't matter
if their declarations are coming from one header or another.
Marcelo
@@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
# Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
#
CONFIG_RTE_LIBRTE_MLX5_PMD=n
+CONFIG_RTE_LIBRTE_MLX5_DLL=y
CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
@@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
+endif
# Basic CFLAGS.
CFLAGS += -O3
CFLAGS += -std=c11 -Wall -Wextra
CFLAGS += -g
CFLAGS += -I.
+CFLAGS += -I$(SRCDIR)
CFLAGS += -D_BSD_SOURCE
CFLAGS += -D_DEFAULT_SOURCE
CFLAGS += -D_XOPEN_SOURCE=600
CFLAGS += $(WERROR_FLAGS)
CFLAGS += -Wno-strict-prototypes
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDLIBS += -ldl
+else
LDLIBS += -libverbs -lmlx5
+endif
LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
LDLIBS += -lrte_bus_pci
@@ -105,26 +112,28 @@ endif
mlx5_autoconf.h.new: FORCE
+VERBS_H := infiniband/verbs.h
+MLX5DV_H := infiniband/mlx5dv.h
mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
$Q $(RM) -f -- '$@'
$Q sh -- '$<' '$@' \
HAVE_IBV_DEVICE_VXLAN_SUPPORT \
- infiniband/verbs.h \
+ $(VERBS_H) \
enum IBV_DEVICE_VXLAN_SUPPORT \
$(AUTOCONF_OUTPUT)
$Q sh -- '$<' '$@' \
HAVE_IBV_WQ_FLAG_RX_END_PADDING \
- infiniband/verbs.h \
+ $(VERBS_H) \
enum IBV_WQ_FLAG_RX_END_PADDING \
$(AUTOCONF_OUTPUT)
$Q sh -- '$<' '$@' \
HAVE_IBV_MLX5_MOD_MPW \
- infiniband/mlx5dv.h \
+ $(MLX5DV_H) \
enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
$(AUTOCONF_OUTPUT)
$Q sh -- '$<' '$@' \
HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
- infiniband/mlx5dv.h \
+ $(MLX5DV_H) \
enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
$(AUTOCONF_OUTPUT)
$Q sh -- '$<' '$@' \
@@ -144,10 +153,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
$(AUTOCONF_OUTPUT)
$Q sh -- '$<' '$@' \
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
- infiniband/verbs.h \
+ $(VERBS_H) \
enum IBV_FLOW_SPEC_ACTION_COUNT \
$(AUTOCONF_OUTPUT)
-
# Create mlx5_autoconf.h or update it in case it differs from the new one.
mlx5_autoconf.h: mlx5_autoconf.h.new
new file mode 100644
@@ -0,0 +1,294 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright 2017 6WIND S.A.
+ * Copyright 2017 Mellanox.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of 6WIND S.A. nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+
+#include <rte_log.h>
+
+#include "mlx5_utils.h"
+#include "mlx5_dll.h"
+
+MLX5_MLX5DV_FUNC(create_cq, struct ibv_cq_ex *,
+ struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *cq_attr,
+ struct mlx5dv_cq_init_attr *mlx5_cq_attr)
+{
+ return __MLX5_MLX5DV_FUNC(create_cq, context, cq_attr, mlx5_cq_attr);
+}
+
+MLX5_MLX5DV_FUNC(init_obj, int, struct mlx5dv_obj *obj, uint64_t obj_type)
+{
+ return __MLX5_MLX5DV_FUNC(init_obj, obj, obj_type);
+}
+
+MLX5_MLX5DV_FUNC(query_device, int, struct ibv_context *ctx_in,
+ struct mlx5dv_context *attrs_out)
+{
+ return __MLX5_MLX5DV_FUNC(query_device, ctx_in, attrs_out);
+}
+
+MLX5_MLX5DV_FUNC(set_context_attr, int, struct ibv_context *context,
+ enum mlx5dv_set_ctx_attr_type type, void *attr)
+{
+ return __MLX5_MLX5DV_FUNC(set_context_attr, context, type, attr);
+}
+
+MLX5_IBVERBS_FUNC(destroy_qp, int, struct ibv_qp *qp)
+{
+ return __MLX5_IBVERBS_FUNC(destroy_qp, qp);
+}
+
+MLX5_IBVERBS_FUNC(destroy_cq, int, struct ibv_cq *cq)
+{
+ return __MLX5_IBVERBS_FUNC(destroy_cq, cq);
+}
+
+MLX5_IBVERBS_FUNC(dealloc_pd, int, struct ibv_pd *pd)
+{
+ return __MLX5_IBVERBS_FUNC(dealloc_pd, pd);
+}
+
+MLX5_IBVERBS_FUNC(get_device_list, struct ibv_device **, int *num_devices)
+{
+ return __MLX5_IBVERBS_FUNC(get_device_list, num_devices);
+}
+
+MLX5_IBVERBS_FUNC(query_device, int, struct ibv_context *context,
+ struct ibv_device_attr *device_attr)
+{
+ return __MLX5_IBVERBS_FUNC(query_device, context, device_attr);
+}
+
+MLX5_IBVERBS_FUNC(close_device, int, struct ibv_context *context)
+{
+ return __MLX5_IBVERBS_FUNC(close_device, context);
+}
+
+MLX5_IBVERBS_FUNC(free_device_list, void, struct ibv_device **list)
+{
+ __MLX5_IBVERBS_FUNC(free_device_list, list);
+}
+
+MLX5_IBVERBS_FUNC(open_device, struct ibv_context *, struct ibv_device *device)
+{
+ return __MLX5_IBVERBS_FUNC(open_device, device);
+}
+
+MLX5_IBVERBS_FUNC(alloc_pd, struct ibv_pd *, struct ibv_context *context)
+{
+ return __MLX5_IBVERBS_FUNC(alloc_pd, context);
+}
+
+MLX5_IBVERBS_FUNC(get_device_name, const char *, struct ibv_device *device)
+{
+ return __MLX5_IBVERBS_FUNC(get_device_name, device);
+}
+
+MLX5_IBVERBS_FUNC(fork_init, int, void)
+{
+ return __MLX5_IBVERBS_FUNC(fork_init,);
+}
+
+MLX5_IBVERBS_FUNC(destroy_comp_channel, int, struct ibv_comp_channel *channel)
+{
+ return __MLX5_IBVERBS_FUNC(destroy_comp_channel, channel);
+}
+
+MLX5_IBVERBS_FUNC(dereg_mr, int, struct ibv_mr *mr)
+{
+ return __MLX5_IBVERBS_FUNC(dereg_mr, mr);
+}
+
+MLX5_IBVERBS_FUNC(create_comp_channel, struct ibv_comp_channel *,
+ struct ibv_context *context)
+{
+ return __MLX5_IBVERBS_FUNC(create_comp_channel, context);
+}
+
+MLX5_IBVERBS_FUNC(get_cq_event, int, struct ibv_comp_channel *channel,
+ struct ibv_cq **cq, void **cq_context)
+{
+ return __MLX5_IBVERBS_FUNC(get_cq_event, channel, cq, cq_context);
+}
+
+MLX5_IBVERBS_FUNC(ack_cq_events, void, struct ibv_cq *cq, unsigned int nevents)
+{
+ __MLX5_IBVERBS_FUNC(ack_cq_events, cq, nevents);
+}
+
+MLX5_IBVERBS_FUNC(create_qp, struct ibv_qp *, struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr)
+{
+ return __MLX5_IBVERBS_FUNC(create_qp, pd, qp_init_attr);
+}
+
+MLX5_IBVERBS_FUNC(ack_async_event, void, struct ibv_async_event *event)
+{
+ __MLX5_IBVERBS_FUNC(ack_async_event, event);
+}
+
+MLX5_IBVERBS_FUNC(get_async_event, int, struct ibv_context *context,
+ struct ibv_async_event *event)
+{
+ return __MLX5_IBVERBS_FUNC(get_async_event, context, event);
+}
+
+MLX5_IBVERBS_FUNC(reg_mr, struct ibv_mr *, struct ibv_pd *pd, void *addr,
+ size_t length, int access)
+{
+ return __MLX5_IBVERBS_FUNC(reg_mr, pd, addr, length, access);
+}
+
+MLX5_IBVERBS_FUNC(create_cq, struct ibv_cq*, struct ibv_context *context,
+ int cqe, void *cq_context, struct ibv_comp_channel *channel,
+ int comp_vector)
+{
+ return __MLX5_IBVERBS_FUNC(create_cq, context, cqe, cq_context, channel,
+ comp_vector);
+}
+
+MLX5_IBVERBS_FUNC(modify_qp, int, struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ int attr_mask)
+{
+ return __MLX5_IBVERBS_FUNC(modify_qp, qp, attr, attr_mask);
+}
+
+MLX5_IBVERBS_FUNC(query_port, int, struct ibv_context *context,
+ uint8_t port_num, struct ibv_port_attr *port_attr)
+{
+ port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+ port_attr->reserved = 0;
+ return __MLX5_IBVERBS_FUNC(query_port, context, port_num, port_attr);
+}
+
+MLX5_IBVERBS_FUNC(port_state_str, const char *, enum ibv_port_state port_state)
+{
+ return __MLX5_IBVERBS_FUNC(port_state_str, port_state);
+}
+
+/**
+ * Register libibverbs APIs.
+ *
+ * @param[in] handle
+ * Library handle.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+int mlx5_libverbs_function_register(void *handle)
+{
+ MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_qp);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_cq);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, dealloc_pd);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, alloc_pd);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_list);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, open_device);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, query_device);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, close_device);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, free_device_list);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_name);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, fork_init);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_comp_channel);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, create_comp_channel);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, dereg_mr);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, get_cq_event);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, create_qp);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, ack_async_event);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, get_async_event);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, reg_mr);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, create_cq);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, query_port);
+ MLX5_DLSYM_IBVERBS_FUNC(handle, port_state_str);
+ return 0;
+}
+
+/**
+ * Register libmlx5 APIs.
+ *
+ * @param[in] handle
+ * Library handle.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+int mlx5_lmlx5_function_register(void *handle)
+{
+ MLX5_DLSYM_MLX5DV_FUNC(handle, create_cq);
+ MLX5_DLSYM_MLX5DV_FUNC(handle, init_obj);
+ MLX5_DLSYM_MLX5DV_FUNC(handle, query_device);
+ MLX5_DLSYM_MLX5DV_FUNC(handle, set_context_attr);
+ return 0;
+}
+
+/**
+ * Load symbols of libibverbs and libmlx5.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+int mlx5_load_libs(void)
+{
+ void *dlhandle;
+ int ret;
+
+ dlhandle = dlopen(MLX5_LIB_IBVERBS_FILENAME, RTLD_LAZY);
+ if (dlhandle == NULL) {
+ ERROR("failed to open %s: %s\n",
+ MLX5_LIB_IBVERBS_FILENAME, dlerror());
+ return -1;
+ }
+ ret = mlx5_libverbs_function_register(dlhandle);
+ if (ret) {
+ ERROR("failed to register a function in %s",
+ MLX5_LIB_IBVERBS_FILENAME);
+ return -1;
+ }
+ dlhandle = dlopen(MLX5_LIB_MLX5_FILENAME, RTLD_LAZY);
+ if (dlhandle == NULL) {
+ ERROR("failed to open %s: %s\n",
+ MLX5_LIB_MLX5_FILENAME, dlerror());
+ return -1;
+ }
+ ret = mlx5_lmlx5_function_register(dlhandle);
+ if (ret) {
+ ERROR("failed to register a function in %s",
+ MLX5_LIB_MLX5_FILENAME);
+ return -1;
+ }
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,103 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright 2017 6WIND S.A.
+ * Copyright 2017 Mellanox.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of 6WIND S.A. nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MLX5_DLL_H_
+#define MLX5_DLL_H_
+
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+
+#define MLX5_LIB_IBVERBS_FILENAME "libibverbs.so"
+#define MLX5_LIB_MLX5_FILENAME "libmlx5.so"
+
+#ifndef RTE_LIBRTE_MLX4_PMD
+
+#define MLX5_IBVERBS_FUNC(name, ret, ...) \
+ ret(*__mlx5_pmd_ibv_##name)(__VA_ARGS__); \
+ ret(ibv_##name)(__VA_ARGS__)
+
+#define __MLX5_IBVERBS_FUNC(name, ...) \
+ (*__mlx5_pmd_ibv_##name)(__VA_ARGS__)
+
+#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) \
+ do { \
+ char *err; \
+ *(void **)(&__mlx5_pmd_ibv_##name) = \
+ dlsym(handle, "ibv_"#name); \
+ err = dlerror(); \
+ if (err != NULL) { \
+ ERROR("failed to link a symbol: %s", err); \
+ return -1; \
+ } \
+ } while (0)
+
+#else /* RTE_LIBRTE_MLX4_PMD */
+
+#define MLX5_IBVERBS_FUNC(name, ret, ...) \
+ ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__); \
+ ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__)
+
+#define __MLX5_IBVERBS_FUNC(name, ...) \
+ (ibv_##name)(__VA_ARGS__)
+
+#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) do { (void)handle; } while (0)
+
+#endif /* RTE_LIBRTE_MLX4_PMD */
+
+#define MLX5_MLX5DV_FUNC(name, ret, ...) \
+ ret(*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__); \
+ ret(mlx5dv_##name)(__VA_ARGS__)
+
+#define __MLX5_MLX5DV_FUNC(name, ...) \
+ (*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__)
+
+#define MLX5_DLSYM_MLX5DV_FUNC(handle, name) \
+ do { \
+ char *err; \
+ *(void **)(&__mlx5_pmd_mlx5dv_##name) = \
+ dlsym(handle, "mlx5dv_"#name); \
+ err = dlerror(); \
+ if (err != NULL) { \
+ ERROR("failed to link a symbol: %s", err); \
+ return -1; \
+ } \
+ } while (0)
+
+#undef ibv_query_port
+
+void *mlx5_lib_load(const char *name);
+int mlx5_libverbs_function_register(void *handle);
+int mlx5_lmlx5_function_register(void *handle);
+int mlx5_load_libs(void);
+
+#endif /* RTE_PMD_MLX5_DLL_H_ */
@@ -30,7 +30,8 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-
+#define _GNU_SOURCE
+#include <stdio.h>
#include <stddef.h>
#include <unistd.h>
#include <string.h>
@@ -39,13 +40,17 @@
#include <stdlib.h>
#include <errno.h>
#include <net/if.h>
-
+#include <dlfcn.h>
/* Verbs header. */
/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -1027,6 +1032,9 @@ struct mlx5_args {
rte_mlx5_pmd_init(void)
{
/* Build the static table for ptype conversion. */
+#ifdef RTE_LIBRTE_MLX5_DLL
+ int ret;
+#endif
mlx5_set_ptype_table();
/*
* RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
@@ -1038,6 +1046,11 @@ struct mlx5_args {
/* Match the size of Rx completion entry to the size of a cacheline. */
if (RTE_CACHE_LINE_SIZE == 128)
setenv("MLX5_CQE_SIZE", "128", 0);
+#ifdef RTE_LIBRTE_MLX5_DLL
+ ret = mlx5_load_libs();
+ if (ret != 0)
+ return;
+#endif
ibv_fork_init();
rte_pci_register(&mlx5_driver);
}
@@ -46,7 +46,11 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -39,7 +39,11 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -46,7 +46,11 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -36,7 +36,11 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -42,7 +42,11 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -40,7 +40,11 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -44,8 +44,12 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
#include <infiniband/mlx5dv.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -41,8 +41,12 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
#include <infiniband/mlx5dv.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -43,12 +43,14 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
-#include <infiniband/mlx5dv.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
-
#include <rte_mbuf.h>
#include <rte_mempool.h>
#include <rte_common.h>
@@ -41,8 +41,12 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
#include <infiniband/mlx5dv.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -44,7 +44,11 @@
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
#include <infiniband/verbs.h>
+#endif
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -51,7 +51,9 @@ endif
# Link only the libraries used in the application
LDFLAGS += --as-needed
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDFLAGS += -ldl
+endif
# default path for libs
_LDLIBS-y += -L$(RTE_SDK_BIN)/lib
@@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI) += -lrte_pmd_kni
endif
_LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += -lrte_pmd_lio
_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs -lmlx4
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -ldl
+else
_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs -lmlx5
+endif
_LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
_LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp
_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += -lrte_pmd_null