[v12,2/2] eal: support for VFIO-PCI VF token
Checks
Commit Message
The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++-
doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++
doc/guides/rel_notes/release_20_05.rst | 6 ++++
lib/librte_eal/common/eal_common_options.c | 2 ++
lib/librte_eal/common/eal_internal_cfg.h | 2 ++
lib/librte_eal/common/eal_options.h | 2 ++
lib/librte_eal/freebsd/eal.c | 5 +++
lib/librte_eal/include/rte_eal.h | 12 +++++++
lib/librte_eal/linux/eal.c | 29 +++++++++++++++
lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++
lib/librte_eal/rte_eal_version.map | 1 +
11 files changed, 116 insertions(+), 1 deletion(-)
Comments
On 05-May-20 11:34 AM, Haiyue Wang wrote:
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support since 5.7.
>
> The VF token can be set by a vfio-pci based PF driver and must be known
> by the vfio-pci based VF driver in order to gain access to the device.
>
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> ---
<snip>
> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
> index 540b7d38c..00b02855e 100644
> --- a/lib/librte_eal/freebsd/eal.c
> +++ b/lib/librte_eal/freebsd/eal.c
> @@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
> return RTE_INTR_MODE_NONE;
> }
>
> +void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
> +{
> + memset(vf_token, 0, sizeof(rte_uuid_t));
> +}
What's the purpose of memset(0) here? Presumably, if the API is not
supposed to be supported, the function should have no effect?
> +
> int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
> __rte_unused const char *dev_addr,
> __rte_unused int *vfio_dev_fd,
> diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
> index 2f9ed298d..d9040ce14 100644
> --- a/lib/librte_eal/include/rte_eal.h
> +++ b/lib/librte_eal/include/rte_eal.h
> @@ -21,6 +21,7 @@
> #include <rte_bus.h>
>
> #include <rte_pci_dev_feature_defs.h>
> +#include <rte_uuid.h>
>
> #ifdef __cplusplus
> extern "C" {
> @@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
> */
> enum rte_intr_mode rte_eal_vfio_intr_mode(void);
>
> +
> +/**
> + * The user-configured vfio VF token.
The description is unclear. Suggested rewording:
Copy the user-configured vfio VF token.
> + *
> + * @param vf_token
> + * vfio VF token configured with the command line is copied
> + * into this parameter, zero uuid by default.
> + */
> +__rte_experimental
> +void rte_eal_vfio_vf_token(rte_uuid_t vf_token);
Maybe rte_eal_vfio_get_vf_token()? I imagine this would be a more
descriptive name for this API.
Once all of the above is addressed (or explained),
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Hi Anatoly,
> -----Original Message-----
> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> Sent: Wednesday, May 6, 2020 18:27
> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; thomas@monjalon.net; jerinj@marvell.com;
> david.marchand@redhat.com
> Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token
>
> On 05-May-20 11:34 AM, Haiyue Wang wrote:
> > The kernel module vfio-pci introduces the VF token to enable SR-IOV
> > support since 5.7.
> >
> > The VF token can be set by a vfio-pci based PF driver and must be known
> > by the vfio-pci based VF driver in order to gain access to the device.
> >
> > Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> > ---
>
> <snip>
>
> > diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
> > index 540b7d38c..00b02855e 100644
> > --- a/lib/librte_eal/freebsd/eal.c
> > +++ b/lib/librte_eal/freebsd/eal.c
> > @@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
> > return RTE_INTR_MODE_NONE;
> > }
> >
> > +void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
> > +{
> > + memset(vf_token, 0, sizeof(rte_uuid_t));
> > +}
>
> What's the purpose of memset(0) here? Presumably, if the API is not
> supposed to be supported, the function should have no effect?
>
Yes, originally I meant to no parse the input, then return zero UUID.
I changed it to the same as Linux:
rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
Since it is inited to zero.
> > +
> > int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
> > __rte_unused const char *dev_addr,
> > __rte_unused int *vfio_dev_fd,
> > diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
> > index 2f9ed298d..d9040ce14 100644
> > --- a/lib/librte_eal/include/rte_eal.h
> > +++ b/lib/librte_eal/include/rte_eal.h
> > @@ -21,6 +21,7 @@
> > #include <rte_bus.h>
> >
> > #include <rte_pci_dev_feature_defs.h>
> > +#include <rte_uuid.h>
> >
> > #ifdef __cplusplus
> > extern "C" {
> > @@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
> > */
> > enum rte_intr_mode rte_eal_vfio_intr_mode(void);
> >
> > +
> > +/**
> > + * The user-configured vfio VF token.
>
> The description is unclear. Suggested rewording:
>
> Copy the user-configured vfio VF token.
>
+1.
> > + *
> > + * @param vf_token
> > + * vfio VF token configured with the command line is copied
> > + * into this parameter, zero uuid by default.
> > + */
> > +__rte_experimental
> > +void rte_eal_vfio_vf_token(rte_uuid_t vf_token);
>
> Maybe rte_eal_vfio_get_vf_token()? I imagine this would be a more
> descriptive name for this API.
>
Yes, this looks better. Fixed.
> Once all of the above is addressed (or explained),
>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
>
> --
> Thanks,
> Anatoly
On 06-May-20 12:35 PM, Wang, Haiyue wrote:
> Hi Anatoly,
>
>> -----Original Message-----
>> From: Burakov, Anatoly <anatoly.burakov@intel.com>
>> Sent: Wednesday, May 6, 2020 18:27
>> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; thomas@monjalon.net; jerinj@marvell.com;
>> david.marchand@redhat.com
>> Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token
>>
>> On 05-May-20 11:34 AM, Haiyue Wang wrote:
>>> The kernel module vfio-pci introduces the VF token to enable SR-IOV
>>> support since 5.7.
>>>
>>> The VF token can be set by a vfio-pci based PF driver and must be known
>>> by the vfio-pci based VF driver in order to gain access to the device.
>>>
>>> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
>>> ---
>>
>> <snip>
>>
>>> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
>>> index 540b7d38c..00b02855e 100644
>>> --- a/lib/librte_eal/freebsd/eal.c
>>> +++ b/lib/librte_eal/freebsd/eal.c
>>> @@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
>>> return RTE_INTR_MODE_NONE;
>>> }
>>>
>>> +void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
>>> +{
>>> + memset(vf_token, 0, sizeof(rte_uuid_t));
>>> +}
>>
>> What's the purpose of memset(0) here? Presumably, if the API is not
>> supposed to be supported, the function should have no effect?
>>
>
> Yes, originally I meant to no parse the input, then return zero UUID.
> I changed it to the same as Linux:
> rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
> Since it is inited to zero.
Yes but on Linux it has effect because Linux supports it. On FreeBSD
there's no VFIO, so the function should IMO have no effect. However, i
don't have a strong opinion on this, so i'm fine either way.
> -----Original Message-----
> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> Sent: Wednesday, May 6, 2020 19:40
> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; thomas@monjalon.net; jerinj@marvell.com;
> david.marchand@redhat.com
> Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token
>
> On 06-May-20 12:35 PM, Wang, Haiyue wrote:
> > Hi Anatoly,
> >
> >> -----Original Message-----
> >> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> >> Sent: Wednesday, May 6, 2020 18:27
> >> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; thomas@monjalon.net; jerinj@marvell.com;
> >> david.marchand@redhat.com
> >> Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token
> >>
> >> On 05-May-20 11:34 AM, Haiyue Wang wrote:
> >>> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> >>> support since 5.7.
> >>>
> >>> The VF token can be set by a vfio-pci based PF driver and must be known
> >>> by the vfio-pci based VF driver in order to gain access to the device.
> >>>
> >>> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> >>> ---
> >>
> >> <snip>
> >>
> >>> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
> >>> index 540b7d38c..00b02855e 100644
> >>> --- a/lib/librte_eal/freebsd/eal.c
> >>> +++ b/lib/librte_eal/freebsd/eal.c
> >>> @@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
> >>> return RTE_INTR_MODE_NONE;
> >>> }
> >>>
> >>> +void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
> >>> +{
> >>> + memset(vf_token, 0, sizeof(rte_uuid_t));
> >>> +}
> >>
> >> What's the purpose of memset(0) here? Presumably, if the API is not
> >> supposed to be supported, the function should have no effect?
> >>
> >
> > Yes, originally I meant to no parse the input, then return zero UUID.
> > I changed it to the same as Linux:
> > rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
> > Since it is inited to zero.
>
> Yes but on Linux it has effect because Linux supports it. On FreeBSD
> there's no VFIO, so the function should IMO have no effect. However, i
> don't have a strong opinion on this, so i'm fine either way.
>
Uh, seems empty function should be good one. I will change it. ;-)
> --
> Thanks,
> Anatoly
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
however please consult your distributions documentation to make sure that is the case.
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+ 1. Generate the VF token by uuid command
+ 14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+ 2. sudo modprobe vfio-pci enable_sriov=1
+
+ 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+ 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+ 4. Start the PF:
+ ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+ --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+ 5. Start the VF:
+ ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+ --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
.. note::
- ``vfio-pci`` module doesn't support the creation of virtual functions.
+ ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
@@ -40,6 +40,10 @@ Device-related options
Use specified interrupt mode for devices bound to VFIO kernel driver.
+* ``--vfio-vf-token <uuid>``
+
+ Use specified VF token for devices bound to VFIO kernel driver.
+
Multiprocessing-related options
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -93,6 +93,12 @@ New Features
* Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows
contexts from the port.
+* **Added the support for vfio-pci new VF token interface.**
+
+ Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+ the trust between SR-IOV PF and the created VFs. Update the method to gain access
+ to the PF and VFs devices by appending the VF token parameter.
+
* **Updated Amazon ena driver.**
Updated ena PMD with new features and improvements, including:
@@ -89,6 +89,7 @@ eal_long_options[] = {
{OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM },
{OPT_VDEV, 1, NULL, OPT_VDEV_NUM },
{OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM },
+ {OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM },
{OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM },
{OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM },
{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
/* if set to NONE, interrupt mode is determined automatically */
internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+ memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
#ifdef RTE_LIBEAL_USE_HPET
internal_cfg->no_hpet = 0;
@@ -72,6 +72,8 @@ struct internal_config {
volatile int syslog_facility; /**< facility passed to openlog() */
/** default interrupt mode for VFIO */
volatile enum rte_intr_mode vfio_intr_mode;
+ /** the shared VF token for VFIO-PCI bound PF and VFs devices */
+ rte_uuid_t vfio_vf_token;
char *hugefile_prefix; /**< the base filename of hugetlbfs files */
char *hugepage_dir; /**< specific hugetlbfs directory to use */
char *user_mbuf_pool_ops_name;
@@ -67,6 +67,8 @@ enum {
OPT_VDEV_NUM,
#define OPT_VFIO_INTR "vfio-intr"
OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN "vfio-vf-token"
+ OPT_VFIO_VF_TOKEN_NUM,
#define OPT_VMWARE_TSC_MAP "vmware-tsc-map"
OPT_VMWARE_TSC_MAP_NUM,
#define OPT_LEGACY_MEM "legacy-mem"
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
return RTE_INTR_MODE_NONE;
}
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+ memset(vf_token, 0, sizeof(rte_uuid_t));
+}
+
int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
__rte_unused const char *dev_addr,
__rte_unused int *vfio_dev_fd,
@@ -21,6 +21,7 @@
#include <rte_bus.h>
#include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
#ifdef __cplusplus
extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
*/
enum rte_intr_mode rte_eal_vfio_intr_mode(void);
+
+/**
+ * The user-configured vfio VF token.
+ *
+ * @param vf_token
+ * vfio VF token configured with the command line is copied
+ * into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token);
+
/**
* A wrap API for syscall gettid.
*
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
" --"OPT_FILE_PREFIX" Prefix for hugepage filenames\n"
" --"OPT_CREATE_UIO_DEV" Create /dev/uioX (usually done by hotplug)\n"
" --"OPT_VFIO_INTR" Interrupt mode for VFIO (legacy|msi|msix)\n"
+ " --"OPT_VFIO_VF_TOKEN" VF token (UUID) shared between SR-IOV PF and VFs\n"
" --"OPT_LEGACY_MEM" Legacy memory mode (no dynamic allocation, contiguous segments)\n"
" --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
" --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
return -1;
}
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+ rte_uuid_t uuid;
+
+ if (!rte_uuid_parse(vf_token, uuid)) {
+ rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+ return 0;
+ }
+
+ return -1;
+}
+
/* Parse the arguments for --log-level only */
static void
eal_log_level_parse(int argc, char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv)
}
break;
+ case OPT_VFIO_VF_TOKEN_NUM:
+ if (eal_parse_vfio_vf_token(optarg) < 0) {
+ RTE_LOG(ERR, EAL, "invalid parameters for --"
+ OPT_VFIO_VF_TOKEN "\n");
+ eal_usage(prgname);
+ ret = -1;
+ goto out;
+ }
+ break;
+
case OPT_CREATE_UIO_DEV_NUM:
internal_config.create_uio_dev = 1;
break;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)
return internal_config.vfio_intr_mode;
}
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+ rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
int
rte_eal_check_module(const char *module_name)
{
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
int vfio_container_fd;
int vfio_group_fd;
int iommu_group_num;
+ rte_uuid_t vf_token;
int i, ret;
/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
t->type_id, t->name);
}
+ rte_eal_vfio_vf_token(vf_token);
+
+ /* get a file descriptor for the device with VF token firstly */
+ if (!rte_uuid_is_null(vf_token)) {
+ char vf_token_str[RTE_UUID_STRLEN];
+ char dev[PATH_MAX];
+
+ rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+ snprintf(dev, sizeof(dev),
+ "%s vf_token=%s", dev_addr, vf_token_str);
+
+ *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+ dev);
+ if (*vfio_dev_fd >= 0)
+ goto dev_get_info;
+ }
+
/* get a file descriptor for the device */
*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
return -1;
}
+dev_get_info:
/* test and setup the device */
ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
if (ret) {
@@ -373,6 +373,7 @@ EXPERIMENTAL {
__rte_trace_point_register;
per_lcore_trace_mem;
per_lcore_trace_point_sz;
+ rte_eal_vfio_vf_token;
rte_log_can_log;
rte_thread_getname;
rte_trace_dump;