[dpdk-dev] [PATCH v4 08/14] virtio: pci: extend virtio pci rw api for vfio interface

[Santosh Shukla]

On Mon, Jan 18, 2016 at 12:29 PM, Jason Wang <jasowang at redhat.com> wrote:
>
>
> On 01/14/2016 09:28 PM, Santosh Shukla wrote:
>> So far virtio handle rw access for uio / ioport interface, This patch to extend
>> the support for vfio interface. For that introducing private struct
>> virtio_vfio_dev{
>>       - is_vfio
>>       - pci_dev
>>       };
>> Signed-off-by: Santosh Shukla <sshukla at mvista.com>
>> ---
>> v3->v4:
>> - Removed #indef RTE_EAL_VFIO and made it arch agnostic such now virtio_pci
>>   rd/wr api to handle both vfio and ig_uio/ioport interfaces, depending upon
>>   is_vfio flags set or unset.
>> - Tested for x86 for igb_uio and vfio interface, also  tested for arm64 for vfio
>>   interface.
>>
>> drivers/net/virtio/virtio_pci.h |   84 ++++++++++++++++++++++++++++++++-------
>>  1 file changed, 70 insertions(+), 14 deletions(-)
>
> Interesting. I'm working on IOMMU cooperation with virtio[1]. For pmd,
> it looks like the only thing missed is RTE_PCI_DRV_NEED_MAPPING for
> virito-net pmd.
>

not missing anymore, I am using rte_eal_pci_map() api so to map virtio
pci dev to userspace.

> So I'm curious whether this is still necessary if IOMMU can work with
> virito in the future.
>
So far I'm using pmd driver in vfio-noiommu way. AFAIk, pmd driver use
dma for tx side. I glanced through your patch, to me it would be
interesting to try out, but right now I am not sure. We'll come back I
guess. Thanks

> [1] https://www.mail-archive.com/qemu-devel@nongnu.org/msg337079.html
>
> Thanks
>
>>
>> diff --git a/drivers/net/virtio/virtio_pci.h b/drivers/net/virtio/virtio_pci.h
>> index 8b5b031..8526c07 100644
>> --- a/drivers/net/virtio/virtio_pci.h
>> +++ b/drivers/net/virtio/virtio_pci.h
>> @@ -46,6 +46,8 @@
>>  #endif
>>
>>  #include <rte_ethdev.h>
>> +#include <stdbool.h>
>> +#include "virtio_vfio_rw.h"
>>
>>  struct virtqueue;
>>
>> @@ -165,6 +167,14 @@ struct virtqueue;
>>   */
>>  #define VIRTIO_MAX_VIRTQUEUES 8
>>
>> +/* For vfio only */
>> +struct virtio_vfio_dev {
>> +     bool            is_vfio;        /* True: vfio i/f,
>> +                                      * False: not a vfio i/f
>> +                                      */
>> +     struct rte_pci_device *pci_dev; /* vfio dev */
>> +};
>> +
>>  struct virtio_hw {
>>       struct virtqueue *cvq;
>>       uint32_t    io_base;
>> @@ -176,6 +186,7 @@ struct virtio_hw {
>>       uint8_t     use_msix;
>>       uint8_t     started;
>>       uint8_t     mac_addr[ETHER_ADDR_LEN];
>> +     struct virtio_vfio_dev dev;
>>  };
>>
>>  /*
>> @@ -231,20 +242,65 @@ outl_p(unsigned int data, unsigned int port)
>>  #define VIRTIO_PCI_REG_ADDR(hw, reg) \
>>       (unsigned short)((hw)->io_base + (reg))
>>
>> -#define VIRTIO_READ_REG_1(hw, reg) \
>> -     inb((VIRTIO_PCI_REG_ADDR((hw), (reg))))
>> -#define VIRTIO_WRITE_REG_1(hw, reg, value) \
>> -     outb_p((unsigned char)(value), (VIRTIO_PCI_REG_ADDR((hw), (reg))))
>> -
>> -#define VIRTIO_READ_REG_2(hw, reg) \
>> -     inw((VIRTIO_PCI_REG_ADDR((hw), (reg))))
>> -#define VIRTIO_WRITE_REG_2(hw, reg, value) \
>> -     outw_p((unsigned short)(value), (VIRTIO_PCI_REG_ADDR((hw), (reg))))
>> -
>> -#define VIRTIO_READ_REG_4(hw, reg) \
>> -     inl((VIRTIO_PCI_REG_ADDR((hw), (reg))))
>> -#define VIRTIO_WRITE_REG_4(hw, reg, value) \
>> -     outl_p((unsigned int)(value), (VIRTIO_PCI_REG_ADDR((hw), (reg))))
>> +#define VIRTIO_READ_REG_1(hw, reg)                                   \
>> +({                                                                   \
>> +     uint8_t ret;                                                    \
>> +     struct virtio_vfio_dev *vdev;                                   \
>> +     (vdev) = (&(hw)->dev);                                          \
>> +     (((vdev)->is_vfio) ?                                            \
>> +     (ioport_inb(((vdev)->pci_dev), reg, &ret)) :                    \
>> +     ((ret) = (inb((VIRTIO_PCI_REG_ADDR((hw), (reg)))))));           \
>> +     ret;                                                            \
>> +})
>> +
>> +#define VIRTIO_WRITE_REG_1(hw, reg, value)                           \
>> +({                                                                   \
>> +     struct virtio_vfio_dev *vdev;                                   \
>> +     (vdev) = (&(hw)->dev);                                          \
>> +     (((vdev)->is_vfio) ?                                            \
>> +     (ioport_outb_p(((vdev)->pci_dev), reg, (uint8_t)(value))) :     \
>> +     (outb_p((unsigned char)(value), (VIRTIO_PCI_REG_ADDR((hw), (reg)))))); \
>> +})
>> +
>> +#define VIRTIO_READ_REG_2(hw, reg)                                   \
>> +({                                                                   \
>> +     uint16_t ret;                                                   \
>> +     struct virtio_vfio_dev *vdev;                                   \
>> +     (vdev) = (&(hw)->dev);                                          \
>> +     (((vdev)->is_vfio) ?                                            \
>> +     (ioport_inw(((vdev)->pci_dev), reg, &ret)) :                    \
>> +     ((ret) = (inw((VIRTIO_PCI_REG_ADDR((hw), (reg)))))));           \
>> +     ret;                                                            \
>> +})
>> +
>> +#define VIRTIO_WRITE_REG_2(hw, reg, value)                           \
>> +({                                                                   \
>> +     struct virtio_vfio_dev *vdev;                                   \
>> +     (vdev) = (&(hw)->dev);                                          \
>> +     (((vdev)->is_vfio) ?                                            \
>> +     (ioport_outw_p(((vdev)->pci_dev), reg, (uint16_t)(value))) :    \
>> +     (outw_p((unsigned short)(value), (VIRTIO_PCI_REG_ADDR((hw), (reg)))))); \
>> +})
>> +
>> +#define VIRTIO_READ_REG_4(hw, reg)                                   \
>> +({                                                                   \
>> +     uint32_t ret;                                                   \
>> +     struct virtio_vfio_dev *vdev;                                   \
>> +     (vdev) = (&(hw)->dev);                                          \
>> +     (((vdev)->is_vfio) ?                                            \
>> +     (ioport_inl(((vdev)->pci_dev), reg, &ret)) :                    \
>> +     ((ret) = (inl((VIRTIO_PCI_REG_ADDR((hw), (reg)))))));           \
>> +     ret;                                                            \
>> +})
>> +
>> +#define VIRTIO_WRITE_REG_4(hw, reg, value)                           \
>> +({                                                                   \
>> +     struct virtio_vfio_dev *vdev;                                   \
>> +     (vdev) = (&(hw)->dev);                                          \
>> +     (((vdev)->is_vfio) ?                                            \
>> +     (ioport_outl_p(((vdev)->pci_dev), reg, (uint32_t)(value))) :    \
>> +     (outl_p((unsigned int)(value), (VIRTIO_PCI_REG_ADDR((hw), (reg)))))); \
>> +})
>>
>>  static inline int
>>  vtpci_with_feature(struct virtio_hw *hw, uint32_t bit)
>