[dpdk-dev,v3,02/20] thunderx/nicvf: add pmd skeleton

  Introduce driver initialization and enable build infrastructure for
nicvf pmd driver.

By default, It is enabled only for defconfig_arm64-thunderx-*
config as it is an inbuilt NIC device.

Signed-off-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
Signed-off-by: Maciej Czekaj <maciej.czekaj@caviumnetworks.com>
Signed-off-by: Kamil Rytarowski <Kamil.Rytarowski@caviumnetworks.com>
Signed-off-by: Zyta Szpak <zyta.szpak@semihalf.com>
Signed-off-by: Slawomir Rosek <slawomir.rosek@semihalf.com>
Signed-off-by: Radoslaw Biernacki <rad@semihalf.com>
---
 config/common_base                                 |  10 +
 config/defconfig_arm64-thunderx-linuxapp-gcc       |  10 +
 drivers/net/Makefile                               |   1 +
 drivers/net/thunderx/Makefile                      |  63 ++++++
 drivers/net/thunderx/nicvf_ethdev.c                | 251 +++++++++++++++++++++
 drivers/net/thunderx/nicvf_ethdev.h                |  48 ++++
 drivers/net/thunderx/nicvf_logs.h                  |  83 +++++++
 drivers/net/thunderx/nicvf_struct.h                | 124 ++++++++++
 .../thunderx/rte_pmd_thunderx_nicvf_version.map    |   4 +
 mk/rte.app.mk                                      |   2 +
 10 files changed, 596 insertions(+)
 create mode 100644 drivers/net/thunderx/Makefile
 create mode 100644 drivers/net/thunderx/nicvf_ethdev.c
 create mode 100644 drivers/net/thunderx/nicvf_ethdev.h
 create mode 100644 drivers/net/thunderx/nicvf_logs.h
 create mode 100644 drivers/net/thunderx/nicvf_struct.h
 create mode 100644 drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map
  

On 6/7/2016 5:40 PM, Jerin Jacob wrote:
> Introduce driver initialization and enable build infrastructure for
> nicvf pmd driver.
> 
> By default, It is enabled only for defconfig_arm64-thunderx-*
> config as it is an inbuilt NIC device.
> 
> Signed-off-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> Signed-off-by: Maciej Czekaj <maciej.czekaj@caviumnetworks.com>
> Signed-off-by: Kamil Rytarowski <Kamil.Rytarowski@caviumnetworks.com>
> Signed-off-by: Zyta Szpak <zyta.szpak@semihalf.com>
> Signed-off-by: Slawomir Rosek <slawomir.rosek@semihalf.com>
> Signed-off-by: Radoslaw Biernacki <rad@semihalf.com>
> ---
>  config/common_base                                 |  10 +
>  config/defconfig_arm64-thunderx-linuxapp-gcc       |  10 +
>  drivers/net/Makefile                               |   1 +
>  drivers/net/thunderx/Makefile                      |  63 ++++++
>  drivers/net/thunderx/nicvf_ethdev.c                | 251 +++++++++++++++++++++
>  drivers/net/thunderx/nicvf_ethdev.h                |  48 ++++
>  drivers/net/thunderx/nicvf_logs.h                  |  83 +++++++
>  drivers/net/thunderx/nicvf_struct.h                | 124 ++++++++++
>  .../thunderx/rte_pmd_thunderx_nicvf_version.map    |   4 +
>  mk/rte.app.mk                                      |   2 +
>  10 files changed, 596 insertions(+)
>  create mode 100644 drivers/net/thunderx/Makefile
>  create mode 100644 drivers/net/thunderx/nicvf_ethdev.c
>  create mode 100644 drivers/net/thunderx/nicvf_ethdev.h
>  create mode 100644 drivers/net/thunderx/nicvf_logs.h
>  create mode 100644 drivers/net/thunderx/nicvf_struct.h
>  create mode 100644 drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map
> 

...

> +
> +	if (nic->sqs_mode) {
> +		PMD_INIT_LOG(INFO, "Unsupported SQS VF detected, Detaching...");
> +		/* Detach port by returning Postive error number */

s/Postive/Positive ?

...
  

On 6/7/2016 5:40 PM, Jerin Jacob wrote:
> Introduce driver initialization and enable build infrastructure for
> nicvf pmd driver.
> 
> By default, It is enabled only for defconfig_arm64-thunderx-*
> config as it is an inbuilt NIC device.
> 
> Signed-off-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> Signed-off-by: Maciej Czekaj <maciej.czekaj@caviumnetworks.com>
> Signed-off-by: Kamil Rytarowski <Kamil.Rytarowski@caviumnetworks.com>
> Signed-off-by: Zyta Szpak <zyta.szpak@semihalf.com>
> Signed-off-by: Slawomir Rosek <slawomir.rosek@semihalf.com>
> Signed-off-by: Radoslaw Biernacki <rad@semihalf.com>
> ---
>  config/common_base                                 |  10 +
>  config/defconfig_arm64-thunderx-linuxapp-gcc       |  10 +
>  drivers/net/Makefile                               |   1 +
>  drivers/net/thunderx/Makefile                      |  63 ++++++
>  drivers/net/thunderx/nicvf_ethdev.c                | 251 +++++++++++++++++++++
>  drivers/net/thunderx/nicvf_ethdev.h                |  48 ++++
>  drivers/net/thunderx/nicvf_logs.h                  |  83 +++++++
>  drivers/net/thunderx/nicvf_struct.h                | 124 ++++++++++
>  .../thunderx/rte_pmd_thunderx_nicvf_version.map    |   4 +
>  mk/rte.app.mk                                      |   2 +
>  10 files changed, 596 insertions(+)
>  create mode 100644 drivers/net/thunderx/Makefile
>  create mode 100644 drivers/net/thunderx/nicvf_ethdev.c
>  create mode 100644 drivers/net/thunderx/nicvf_ethdev.h
>  create mode 100644 drivers/net/thunderx/nicvf_logs.h
>  create mode 100644 drivers/net/thunderx/nicvf_struct.h
>  create mode 100644 drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map
> 
> diff --git a/config/common_base b/config/common_base
> index 47c26f6..ad5686b 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -259,6 +259,16 @@ CONFIG_RTE_LIBRTE_PMD_SZEDATA2=n
>  CONFIG_RTE_LIBRTE_PMD_SZEDATA2_AS=0
>  
>  #
> +# Compile burst-oriented Cavium Thunderx NICVF PMD driver
> +#
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD=n
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_INIT=n
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_RX=n
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_TX=n
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_DRIVER=n
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_MBOX=n
> +
> +#
>  # Compile burst-oriented VIRTIO PMD driver
>  #
>  CONFIG_RTE_LIBRTE_VIRTIO_PMD=y
> diff --git a/config/defconfig_arm64-thunderx-linuxapp-gcc b/config/defconfig_arm64-thunderx-linuxapp-gcc
> index fe5e987..7940bbd 100644
> --- a/config/defconfig_arm64-thunderx-linuxapp-gcc
> +++ b/config/defconfig_arm64-thunderx-linuxapp-gcc
> @@ -34,3 +34,13 @@
>  CONFIG_RTE_MACHINE="thunderx"
>  
>  CONFIG_RTE_CACHE_LINE_SIZE=128
> +
> +#
> +# Compile Cavium Thunderx NICVF PMD driver
> +#
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD=y
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_INIT=n
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_RX=n
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_TX=n
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_DRIVER=n
> +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_MBOX=n
> diff --git a/drivers/net/Makefile b/drivers/net/Makefile
> index 6ba7658..0e29a33 100644
> --- a/drivers/net/Makefile
> +++ b/drivers/net/Makefile
> @@ -50,6 +50,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += pcap
>  DIRS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede
>  DIRS-$(CONFIG_RTE_LIBRTE_PMD_RING) += ring
>  DIRS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += szedata2
> +DIRS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += thunderx
>  DIRS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio
>  DIRS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += vmxnet3
>  DIRS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += xenvirt
> diff --git a/drivers/net/thunderx/Makefile b/drivers/net/thunderx/Makefile
> new file mode 100644
> index 0000000..eb9f100
> --- /dev/null
> +++ b/drivers/net/thunderx/Makefile
> @@ -0,0 +1,63 @@
> +#   BSD LICENSE
> +#
> +#   Copyright(c) 2016 Cavium Networks. All rights reserved.
> +#   All rights reserved.
> +#
> +#   Redistribution and use in source and binary forms, with or without
> +#   modification, are permitted provided that the following conditions
> +#   are met:
> +#
> +#     * Redistributions of source code must retain the above copyright
> +#       notice, this list of conditions and the following disclaimer.
> +#     * Redistributions in binary form must reproduce the above copyright
> +#       notice, this list of conditions and the following disclaimer in
> +#       the documentation and/or other materials provided with the
> +#       distribution.
> +#     * Neither the name of Cavium Networks nor the names of its
> +#       contributors may be used to endorse or promote products derived
> +#       from this software without specific prior written permission.
> +#
> +#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +#
> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_pmd_thunderx_nicvf.a
> +
> +CFLAGS += $(WERROR_FLAGS)
> +
> +EXPORT_MAP := rte_pmd_thunderx_nicvf_version.map
> +
> +LIBABIVER := 1
> +
> +OBJS_BASE_DRIVER=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))
> +$(foreach obj, $(OBJS_BASE_DRIVER), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER)))
> +
> +VPATH += $(SRCDIR)/base
> +
> +#
> +# all source are stored in SRCS-y
> +#
> +SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_hw.c
> +SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_mbox.c
> +SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_ethdev.c
> +
> +
> +# this lib depends upon:
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += lib/librte_eal lib/librte_ether
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += lib/librte_mempool lib/librte_mbuf
> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c
> new file mode 100644
> index 0000000..45bfc13
> --- /dev/null
> +++ b/drivers/net/thunderx/nicvf_ethdev.c
> @@ -0,0 +1,251 @@
> +/*
> + *   BSD LICENSE
> + *
> + *   Copyright (C) Cavium networks Ltd. 2016.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Cavium networks nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#include <assert.h>
> +#include <stdio.h>
> +#include <stdbool.h>
> +#include <errno.h>
> +#include <stdint.h>
> +#include <string.h>
> +#include <unistd.h>
> +#include <stdarg.h>
> +#include <inttypes.h>
> +#include <netinet/in.h>
> +#include <sys/queue.h>
> +#include <sys/timerfd.h>
> +
> +#include <rte_common.h>
> +#include <rte_byteorder.h>
> +#include <rte_cycles.h>
> +#include <rte_interrupts.h>
> +#include <rte_log.h>
> +#include <rte_debug.h>
> +#include <rte_pci.h>
> +#include <rte_atomic.h>
> +#include <rte_branch_prediction.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_tailq.h>
> +#include <rte_eal.h>
> +#include <rte_alarm.h>
> +#include <rte_ether.h>
> +#include <rte_ethdev.h>
> +#include <rte_malloc.h>
> +#include <rte_random.h>
> +#include <rte_dev.h>
I guess this is not part of convention but I think these are better
sorted alphabetically, just personal taste

> +
> +#include "base/nicvf_plat.h"
> +
> +#include "nicvf_ethdev.h"
> +
> +#include "nicvf_logs.h"
> +
> +static void
> +nicvf_interrupt(void *arg)
> +{
> +	struct nicvf *nic = (struct nicvf *)arg;
unnecessary cast

> +
> +	nicvf_reg_poll_interrupts(nic);
> +
> +	rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
> +				nicvf_interrupt, nic);
> +}
> +
> +static int
> +nicvf_periodic_alarm_start(struct nicvf *nic)
> +{
> +	return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
> +					nicvf_interrupt, nic);
> +}
> +
> +static int
> +nicvf_periodic_alarm_stop(struct nicvf *nic)
> +{
> +	return rte_eal_alarm_cancel(nicvf_interrupt, nic);
> +}
> +
> +/* Initialize and register driver with DPDK Application */
> +static const struct eth_dev_ops nicvf_eth_dev_ops = {
> +};
no assignment, does {} required?

> +
> +static int
> +nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
> +{
> +	int ret;
> +	struct rte_pci_device *pci_dev;
> +	struct nicvf *nic = nicvf_pmd_priv(eth_dev);
> +
> +	PMD_INIT_FUNC_TRACE();
> +
> +	eth_dev->dev_ops = &nicvf_eth_dev_ops;
> +
> +	pci_dev = eth_dev->pci_dev;
> +	rte_eth_copy_pci_info(eth_dev, pci_dev);
> +
> +	nic->device_id = pci_dev->id.device_id;
> +	nic->vendor_id = pci_dev->id.vendor_id;
> +	nic->subsystem_device_id = pci_dev->id.subsystem_device_id;
> +	nic->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
> +	nic->eth_dev = eth_dev;
> +
> +	PMD_INIT_LOG(DEBUG, "nicvf: device (%x:%x) %u:%u:%u:%u",
> +			pci_dev->id.vendor_id, pci_dev->id.device_id,
> +			pci_dev->addr.domain, pci_dev->addr.bus,
> +			pci_dev->addr.devid, pci_dev->addr.function);
> +
> +	nic->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr;
> +	if (!nic->reg_base) {
> +		PMD_INIT_LOG(ERR, "Failed to map BAR0");
> +		ret = -ENODEV;
> +		goto fail;
> +	}
> +
> +	nicvf_disable_all_interrupts(nic);
> +
> +	ret = nicvf_periodic_alarm_start(nic);
> +	if (ret) {
> +		PMD_INIT_LOG(ERR, "Failed to start period alarm");
> +		goto fail;
> +	}
> +
> +	ret = nicvf_mbox_check_pf_ready(nic);
> +	if (ret) {
> +		PMD_INIT_LOG(ERR, "Failed to get ready message from PF");
> +		goto alarm_fail;
> +	} else {
> +		PMD_INIT_LOG(INFO,
> +			"node=%d vf=%d mode=%s sqs=%s loopback_supported=%s",
> +			nic->node, nic->vf_id,
> +			nic->tns_mode == NIC_TNS_MODE ? "tns" : "tns-bypass",
> +			nic->sqs_mode ? "true" : "false",
> +			nic->loopback_supported ? "true" : "false"
> +			);
> +	}
> +
> +	if (nic->sqs_mode) {
> +		PMD_INIT_LOG(INFO, "Unsupported SQS VF detected, Detaching...");
> +		/* Detach port by returning Postive error number */
> +		ret = ENOTSUP;
Intended to have negative value? Although this looks valid since
rte_eth_dev_init() just check zero or not.

> +		goto alarm_fail;
> +	}
> +
> +	eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", ETHER_ADDR_LEN, 0);
> +	if (eth_dev->data->mac_addrs == NULL) {
> +		PMD_INIT_LOG(ERR, "Failed to allocate memory for mac addr");
> +		ret = -ENOMEM;
> +		goto alarm_fail;
> +	}
> +	if (is_zero_ether_addr((struct ether_addr *)nic->mac_addr))
> +		eth_random_addr(&nic->mac_addr[0]);
> +
> +	ether_addr_copy((struct ether_addr *)nic->mac_addr,
> +			&eth_dev->data->mac_addrs[0]);
> +
> +	ret = nicvf_mbox_set_mac_addr(nic, nic->mac_addr);
> +	if (ret) {
> +		PMD_INIT_LOG(ERR, "Failed to set mac addr");
> +		goto malloc_fail;
> +	}
> +
> +	ret = nicvf_base_init(nic);
> +	if (ret) {
> +		PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init");
> +		goto malloc_fail;
> +	}
> +
> +	ret = nicvf_mbox_get_rss_size(nic);
> +	if (ret) {
> +		PMD_INIT_LOG(ERR, "Failed to get rss table size");
> +		goto malloc_fail;
> +	}
> +
> +	PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x",
> +		eth_dev->data->port_id, nic->vendor_id, nic->device_id,
> +		nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2],
> +		nic->mac_addr[3], nic->mac_addr[4], nic->mac_addr[5]);
> +
> +	return 0;
> +
> +malloc_fail:
> +	rte_free(eth_dev->data->mac_addrs);
> +alarm_fail:
> +	nicvf_periodic_alarm_stop(nic);
> +fail:
> +	return ret;
> +}
> +
> +static const struct rte_pci_id pci_id_nicvf_map[] = {
> +	{
> +		.vendor_id = PCI_VENDOR_ID_CAVIUM,
> +		.device_id = PCI_DEVICE_ID_THUNDERX_PASS1_NICVF,
> +		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
> +		.subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS1_NICVF,
> +	},
> +	{
> +		.vendor_id = PCI_VENDOR_ID_CAVIUM,
> +		.device_id = PCI_DEVICE_ID_THUNDERX_PASS2_NICVF,
> +		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
> +		.subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS2_NICVF,
> +	},
> +	{
> +		.vendor_id = 0,
> +	},
> +};
> +
> +static struct eth_driver rte_nicvf_pmd = {
> +	.pci_drv = {
> +		.name = "rte_nicvf_pmd",
> +		.id_table = pci_id_nicvf_map,
> +		.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
> +	},
> +	.eth_dev_init = nicvf_eth_dev_init,
> +	.dev_private_size = sizeof(struct nicvf),
> +};
> +
> +static int
> +rte_nicvf_pmd_init(const char *name __rte_unused, const char *para __rte_unused)
> +{
> +	PMD_INIT_FUNC_TRACE();
> +	PMD_INIT_LOG(INFO, "librte_pmd_thunderx nicvf version %s",
> +			THUNDERX_NICVF_PMD_VERSION);
> +
> +	rte_eth_driver_register(&rte_nicvf_pmd);
> +	return 0;
> +}
> +
> +static struct rte_driver rte_nicvf_driver = {
> +	.name = "nicvf_driver",
> +	.type = PMD_PDEV,
> +	.init = rte_nicvf_pmd_init,
> +};
> +
> +PMD_REGISTER_DRIVER(rte_nicvf_driver);
> diff --git a/drivers/net/thunderx/nicvf_ethdev.h b/drivers/net/thunderx/nicvf_ethdev.h
> new file mode 100644
> index 0000000..d4d2071
> --- /dev/null
> +++ b/drivers/net/thunderx/nicvf_ethdev.h
> @@ -0,0 +1,48 @@
> +/*
> + *   BSD LICENSE
> + *
> + *   Copyright (C) Cavium networks Ltd. 2016.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Cavium networks nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef __THUNDERX_NICVF_ETHDEV_H__
> +#define __THUNDERX_NICVF_ETHDEV_H__
> +
> +#include <rte_ethdev.h>
> +
> +#define THUNDERX_NICVF_PMD_VERSION      "1.0"
> +
> +#define NICVF_INTR_POLL_INTERVAL_MS	50
> +
> +static inline struct nicvf *
> +nicvf_pmd_priv(struct rte_eth_dev *eth_dev)
> +{
> +	return eth_dev->data->dev_private;
> +}
> +
> +#endif /* __THUNDERX_NICVF_ETHDEV_H__  */
> diff --git a/drivers/net/thunderx/nicvf_logs.h b/drivers/net/thunderx/nicvf_logs.h
> new file mode 100644
> index 0000000..0667d46
> --- /dev/null
> +++ b/drivers/net/thunderx/nicvf_logs.h
> @@ -0,0 +1,83 @@
> +/*
> + *   BSD LICENSE
> + *
> + *   Copyright (C) Cavium networks Ltd. 2016.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Cavium networks nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef __THUNDERX_NICVF_LOGS__
> +#define __THUNDERX_NICVF_LOGS__
> +
> +#include <assert.h>
> +
> +#define PMD_INIT_LOG(level, fmt, args...) \
> +	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +
> +#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_INIT
> +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, ">>")
> +#else
> +#define PMD_INIT_FUNC_TRACE() do { } while (0)
> +#endif
> +
> +#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_RX
> +#define PMD_RX_LOG(level, fmt, args...) \
> +	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +#define NICVF_RX_ASSERT(x) assert(x)
> +#else
> +#define PMD_RX_LOG(level, fmt, args...) do { } while (0)
> +#define NICVF_RX_ASSERT(x) do { } while (0)
> +#endif
> +
> +#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_TX
> +#define PMD_TX_LOG(level, fmt, args...) \
> +	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +#define NICVF_TX_ASSERT(x) assert(x)
> +#else
> +#define PMD_TX_LOG(level, fmt, args...) do { } while (0)
> +#define NICVF_TX_ASSERT(x) do { } while (0)
> +#endif
> +
> +#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_DRIVER
> +#define PMD_DRV_LOG(level, fmt, args...) \
> +	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +#define PMD_DRV_FUNC_TRACE() PMD_DRV_LOG(DEBUG, ">>")
> +#else
> +#define PMD_DRV_LOG(level, fmt, args...) do { } while (0)
> +#define PMD_DRV_FUNC_TRACE() do { } while (0)
> +#endif
> +
> +#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_MBOX
> +#define PMD_MBOX_LOG(level, fmt, args...) \
> +	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +#define PMD_MBOX_FUNC_TRACE() PMD_DRV_LOG(DEBUG, ">>")
> +#else
> +#define PMD_MBOX_LOG(level, fmt, args...) do { } while (0)
> +#define PMD_MBOX_FUNC_TRACE() do { } while (0)
> +#endif
> +
> +#endif /* __THUNDERX_NICVF_LOGS__ */
> diff --git a/drivers/net/thunderx/nicvf_struct.h b/drivers/net/thunderx/nicvf_struct.h
> new file mode 100644
> index 0000000..c52545d
> --- /dev/null
> +++ b/drivers/net/thunderx/nicvf_struct.h
> @@ -0,0 +1,124 @@
> +/*
> + *   BSD LICENSE
> + *
> + *   Copyright (C) Cavium networks Ltd. 2016.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Cavium networks nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef _THUNDERX_NICVF_STRUCT_H
> +#define _THUNDERX_NICVF_STRUCT_H
> +
> +#include <stdint.h>
> +
> +#include <rte_spinlock.h>
> +#include <rte_mempool.h>
> +#include <rte_mbuf.h>
> +#include <rte_interrupts.h>
> +#include <rte_ethdev.h>
> +#include <rte_memory.h>
> +
> +struct nicvf_rbdr {
> +	uint64_t rbdr_status;
> +	uint64_t rbdr_door;
> +	struct rbdr_entry_t *desc;
> +	nicvf_phys_addr_t phys;
> +	uint32_t buffsz;
> +	uint32_t tail;
> +	uint32_t next_tail;
> +	uint32_t head;
> +	uint32_t qlen_mask;
> +} __rte_cache_aligned;
> +
> +struct nicvf_txq {
> +	union sq_entry_t *desc;
> +	nicvf_phys_addr_t phys;
> +	struct rte_mbuf **txbuffs;
> +	uint64_t sq_head;
> +	uint64_t sq_door;
> +	struct rte_mempool *pool;
> +	struct nicvf *nic;
> +	void (*pool_free)(struct nicvf_txq *sq);
> +	uint32_t head;
> +	uint32_t tail;
> +	int32_t xmit_bufs;
> +	uint32_t qlen_mask;
> +	uint32_t txq_flags;
> +	uint16_t queue_id;
> +	uint16_t tx_free_thresh;
> +} __rte_cache_aligned;
> +
> +struct nicvf_rxq {
> +	uint64_t mbuf_phys_off;
> +	uint64_t cq_status;
> +	uint64_t cq_door;
> +	nicvf_phys_addr_t phys;
> +	union cq_entry_t *desc;
> +	struct nicvf_rbdr *shared_rbdr;
> +	struct nicvf *nic;
> +	struct rte_mempool *pool;
> +	uint32_t head;
> +	uint32_t qlen_mask;
> +	int32_t available_space;
> +	int32_t recv_buffers;
> +	uint16_t rx_free_thresh;
> +	uint16_t queue_id;
> +	uint16_t precharge_cnt;
> +	uint8_t rx_drop_en;
> +	uint8_t  port_id;
> +	uint8_t  rbptr_offset;
> +} __rte_cache_aligned;
> +
> +struct nicvf {
> +	uint8_t vf_id;
> +	uint8_t node;
> +	uintptr_t reg_base;
> +	bool tns_mode;
> +	bool sqs_mode;
> +	bool loopback_supported;
> +	bool pf_acked:1;
> +	bool pf_nacked:1;
> +	uint64_t hwcap;
> +	uint8_t link_up;
> +	uint8_t	duplex;
> +	uint32_t speed;
> +	uint32_t msg_enable;
> +	uint16_t device_id;
> +	uint16_t vendor_id;
> +	uint16_t subsystem_device_id;
> +	uint16_t subsystem_vendor_id;
> +	struct nicvf_rbdr *rbdr;
> +	struct nicvf_rss_reta_info rss_info;
> +	struct rte_eth_dev *eth_dev;
> +	struct rte_intr_handle intr_handle;
> +	uint8_t cpi_alg;
> +	uint16_t mtu;
> +	bool vlan_filter_en;
> +	uint8_t mac_addr[ETHER_ADDR_LEN];
> +} __rte_cache_aligned;
> +
> +#endif /* _THUNDERX_NICVF_STRUCT_H */
> diff --git a/drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map b/drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map
> new file mode 100644
> index 0000000..349c6e1
> --- /dev/null
> +++ b/drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map
> @@ -0,0 +1,4 @@
> +DPDK_16.04 {
DPDK_16.07?

> +
> +	local: *;
> +};
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk
> index b84b56d..1d8d8cd 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -102,6 +102,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT)    += -lxenstore
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_MPIPE_PMD)      += -lgxio
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lm
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_QEDE_PMD)       += -lz
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += -lm
>  # QAT / AESNI GCM PMDs are dependent on libcrypto (from openssl)
>  # for calculating HMAC precomputes
>  ifeq ($(CONFIG_RTE_LIBRTE_PMD_QAT),y)
> @@ -150,6 +151,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_PCAP)       += -lrte_pmd_pcap
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET)  += -lrte_pmd_af_packet
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_QEDE_PMD)       += -lrte_pmd_qede
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += -lrte_pmd_thunderx_nicvf
>  
>  ifeq ($(CONFIG_RTE_LIBRTE_CRYPTODEV),y)
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_QAT)        += -lrte_pmd_qat
>