[dpdk-dev] [PATCH v3 1/7] net/ark: PMD for Atomic Rules Arkville driver stub

Ferruh Yigit ferruh.yigit at intel.com
Wed Mar 22 19:16:19 CET 2017


On 3/21/2017 9:43 PM, Ed Czeck wrote:
> Enable Arkville on supported configurations
> Add overview documentation
> Minimum driver support for valid compile
> 
> 
> Signed-off-by: Ed Czeck <ed.czeck at atomicrules.com>
> ---
>  MAINTAINERS                                 |   8 +
>  config/common_base                          |  11 ++
>  config/defconfig_arm-armv7a-linuxapp-gcc    |   1 +
>  config/defconfig_ppc_64-power8-linuxapp-gcc |   1 +
>  doc/guides/nics/ark.rst                     | 237 +++++++++++++++++++++++
>  doc/guides/nics/features/ark.ini            |  15 ++
>  doc/guides/nics/index.rst                   |   1 +
>  drivers/net/Makefile                        |   1 +
>  drivers/net/ark/Makefile                    |  63 +++++++
>  drivers/net/ark/ark_debug.h                 |  74 ++++++++
>  drivers/net/ark/ark_ethdev.c                | 281 ++++++++++++++++++++++++++++
>  drivers/net/ark/ark_ethdev.h                |  39 ++++
>  drivers/net/ark/ark_global.h                | 108 +++++++++++
>  drivers/net/ark/rte_pmd_ark_version.map     |   4 +
>  mk/rte.app.mk                               |   1 +
>  15 files changed, 845 insertions(+)
>  create mode 100644 doc/guides/nics/ark.rst
>  create mode 100644 doc/guides/nics/features/ark.ini
>  create mode 100644 drivers/net/ark/Makefile
>  create mode 100644 drivers/net/ark/ark_debug.h
>  create mode 100644 drivers/net/ark/ark_ethdev.c
>  create mode 100644 drivers/net/ark/ark_ethdev.h
>  create mode 100644 drivers/net/ark/ark_global.h
>  create mode 100644 drivers/net/ark/rte_pmd_ark_version.map
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 0c78b58..8043d75 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -278,6 +278,14 @@ M: Evgeny Schemeilin <evgenys at amazon.com>
>  F: drivers/net/ena/
>  F: doc/guides/nics/ena.rst
>  
> +Atomic Rules ark

Should prefer uppercase "ARK" here?

> +M: Shepard Siegel <shepard.siegel at atomicrules.com>
> +M: Ed Czeck       <ed.czeck at atomicrules.com>
> +M: John Miller    <john.miller at atomicrules.com>
> +F: /drivers/net/ark/

Can you please drop the leading "/". There is a script
"check-maintainers.sh", which is broken with that.

> +F: doc/guides/nics/ark.rst
> +F: doc/guides/nics/features/ark.ini
> +
>  Broadcom bnxt
>  M: Stephen Hurd <stephen.hurd at broadcom.com>
>  M: Ajit Khaparde <ajit.khaparde at broadcom.com>
> diff --git a/config/common_base b/config/common_base
> index 37aa1e1..0916c44 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -353,6 +353,17 @@ CONFIG_RTE_LIBRTE_QEDE_FW=""
>  CONFIG_RTE_LIBRTE_PMD_AF_PACKET=n
>  
>  #
> +# Compile ARK PMD
> +#
> +CONFIG_RTE_LIBRTE_ARK_PMD=y
> +CONFIG_RTE_LIBRTE_ARK_PAD_TX=y
> +CONFIG_RTE_LIBRTE_ARK_DEBUG_RX=n
> +CONFIG_RTE_LIBRTE_ARK_DEBUG_TX=n
> +CONFIG_RTE_LIBRTE_ARK_DEBUG_STATS=n
> +CONFIG_RTE_LIBRTE_ARK_DEBUG_TRACE=n
> +
> +

Extra line

> +#
>  # Compile the TAP PMD
>  # It is enabled by default for Linux only.
>  #
> diff --git a/config/defconfig_arm-armv7a-linuxapp-gcc b/config/defconfig_arm-armv7a-linuxapp-gcc
> index d9bd2a8..6d2b5e0 100644
> --- a/config/defconfig_arm-armv7a-linuxapp-gcc
> +++ b/config/defconfig_arm-armv7a-linuxapp-gcc
> @@ -61,6 +61,7 @@ CONFIG_RTE_SCHED_VECTOR=n
>  
>  # cannot use those on ARM
>  CONFIG_RTE_KNI_KMOD=n
> +CONFIG_RTE_LIBRTE_ARK_PMD=n
>  CONFIG_RTE_LIBRTE_EM_PMD=n
>  CONFIG_RTE_LIBRTE_IGB_PMD=n
>  CONFIG_RTE_LIBRTE_CXGBE_PMD=n
> diff --git a/config/defconfig_ppc_64-power8-linuxapp-gcc b/config/defconfig_ppc_64-power8-linuxapp-gcc
> index 35f7fb6..89bc396 100644
> --- a/config/defconfig_ppc_64-power8-linuxapp-gcc
> +++ b/config/defconfig_ppc_64-power8-linuxapp-gcc
> @@ -48,6 +48,7 @@ CONFIG_RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT=n
>  
>  # Note: Initially, all of the PMD drivers compilation are turned off on Power
>  # Will turn on them only after the successful testing on Power
> +CONFIG_RTE_LIBRTE_ARK_PMD=n

Is it not tested or known that it is not supported?

>  CONFIG_RTE_LIBRTE_IXGBE_PMD=n
>  CONFIG_RTE_LIBRTE_I40E_PMD=n
>  CONFIG_RTE_LIBRTE_VIRTIO_PMD=y
> diff --git a/doc/guides/nics/ark.rst b/doc/guides/nics/ark.rst
> new file mode 100644
> index 0000000..72fb8d6
> --- /dev/null
> +++ b/doc/guides/nics/ark.rst
> @@ -0,0 +1,237 @@
> +.. BSD LICENSE
> +
> +    Copyright (c) 2015-2017 Atomic Rules LLC
> +    All rights reserved.
> +
> +    Redistribution and use in source and binary forms, with or without
> +    modification, are permitted provided that the following conditions
> +    are met:
> +
> +    * Redistributions of source code must retain the above copyright
> +    notice, this list of conditions and the following disclaimer.
> +    * Redistributions in binary form must reproduce the above copyright
> +    notice, this list of conditions and the following disclaimer in
> +    the documentation and/or other materials provided with the
> +    distribution.
> +    * Neither the name of Atomic Rules LLC nor the names of its
> +    contributors may be used to endorse or promote products derived
> +    from this software without specific prior written permission.
> +
> +    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +ARK Poll Mode Driver
> +====================
> +
> +The ARK PMD is a DPDK poll-mode driver for the Atomic Rules Arkville
> +(ARK) family of devices.
> +
> +More information can be found at the `Atomic Rules website
> +<http://atomicrules.com>`_.
> +
> +Overview
> +--------
> +
> +The Atomic Rules Arkville product is DPDK and AXI compliant product
> +that marshals packets across a PCIe conduit between host DPDK mbufs and
> +FPGA AXI streams.
> +
> +The ARK PMD, and the spirit of the overall Arkville product,
> +has been to take the DPDK API/ABI as a fixed specification;
> +then implement much of the business logic in FPGA RTL circuits.
> +The approach of *working backwards* from the DPDK API/ABI and having
> +the GPP host software *dictate*, while the FPGA hardware *copes*,
> +results in significant performance gains over a naive implementation.
> +
> +While this document describes the ARK PMD software, it is helpful to
> +understand what the FPGA hardware is and is not. The Arkville RTL
> +component provides a single PCIe Physical Function (PF) supporting
> +some number of RX/Ingress and TX/Egress Queues. The ARK PMD controls
> +the Arkville core through a dedicated opaque Core BAR (CBAR).
> +To allow users full freedom for their own FPGA application IP,
> +an independent FPGA Application BAR (ABAR) is provided.
> +
> +One popular way to imagine Arkville's FPGA hardware aspect is as the
> +FPGA PCIe-facing side of a so-called Smart NIC. The Arkville core does
> +not contain any MACs, and is link-speed independent, as well as
> +agnostic to the number of physical ports the application chooses to
> +use. The ARK driver exposes the familiar PMD interface to allow packet
> +movement to and from mbufs across multiple queues.
> +
> +However FPGA RTL applications could contain a universe of added
> +functionality that an Arkville RTL core does not provide or can
> +not anticipate. To allow for this expectation of user-defined
> +innovation, the ARK PMD provides a dynamic mechanism of adding
> +capabilities without having to modify the ARK PMD.
> +
> +The ARK PMD is intended to support all instances of the Arkville
> +RTL Core, regardless of configuration, FPGA vendor, or target
> +board. While specific capabilities such as number of physical
> +hardware queue-pairs are negotiated; the driver is designed to
> +remain constant over a broad and extendable feature set.
> +
> +Intentionally, Arkville by itself DOES NOT provide common NIC
> +capabilities such as offload or receive-side scaling (RSS).
> +These capabilities would be viewed as a gate-level "tax" on
> +Green-box FPGA applications that do not require such function.
> +Instead, they can be added as needed with essentially no
> +overhead to the FPGA Application.
> +
> +Data Path Interface
> +-------------------
> +
> +Ingress RX and Egress TX operation is by the nominal DPDK API .
> +The driver supports single-port, multi-queue for both RX and TX.
> +
> +Refer to ``ark_ethdev.h`` for the list of supported methods to
> +act upon RX and TX Queues.
> +
> +Configuration Information
> +-------------------------
> +
> +**DPDK Configuration Parameters**
> +
> +  The following configuration options are available for the ARK PMD:
> +
> +   * **CONFIG_RTE_LIBRTE_ARK_PMD** (default y): Enables or disables inclusion
> +     of the ARK PMD driver in the DPDK compilation.

Missing "CONFIG_RTE_LIBRTE_ARK_PAD_TX"

> +
> +   * **CONFIG_RTE_LIBRTE_ARK_DEBUG_RX** (default n): Enables or disables debug
> +     logging and internal checking of RX ingress logic within the ARK PMD driver.
> +
> +   * **CONFIG_RTE_LIBRTE_ARK_DEBUG_TX** (default n): Enables or disables debug
> +     logging and internal checking of TX egress logic within the ARK PMD driver.
> +
> +   * **CONFIG_RTE_LIBRTE_ARK_DEBUG_STATS** (default n): Enables or disables debug
> +     logging of detailed packet and performance statistics gathered in
> +     the PMD and FPGA.
> +
> +   * **CONFIG_RTE_LIBRTE_ARK_DEBUG_TRACE** (default n): Enables or disables debug
> +     logging of detailed PMD events and status.
> +
> +

Can you also please document the device arguments in this file?

> +Building DPDK
> +-------------
> +
> +See the :ref:`DPDK Getting Started Guide for Linux <linux_gsg>` for
> +instructions on how to build DPDK.
> +
> +By default the ARK PMD library will be built into the DPDK library.
> +
> +For configuring and using UIO and VFIO frameworks, please also refer :ref:`the
> +documentation that comes with DPDK suite <linux_gsg>`.
> +
> +Supported ARK RTL PCIe Instances
> +--------------------------------
> +
> +ARK PMD supports the following Arkville RTL PCIe instances including:
> +
> +* ``1d6c:100d`` - AR-ARKA-FX0 [Arkville 32B DPDK Data Mover]
> +* ``1d6c:100e`` - AR-ARKA-FX1 [Arkville 64B DPDK Data Mover]
> +
> +Supported Operating Systems
> +---------------------------
> +
> +Any Linux distribution fulfilling the conditions described in ``System Requirements``
> +section of :ref:`the DPDK documentation <linux_gsg>` or refer to *DPDK Release Notes*.
> +
> +Supported Features
> +------------------
> +
> +* Dynamic ARK PMD extensions
> +* Multiple receive and transmit queues
> +* Jumbo frames up to 9K
> +* Hardware Statistics
> +
> +Unsupported Features
> +--------------------
> +
> +Features that may be part of, or become part of, the Arkville RTL IP that are
> +not currently supported or exposed by the ARK PMD include:
> +
> +* PCIe SR-IOV Virtual Functions (VFs)
> +* Arkville's Packet Generator Control and Status
> +* Arkville's Packet Director Control and Status
> +* Arkville's Packet Checker Control and Status
> +* Arkville's Timebase Management
> +
> +Pre-Requisites
> +--------------
> +
> +#. Prepare the system as recommended by DPDK suite.  This includes environment
> +   variables, hugepages configuration, tool-chains and configuration
> +
> +#. Insert igb_uio kernel module using the command 'modprobe igb_uio'
> +
> +#. Bind the intended ARK device to igb_uio module
> +
> +At this point the system should be ready to run DPDK applications. Once the
> +application runs to completion, the ARK PMD can be detached from igb_uio if necessary.
> +
> +Usage Example
> +-------------
> +
> +This section demonstrates how to launch **testpmd** with Atomic Rules ARK
> +devices managed by librte_pmd_ark.
> +
> +#. Load the kernel modules:
> +
> +   .. code-block:: console
> +
> +      modprobe uio
> +      insmod ./x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
> +
> +   .. note::
> +
> +      The ARK PMD driver depends upon the igb_uio user space I/O kernel module
> +
> +#. Mount and request huge pages:
> +
> +   .. code-block:: console
> +
> +      mount -t hugetlbfs nodev /mnt/huge
> +      echo 256 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
> +
> +#. Bind UIO driver to ARK device at 0000:01:00.0 (using dpdk-devbind.py):
> +
> +   .. code-block:: console
> +
> +      ./usertools/dpdk-devbind.py --bind=igb_uio 0000:01:00.0
> +
> +   .. note::
> +
> +      The last argument to dpdk-devbind.py is the 4-tuple that indentifies a specific PCIe
> +      device. You can use lspci -d 1d6c: to indentify all Atomic Rules devices in the system,
> +      and thus determine the correct 4-tuple argument to dpdk-devbind.py
> +
> +#. Start testpmd with basic parameters:
> +
> +   .. code-block:: console
> +
> +      ./x86_64-native-linuxapp-gcc/app/testpmd -l 0-3 -n 4 -- -i
> +
> +   Example output:
> +
> +   .. code-block:: console
> +
> +      [...]
> +      EAL: PCI device 0000:01:00.0 on NUMA socket -1
> +      EAL:   probe driver: 1d6c:100e rte_ark_pmd
> +      EAL:   PCI memory mapped at 0x7f9b6c400000
> +      PMD: eth_ark_dev_init(): Initializing 0:2:0.1
> +      ARKP PMD CommitID: 378f3a67
> +      Configuring Port 0 (socket 0)
> +      Port 0: DC:3C:F6:00:00:01
> +      Checking link statuses...
> +      Port 0 Link Up - speed 100000 Mbps - full-duplex
> +      Done
> +      testpmd>
> diff --git a/doc/guides/nics/features/ark.ini b/doc/guides/nics/features/ark.ini
> new file mode 100644
> index 0000000..dc8a0e2
> --- /dev/null
> +++ b/doc/guides/nics/features/ark.ini
> @@ -0,0 +1,15 @@
> +;
> +; Supported features of the 'ark' poll mode driver.
> +;
> +; Refer to default.ini for the full list of available PMD features.
> +;
> +[Features]
> +Queue start/stop     = Y
> +Jumbo frame          = Y
> +Scattered Rx         = Y
> +Basic stats          = Y
> +Stats per queue      = Y
> +FW version           = Y

Features can be added with the patch that adds functionality. I believe
above features not supported with current patch.

> +Linux UIO            = Y
> +x86-64               = Y
> +Usage doc            = Y
> diff --git a/doc/guides/nics/index.rst b/doc/guides/nics/index.rst
> index 87f9334..381d82c 100644
> --- a/doc/guides/nics/index.rst
> +++ b/doc/guides/nics/index.rst
> @@ -36,6 +36,7 @@ Network Interface Controller Drivers
>      :numbered:
>  
>      overview
> +    ark
>      bnx2x
>      bnxt
>      cxgbe
> diff --git a/drivers/net/Makefile b/drivers/net/Makefile
> index a16f25e..ea9868b 100644
> --- a/drivers/net/Makefile
> +++ b/drivers/net/Makefile
> @@ -32,6 +32,7 @@
>  include $(RTE_SDK)/mk/rte.vars.mk
>  
>  DIRS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += af_packet
> +DIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark
>  DIRS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += bnx2x
>  DIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += bonding
>  DIRS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe
> diff --git a/drivers/net/ark/Makefile b/drivers/net/ark/Makefile
> new file mode 100644
> index 0000000..217bd34
> --- /dev/null
> +++ b/drivers/net/ark/Makefile
> @@ -0,0 +1,63 @@
> +# BSD LICENSE
> +#
> +# Copyright (c) 2015-2017 Atomic Rules LLC
> +# All rights reserved.
> +#
> +# Redistribution and use in source and binary forms, with or without
> +# modification, are permitted provided that the following conditions
> +# are met:
> +#
> +# * Redistributions of source code must retain the above copyright
> +#   notice, this list of conditions and the following disclaimer.
> +# * Redistributions in binary form must reproduce the above copyright
> +#   notice, this list of conditions and the following disclaimer in
> +#   the documentation and/or other materials provided with the
> +#   distribution.
> +# * Neither the name of copyright holder nor the names of its
> +#   contributors may be used to endorse or promote products derived
> +#   from this software without specific prior written permission.
> +#
> +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_pmd_ark.a
> +
> +CFLAGS += -O3 -I./
> +CFLAGS += $(WERROR_FLAGS)
> +
> +EXPORT_MAP := rte_pmd_ark_version.map
> +
> +LIBABIVER := 1
> +
> +#
> +# all source are stored in SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD)

No need to put config option to comment, SRCS-y looks more proper.

> +#
> +
> +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_ethdev.c
> +
> +
> +# this lib depends upon:
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += lib/librte_mbuf
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += lib/librte_ether
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += lib/librte_kvargs
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += lib/librte_eal
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += lib/librte_mempool

> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += lib/libpthread
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += lib/libdl

DEPDIRS is for internal library dependencies. Please use LDLIBS for
external dependencies, like:

LDLIBS += -lpthread
LDLIBS += -ldl

> +
> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/net/ark/ark_debug.h b/drivers/net/ark/ark_debug.h
> new file mode 100644
> index 0000000..a108c28
> --- /dev/null
> +++ b/drivers/net/ark/ark_debug.h
> @@ -0,0 +1,74 @@
> +/*-
> + * BSD LICENSE
> + *
> + * Copyright (c) 2015-2017 Atomic Rules LLC
> + * All rights reserved.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + *
> + * * Redistributions of source code must retain the above copyright
> + * notice, this list of conditions and the following disclaimer.
> + * * Redistributions in binary form must reproduce the above copyright
> + * notice, this list of conditions and the following disclaimer in
> + * the documentation and/or other materials provided with the
> + * distribution.
> + * * Neither the name of copyright holder nor the names of its
> + * contributors may be used to endorse or promote products derived
> + * from this software without specific prior written permission.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef _ARK_DEBUG_H_
> +#define _ARK_DEBUG_H_
> +
> +#include <inttypes.h>
> +#include <rte_log.h>
> +
> +/* Format specifiers for string data pairs */
> +#define ARK_SU32  "\n\t%-20s    %'20" PRIu32
> +#define ARK_SU64  "\n\t%-20s    %'20" PRIu64
> +#define ARK_SU64X "\n\t%-20s    %#20" PRIx64
> +#define ARK_SPTR  "\n\t%-20s    %20p"
> +
> +#define ARK_TRACE_ON(fmt, ...) \
> +	PMD_DRV_LOG(ERR, fmt, ##__VA_ARGS__)
> +
> +#define ARK_TRACE_OFF(fmt, ...) \
> +	do {if (0) PMD_DRV_LOG(ERR, fmt, ##__VA_ARGS__); } while (0)
> +
> +/* Debug macro for reporting Packet stats */
> +#ifdef RTE_LIBRTE_ARK_DEBUG_STATS
> +#define ARK_DEBUG_STATS(fmt, ...) ARK_TRACE_ON(fmt, ##__VA_ARGS__)
> +#else
> +#define ARK_DEBUG_STATS(fmt, ...)  ARK_TRACE_OFF(fmt, ##__VA_ARGS__)
> +#endif
> +
> +/* Debug macro for tracing full behavior*/
> +#ifdef RTE_LIBRTE_ARK_DEBUG_TRACE
> +#define ARK_DEBUG_TRACE(fmt, ...)  ARK_TRACE_ON(fmt, ##__VA_ARGS__)
> +#else
> +#define ARK_DEBUG_TRACE(fmt, ...)  ARK_TRACE_OFF(fmt, ##__VA_ARGS__)
> +#endif
> +
> +#ifdef ARK_STD_LOG

How this define passed? Should it be something like
RTE_LIBRTE_ARK_DEBUG_DRIVER config option?

> +#define PMD_DRV_LOG(level, fmt, args...) \
> +	fprintf(stderr, fmt, args)

It is possible to use rte log functions instead of fprintf to stderr.

> +#else
> +#define PMD_DRV_LOG(level, fmt, args...) \
> +	RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args)
> +#endif
> +
> +#endif

CONFIG_RTE_LIBRTE_ARK_DEBUG_RX, CONFIG_RTE_LIBRTE_ARK_DEBUG_TX not used,
if so can be removed.

> diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
> new file mode 100644
> index 0000000..124b73c
> --- /dev/null
> +++ b/drivers/net/ark/ark_ethdev.c
> @@ -0,0 +1,281 @@
> +/*-
> + * BSD LICENSE
> + *
> + * Copyright (c) 2015-2017 Atomic Rules LLC
> + * All rights reserved.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + *
> + * * Redistributions of source code must retain the above copyright
> + * notice, this list of conditions and the following disclaimer.
> + * * Redistributions in binary form must reproduce the above copyright
> + * notice, this list of conditions and the following disclaimer in
> + * the documentation and/or other materials provided with the
> + * distribution.
> + * * Neither the name of copyright holder nor the names of its
> + * contributors may be used to endorse or promote products derived
> + * from this software without specific prior written permission.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#include <unistd.h>
> +#include <sys/stat.h>
> +#include <dlfcn.h>
> +
> +#include <rte_kvargs.h>
> +#include <rte_vdev.h>
> +
> +#include "ark_global.h"
> +#include "ark_debug.h"
> +#include "ark_ethdev.h"
> +
> +/*  Internal prototypes */
> +static int eth_ark_check_args(const char *params);
> +static int eth_ark_dev_init(struct rte_eth_dev *dev);
> +static int eth_ark_dev_uninit(struct rte_eth_dev *eth_dev);
> +static int eth_ark_dev_configure(struct rte_eth_dev *dev);
> +static void eth_ark_dev_info_get(struct rte_eth_dev *dev,
> +				 struct rte_eth_dev_info *dev_info);
> +
> +
> +#define ARK_DEV_TO_PCI(eth_dev)			\
> +	RTE_DEV_TO_PCI((eth_dev)->device)
> +
> +#define ARK_MAX_ARG_LEN 256
> +static uint32_t pkt_dir_v;
> +static char pkt_gen_args[ARK_MAX_ARG_LEN];
> +static char pkt_chkr_args[ARK_MAX_ARG_LEN];
> +
> +#define ARK_PKTGEN_ARG "Pkt_gen"
> +#define ARK_PKTCHKR_ARG "Pkt_chkr"
> +#define ARK_PKTDIR_ARG "Pkt_dir"

Is it possible to add one line comments to device arguments. For example
what "Pkt_dir" (packet director) is for?

> +
> +static const char * const valid_arguments[] = {
> +	ARK_PKTGEN_ARG,
> +	ARK_PKTCHKR_ARG,
> +	ARK_PKTDIR_ARG,
> +	"iface",

Why not make this one too a define?

> +	NULL
> +};
> +
> +#define MAX_ARK_PHYS 16
> +struct ark_adapter *gark[MAX_ARK_PHYS];
> +
> +static const struct rte_pci_id pci_id_ark_map[] = {
> +	{RTE_PCI_DEVICE(0x1d6c, 0x100d)},
> +	{RTE_PCI_DEVICE(0x1d6c, 0x100e)},
> +	{.vendor_id = 0, /* sentinel */ },
> +};
> +
> +static struct eth_driver rte_ark_pmd = {
> +	.pci_drv = {
> +		.probe = rte_eth_dev_pci_probe,
> +		.remove = rte_eth_dev_pci_remove,
> +		.id_table = pci_id_ark_map,
> +		.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC
> +	},
> +	.eth_dev_init = eth_ark_dev_init,
> +	.eth_dev_uninit = eth_ark_dev_uninit,
> +	.dev_private_size = sizeof(struct ark_adapter),
> +};
> +
> +static const struct eth_dev_ops ark_eth_dev_ops = {
> +	.dev_configure = eth_ark_dev_configure,
> +	.dev_infos_get = eth_ark_dev_info_get,
> +

Extra line.

> +};
> +
> +

Extra line.

> +static int
> +eth_ark_dev_init(struct rte_eth_dev *dev __rte_unused)
> +{
> +	return -1;					/* STUB */

You may want to set ark_eth_dev_ops here, since they already implemented.

And for proper .dev_infos_get implementation, you may want to have [1] here:

[1]
rte_eth_copy_pci_info(eth_dev, pci_dev);
eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;

Also you may want to parse device arguments in this stage.

> +}
> +
> +
> +static int
> +eth_ark_dev_uninit(struct rte_eth_dev *dev)
> +{
> +	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> +		return 0;
> +
> +	dev->dev_ops = NULL;
> +	dev->rx_pkt_burst = NULL;
> +	dev->tx_pkt_burst = NULL;
> +	return 0;
> +}
> +
> +static int
> +eth_ark_dev_configure(struct rte_eth_dev *dev __rte_unused)
> +{
> +	ARK_DEBUG_TRACE("ARKP: In %s\n", __func__);
> +	return 0;
> +}
> +
> +static void
> +eth_ark_dev_info_get(struct rte_eth_dev *dev,
> +		     struct rte_eth_dev_info *dev_info)
> +{
> +	/* device specific configuration */
> +	memset(dev_info, 0, sizeof(*dev_info));

memset not required, since already done by ethdev abstraction layer,
specially desc_lim values already overwritten below.

> +
> +	dev_info->max_rx_pktlen = (16 * 1024) - 128;
> +	dev_info->min_rx_bufsize = 1024;

Using some macros instead of hardcoded values helps to understand values.

> +	dev_info->rx_offload_capa = 0;
> +	dev_info->tx_offload_capa = 0;
> +
> +	dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
> +		.nb_max = 4096 * 4,
> +		.nb_min = 512,	/* HW Q size for RX */
> +		.nb_align = 2,};
> +
> +	dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
> +		.nb_max = 4096 * 4,
> +		.nb_min = 256,	/* HW Q size for TX */
> +		.nb_align = 2,};
> +
> +	dev_info->rx_offload_capa = 0;
> +	dev_info->tx_offload_capa = 0;

Dublication, please check ~10 lines above.
Also not required to set 0 at all because of memset.

> +
> +	/* ARK PMD supports all line rates, how do we indicate that here ?? */
> +	dev_info->speed_capa = (ETH_LINK_SPEED_1G |
> +				ETH_LINK_SPEED_10G |
> +				ETH_LINK_SPEED_25G |
> +				ETH_LINK_SPEED_40G |
> +				ETH_LINK_SPEED_50G |
> +				ETH_LINK_SPEED_100G);
> +	dev_info->pci_dev = ARK_DEV_TO_PCI(dev);
> +	dev_info->driver_name = dev->data->drv_name;

setting driver_name not required, ethdev layer will overwrite this value.

And to have driver_name correct, rte_eth_copy_pci_info() should be
called, please check above [1].

> +}
> +
> +
> +static inline int
> +process_pktdir_arg(const char *key, const char *value,
> +		   void *extra_args __rte_unused)
> +{
> +	ARK_DEBUG_TRACE("In process_pktdir_arg, key = %s, value = %s\n",
> +			key, value);

The general usage of DEBUG_TRACE is providing backtrace log, function
enterance / exit informations. I guess, that is why it has been
controlled by different config option.
Here what you need looks like regular debugging functions, PMD_DRV_LOG /
RTE_LOG variant.

> +	pkt_dir_v = strtol(value, NULL, 16);
> +	ARK_DEBUG_TRACE("pkt_dir_v = 0x%x\n", pkt_dir_v);
> +	return 0;
> +}
> +
> +static inline int
> +process_file_args(const char *key, const char *value, void *extra_args)
> +{
> +	ARK_DEBUG_TRACE("**** IN process_pktgen_arg, key = %s, value = %s\n",
> +			key, value);
> +	char *args = (char *)extra_args;
> +
> +	/* Open the configuration file */
> +	FILE *file = fopen(value, "r");
> +	char line[256];
> +	int first = 1;
> +
> +	while (fgets(line, sizeof(line), file)) {
> +		/* ARK_DEBUG_TRACE("%s\n", line); */

Please remove dead code.

> +		if (first) {
> +			strncpy(args, line, ARK_MAX_ARG_LEN);

Can this overflow args variable? Any way to prevent possible crash?

> +			first = 0;
> +		} else {
> +			strncat(args, line, ARK_MAX_ARG_LEN);
> +		}
> +	}
> +	ARK_DEBUG_TRACE("file = %s\n", args);
> +	fclose(file);
> +	return 0;
> +}
> +
> +static int
> +eth_ark_check_args(const char *params)
> +{
> +	struct rte_kvargs *kvlist;
> +	unsigned int k_idx;
> +	struct rte_kvargs_pair *pair = NULL;
> +
> +	kvlist = rte_kvargs_parse(params, valid_arguments);
> +	if (kvlist == NULL)
> +		return 0;
> +
> +	pkt_gen_args[0] = 0;
> +	pkt_chkr_args[0] = 0;
> +
> +	for (k_idx = 0; k_idx < kvlist->count; k_idx++) {
> +		pair = &kvlist->pairs[k_idx];
> +		ARK_DEBUG_TRACE("**** Arg passed to PMD = %s:%s\n", pair->key,
> +				pair->value);
> +	}
> +
> +	if (rte_kvargs_process(kvlist,
> +			       ARK_PKTDIR_ARG,
> +			       &process_pktdir_arg,
> +			       NULL) != 0) {
> +		PMD_DRV_LOG(ERR, "Unable to parse arg %s\n", ARK_PKTDIR_ARG);
> +	}
> +
> +	if (rte_kvargs_process(kvlist,
> +			       ARK_PKTGEN_ARG,
> +			       &process_file_args,
> +			       pkt_gen_args) != 0) {
> +		PMD_DRV_LOG(ERR, "Unable to parse arg %s\n", ARK_PKTGEN_ARG);
> +	}
> +
> +	if (rte_kvargs_process(kvlist,
> +			       ARK_PKTCHKR_ARG,
> +			       &process_file_args,
> +			       pkt_chkr_args) != 0) {
> +		PMD_DRV_LOG(ERR, "Unable to parse arg %s\n", ARK_PKTCHKR_ARG);
> +	}

Not processing "iface" device argument?

> +
> +	ARK_DEBUG_TRACE("INFO: packet director set to 0x%x\n", pkt_dir_v);
> +
> +	return 1;
> +}
> +
> +
> +/*
> + * PCIE

Can you elaborate this comment?

> + */
> +static int
> +pmd_ark_probe(const char *name, const char *params)
> +{
> +	RTE_LOG(INFO, PMD, "Initializing pmd_ark for %s params = %s\n", name,
> +		params);
> +
> +	/* Parse off the v index */
> +
> +	eth_ark_check_args(params);
> +	return 0;
> +}
> +
> +static int
> +pmd_ark_remove(const char *name)
> +{
> +	RTE_LOG(INFO, PMD, "Closing ark %s ethdev on numa socket %u\n", name,
> +		rte_socket_id());
> +	return 1;
> +}
> +
> +static struct rte_vdev_driver pmd_ark_drv = {
> +	.probe = pmd_ark_probe,
> +	.remove = pmd_ark_remove,
> +};

Sorry, I am confused here.
Why both virtual and physical initialization routine exists together?
This PMD for physical PCI device, right?

> +
> +RTE_PMD_REGISTER_VDEV(net_ark, pmd_ark_drv);
> +RTE_PMD_REGISTER_ALIAS(net_ark, eth_ark);
> +RTE_PMD_REGISTER_PCI(eth_ark, rte_ark_pmd.pci_drv);
> +RTE_PMD_REGISTER_KMOD_DEP(net_ark, "* igb_uio | uio_pci_generic ");
> +RTE_PMD_REGISTER_PCI_TABLE(eth_ark, pci_id_ark_map);

Can add RTE_PMD_REGISTER_PARAM_STRING macro.

> diff --git a/drivers/net/ark/ark_ethdev.h b/drivers/net/ark/ark_ethdev.h
> new file mode 100644
> index 0000000..08d7fb1
> --- /dev/null
> +++ b/drivers/net/ark/ark_ethdev.h
> @@ -0,0 +1,39 @@
> +/*-
> + * BSD LICENSE
> + *
> + * Copyright (c) 2015-2017 Atomic Rules LLC
> + * All rights reserved.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + *
> + * * Redistributions of source code must retain the above copyright
> + * notice, this list of conditions and the following disclaimer.
> + * * Redistributions in binary form must reproduce the above copyright
> + * notice, this list of conditions and the following disclaimer in
> + * the documentation and/or other materials provided with the
> + * distribution.
> + * * Neither the name of copyright holder nor the names of its
> + * contributors may be used to endorse or promote products derived
> + * from this software without specific prior written permission.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef _ARK_ETHDEV_H_
> +#define _ARK_ETHDEV_H_
> +
> +/* STUB */
> +
> +#endif
> diff --git a/drivers/net/ark/ark_global.h b/drivers/net/ark/ark_global.h
> new file mode 100644
> index 0000000..7cd62d5
> --- /dev/null
> +++ b/drivers/net/ark/ark_global.h
> @@ -0,0 +1,108 @@
> +/*-
> + * BSD LICENSE
> + *
> + * Copyright (c) 2015-2017 Atomic Rules LLC
> + * All rights reserved.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + *
> + * * Redistributions of source code must retain the above copyright
> + * notice, this list of conditions and the following disclaimer.
> + * * Redistributions in binary form must reproduce the above copyright
> + * notice, this list of conditions and the following disclaimer in
> + * the documentation and/or other materials provided with the
> + * distribution.
> + * * Neither the name of copyright holder nor the names of its
> + * contributors may be used to endorse or promote products derived
> + * from this software without specific prior written permission.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef _ARK_GLOBAL_H_
> +#define _ARK_GLOBAL_H_
> +
> +#include <time.h>
> +#include <assert.h>
> +
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +#include <rte_malloc.h>
> +#include <rte_memcpy.h>
> +#include <rte_string_fns.h>
> +#include <rte_cycles.h>
> +#include <rte_kvargs.h>
> +#include <rte_dev.h>
> +#include <rte_version.h>
> +
> +#define ETH_ARK_ARG_MAXLEN	64
> +#define ARK_SYSCTRL_BASE  0x0
> +#define ARK_PKTGEN_BASE   0x10000
> +#define ARK_MPU_RX_BASE   0x20000
> +#define ARK_UDM_BASE      0x30000
> +#define ARK_MPU_TX_BASE   0x40000
> +#define ARK_DDM_BASE      0x60000
> +#define ARK_CMAC_BASE     0x80000
> +#define ARK_PKTDIR_BASE   0xa0000
> +#define ARK_PKTCHKR_BASE  0x90000
> +#define ARK_RCPACING_BASE 0xb0000
> +#define ARK_EXTERNAL_BASE 0x100000
> +#define ARK_MPU_QOFFSET   0x00100
> +#define ARK_MAX_PORTS     8
> +
> +#define offset8(n)     n
> +#define offset16(n)   ((n) / 2)
> +#define offset32(n)   ((n) / 4)
> +#define offset64(n)   ((n) / 8)
> +
> +/*
> + * Structure to store private data for each PF/VF instance.
> + */
> +#define def_ptr(type, name) \
> +	union type {		   \
> +		uint64_t *t64;	   \
> +		uint32_t *t32;	   \
> +		uint16_t *t16;	   \
> +		uint8_t  *t8;	   \
> +		void     *v;	   \
> +	} name
> +
> +struct ark_port {
> +	struct rte_eth_dev *eth_dev;
> +	int id;
> +};
> +
> +struct ark_adapter {
> +	/* User extension private data */
> +	void *user_data;
> +
> +	struct ark_port port[ARK_MAX_PORTS];
> +	int num_ports;
> +
> +	/* Common for both PF and VF */
> +	struct rte_eth_dev *eth_dev;
> +
> +	void *d_handle;
> +
> +	/* Our Bar 0 */
> +	uint8_t *bar0;
> +
> +	/* Application Bar */
> +	uint8_t *a_bar;
> +};
> +
> +typedef uint32_t *ark_t;
> +
> +#endif
> diff --git a/drivers/net/ark/rte_pmd_ark_version.map b/drivers/net/ark/rte_pmd_ark_version.map
> new file mode 100644
> index 0000000..7f84780
> --- /dev/null
> +++ b/drivers/net/ark/rte_pmd_ark_version.map
> @@ -0,0 +1,4 @@
> +DPDK_2.0 {

DPDK_17.05

> +	 local: *;
> +
> +};
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk
> index 0e0b600..da23898 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -104,6 +104,7 @@ ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),n)
>  # plugins (link only if static libraries)
>  
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET)  += -lrte_pmd_af_packet
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_ARK_PMD)        += -lrte_pmd_ark
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD)      += -lrte_pmd_bnx2x -lz
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_BNXT_PMD)       += -lrte_pmd_bnxt
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_BOND)       += -lrte_pmd_bond
> 



More information about the dev mailing list