[dpdk-dev] [PATCH v2 4/4] DPDK polling-mode driver for Amazon Elastic Network Adapters (ENA)
Jan Mędala
jan at semihalf.com
Mon Feb 15 10:25:09 CET 2016
ping for comments/review
Thanks,
Jan
2016-02-05 19:20 GMT+01:00 Jan Medala <jan at semihalf.com>:
> This is a PMD for the Amazon ethernet ENA family.
> The driver operates variety of ENA adapters through feature negotiation
> with the adapter and upgradable commands set.
> ENA driver handles PCI Physical and Virtual ENA functions.
>
> Signed-off-by: Evgeny Schemeilin <evgenys at amazon.com>
> Signed-off-by: Jan Medala <jan at semihalf.com>
> Signed-off-by: Jakub Palider <jpa at semihalf.com>
> ---
> config/common_linuxapp | 12 +
> drivers/net/Makefile | 1 +
> drivers/net/ena/Makefile | 62 +++
> drivers/net/ena/ena_ethdev.c | 1129
> ++++++++++++++++++++++++++++++++++++++++
> drivers/net/ena/ena_ethdev.h | 151 ++++++
> drivers/net/ena/ena_logs.h | 76 +++
> drivers/net/ena/ena_platform.h | 58 +++
> mk/rte.app.mk | 1 +
> 8 files changed, 1490 insertions(+)
> create mode 100755 drivers/net/ena/Makefile
> create mode 100644 drivers/net/ena/ena_ethdev.c
> create mode 100755 drivers/net/ena/ena_ethdev.h
> create mode 100644 drivers/net/ena/ena_logs.h
> create mode 100644 drivers/net/ena/ena_platform.h
>
> diff --git a/config/common_linuxapp b/config/common_linuxapp
> index 74bc515..261a54f 100644
> --- a/config/common_linuxapp
> +++ b/config/common_linuxapp
> @@ -249,6 +249,18 @@ CONFIG_RTE_LIBRTE_CXGBE_DEBUG_TX=n
> CONFIG_RTE_LIBRTE_CXGBE_DEBUG_RX=n
>
> #
> +# Compile burst-oriented Amazon ENA PMD driver
> +#
> +CONFIG_RTE_LIBRTE_ENA_PMD=y
> +CONFIG_RTE_LIBRTE_ENA_DEBUG_INIT=y
> +CONFIG_RTE_LIBRTE_ENA_DEBUG_RX=n
> +CONFIG_RTE_LIBRTE_ENA_DEBUG_TX=n
> +CONFIG_RTE_LIBRTE_ENA_DEBUG_TX_FREE=n
> +CONFIG_RTE_LIBRTE_ENA_DEBUG_DRIVER=n
> +CONFIG_RTE_LIBRTE_ENA_COM_DEBUG=n
> +CONFIG_RTE_EAL_ENA_UIO=y
> +
> +#
> # Compile burst-oriented Cisco ENIC PMD driver
> #
> CONFIG_RTE_LIBRTE_ENIC_PMD=y
> diff --git a/drivers/net/Makefile b/drivers/net/Makefile
> index 6e4497e..8f2649f 100644
> --- a/drivers/net/Makefile
> +++ b/drivers/net/Makefile
> @@ -36,6 +36,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += bnx2x
> DIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += bonding
> DIRS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe
> DIRS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += e1000
> +DIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena
> DIRS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic
> DIRS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += fm10k
> DIRS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e
> diff --git a/drivers/net/ena/Makefile b/drivers/net/ena/Makefile
> new file mode 100755
> index 0000000..960e4cd
> --- /dev/null
> +++ b/drivers/net/ena/Makefile
> @@ -0,0 +1,62 @@
> +#
> +# BSD LICENSE
> +#
> +# Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates.
> +# All rights reserved.
> +#
> +# Redistribution and use in source and binary forms, with or without
> +# modification, are permitted provided that the following conditions
> +# are met:
> +#
> +# * Redistributions of source code must retain the above copyright
> +# notice, this list of conditions and the following disclaimer.
> +# * Redistributions in binary form must reproduce the above copyright
> +# notice, this list of conditions and the following disclaimer in
> +# the documentation and/or other materials provided with the
> +# distribution.
> +# * Neither the name of copyright holder nor the names of its
> +# contributors may be used to endorse or promote products derived
> +# from this software without specific prior written permission.
> +#
> +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +#
> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_pmd_ena.a
> +CFLAGS += $(WERROR_FLAGS) -O2
> +INCLUDES :=-I$(SRCDIR) -I$(SRCDIR)/base/ena_defs -I$(SRCDIR)/base
> +
> +VPATH += $(SRCDIR)/base
> +#
> +# all source are stored in SRCS-y
> +#
> +SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_ethdev.c
> +SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_com.c
> +SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_eth_com.c
> +
> +# this lib depends upon:
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += lib/librte_eal lib/librte_ether
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += lib/librte_mempool lib/librte_mbuf
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += lib/librte_net lib/librte_malloc
> +
> +ifeq ($(CONFIG_RTE_EXEC_ENV),"cvos")
> +CFLAGS += -Wno-old-style-definition
> +endif
> +
> +CFLAGS += $(INCLUDES)
> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
> new file mode 100644
> index 0000000..bfa212d
> --- /dev/null
> +++ b/drivers/net/ena/ena_ethdev.c
> @@ -0,0 +1,1129 @@
> +/*-
> +* BSD LICENSE
> +*
> +* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates.
> +* All rights reserved.
> +*
> +* Redistribution and use in source and binary forms, with or without
> +* modification, are permitted provided that the following conditions
> +* are met:
> +*
> +* * Redistributions of source code must retain the above copyright
> +* notice, this list of conditions and the following disclaimer.
> +* * Redistributions in binary form must reproduce the above copyright
> +* notice, this list of conditions and the following disclaimer in
> +* the documentation and/or other materials provided with the
> +* distribution.
> +* * Neither the name of copyright holder nor the names of its
> +* contributors may be used to endorse or promote products derived
> +* from this software without specific prior written permission.
> +*
> +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +*/
> +
> +#include <rte_ether.h>
> +#include <rte_ethdev.h>
> +#include <rte_tcp.h>
> +#include <rte_atomic.h>
> +#include <rte_dev.h>
> +#include <rte_errno.h>
> +
> +#include "ena_ethdev.h"
> +#include "ena_logs.h"
> +#include "ena_platform.h"
> +#include "ena_com.h"
> +#include "ena_eth_com.h"
> +
> +#include <ena_common_defs.h>
> +#include <ena_regs_defs.h>
> +#include <ena_admin_defs.h>
> +#include <ena_eth_io_defs.h>
> +
> +#define ENA_IO_TXQ_IDX(q) (2 * (q))
> +#define ENA_IO_RXQ_IDX(q) (2 * (q) + 1)
> +
> +#define ENA_MAX_MTU (ENA_MAX_FRAME_LEN - ETHER_HDR_LEN -
> ETHER_CRC_LEN)
> +
> +/* While processing submitted and completed desciptors (rx and tx path
> + * respectively) in a loop it is desired to:
> + * - perform batch submissions while populating sumbissmion queue
> + * - avoid blocking transmission of other packets during cleanup phase
> + * Hence the utilization ratio of 1/8 of a queue size.
> + */
> +#define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8)
> +
> +#define __MERGE_64B_H_L(h, l) (((long)h << 32) | l )
> +
> +static struct rte_pci_id pci_id_ena_map[] = {
> +#define RTE_PCI_DEV_ID_DECL_ENA(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
> +#include "rte_pci_dev_ids.h"
> + {.device_id = 0},
> +};
> +
> +static int ena_device_init(struct ena_com_dev *ena_dev,
> + struct ena_com_dev_get_features_ctx
> *get_feat_ctx);
> +static int ena_dev_configure(struct rte_eth_dev *dev);
> +static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf
> **tx_pkts,
> + uint16_t nb_pkts);
> +static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
> + uint16_t nb_desc, unsigned int socket_id,
> + const struct rte_eth_txconf *tx_conf);
> +static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
> + uint16_t nb_desc, unsigned int socket_id,
> + const struct rte_eth_rxconf *rx_conf, struct
> rte_mempool *mp);
> +static uint16_t eth_ena_recv_pkts(void *rx_queue,
> + struct rte_mbuf **rx_pkts, uint16_t
> nb_pkts);
> +static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int
> count);
> +static void ena_init_rings(struct ena_adapter *adapter);
> +static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
> +static int ena_start(struct rte_eth_dev *dev);
> +static void ena_close(struct rte_eth_dev *dev);
> +static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats
> *stats);
> +static void ena_rx_queue_release_all(struct rte_eth_dev *dev);
> +static void ena_tx_queue_release_all(struct rte_eth_dev *dev);
> +static void ena_rx_queue_release(void *queue);
> +static void ena_tx_queue_release(void *queue);
> +static void ena_rx_queue_release_bufs(struct ena_ring *ring);
> +static void ena_tx_queue_release_bufs(struct ena_ring *ring);
> +static int ena_link_update(struct rte_eth_dev *dev,
> + __attribute__((unused))int wait_to_complete);
> +static int ena_queue_restart(struct ena_ring *ring) ;
> +static int ena_queue_restart_all(struct rte_eth_dev *dev,
> + enum ena_ring_type ring_type);
> +static void ena_stats_restart(struct rte_eth_dev *dev);
> +static void ena_infos_get(__attribute__((unused))struct rte_eth_dev *dev,
> + struct rte_eth_dev_info *dev_info);
> +
> +static struct eth_dev_ops ena_dev_ops = {
> + .dev_configure = ena_dev_configure,
> + .dev_infos_get = ena_infos_get,
> + .rx_queue_setup = ena_rx_queue_setup,
> + .tx_queue_setup = ena_tx_queue_setup,
> + .dev_start = ena_start,
> + .link_update = ena_link_update,
> + .stats_get = ena_stats_get,
> + .mtu_set = ena_mtu_set,
> + .rx_queue_release = ena_rx_queue_release,
> + .tx_queue_release = ena_tx_queue_release,
> + .dev_close = ena_close,
> +};
> +
> +static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf,
> + struct ena_com_rx_ctx *ena_rx_ctx)
> +{
> + uint64_t ol_flags = 0;
> +
> + if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) {
> + ol_flags |= PKT_TX_TCP_CKSUM;
> + } else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) {
> + ol_flags |= PKT_TX_UDP_CKSUM;
> + }
> +
> + if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) {
> + ol_flags |= PKT_TX_IPV4;
> + } else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) {
> + ol_flags |= PKT_TX_IPV6;
> + }
> +
> + if (unlikely(ena_rx_ctx->l4_csum_err))
> + ol_flags |= PKT_RX_L4_CKSUM_BAD;
> + if (unlikely(ena_rx_ctx->l3_csum_err))
> + ol_flags |= PKT_RX_IP_CKSUM_BAD;
> +
> + mbuf->ol_flags = ol_flags;
> +
> + return;
> +}
> +
> +static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf,
> + struct ena_com_tx_ctx *ena_tx_ctx)
> +{
> + struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
> +
> + if (mbuf->ol_flags & (PKT_TX_L4_MASK || PKT_TX_IP_CKSUM ||
> + PKT_TX_TCP_SEG)) {
> +
> + /* check if TSO is required */
> + if (mbuf->ol_flags & PKT_TX_TCP_SEG) {
> + ena_tx_ctx->tso_enable = true;
> +
> + ena_meta->l4_hdr_len =
> ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr*,
> +
> mbuf->l3_len + mbuf->l2_len)->data_off) >> 4);
> + }
> +
> + /* check if L3 checksum is needed */
> + if (mbuf->ol_flags & PKT_TX_IP_CKSUM)
> + ena_tx_ctx->l3_csum_enable = true;
> +
> + if (mbuf->ol_flags & PKT_TX_IPV6) {
> + ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
> + } else {
> + ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
> +
> + /* set don't fragment (DF) flag */
> + if (mbuf->packet_type & (RTE_PTYPE_L4_NONFRAG ||
> + RTE_PTYPE_INNER_L4_NONFRAG))
> + ena_tx_ctx->df = true;
> + }
> +
> + /* check if L4 checksum is needed */
> + switch (mbuf->ol_flags & PKT_TX_L4_MASK) {
> + case PKT_TX_TCP_CKSUM:
> + ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
> + ena_tx_ctx->l4_csum_enable = true;
> + break;
> + case PKT_TX_UDP_CKSUM:
> + ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
> + ena_tx_ctx->l4_csum_enable = true;
> + break;
> + default:
> + ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN;
> + ena_tx_ctx->l4_csum_enable = false;
> + break;
> + }
> +
> + ena_meta->mss = mbuf->tso_segsz;
> + ena_meta->l3_hdr_len = mbuf->l3_len;
> + ena_meta->l3_hdr_offset = mbuf->l2_len;
> + /* this param needed only for TSO */
> + ena_meta->l3_outer_hdr_len = 0;
> + ena_meta->l3_outer_hdr_offset = 0;
> +
> + ena_tx_ctx->meta_valid = true;
> + } else {
> + ena_tx_ctx->meta_valid = false;
> + }
> +
> + return;
> +}
> +
> +static void ena_close(struct rte_eth_dev *dev)
> +{
> + struct ena_adapter *adapter = (struct
> ena_adapter*)(dev->data->dev_private);
> +
> + adapter->state = ENA_ADAPTER_STATE_STOPPED;
> +
> + ena_rx_queue_release_all(dev);
> + ena_tx_queue_release_all(dev);
> +
> + return;
> +}
> +
> +static void ena_rx_queue_release_all(struct rte_eth_dev *dev)
> +{
> + struct ena_ring **queues = (struct ena_ring**)dev->data->rx_queues;
> + int nb_queues = dev->data->nb_rx_queues;
> + int i;
> +
> + for (i = 0; i < nb_queues; i++)
> + ena_rx_queue_release(queues[i]);
> +
> + return;
> +}
> +
> +static void ena_tx_queue_release_all(struct rte_eth_dev *dev)
> +{
> + struct ena_ring **queues = (struct ena_ring**)dev->data->tx_queues;
> + int nb_queues = dev->data->nb_tx_queues;
> + int i;
> +
> + for (i = 0; i < nb_queues; i++)
> + ena_tx_queue_release(queues[i]);
> +
> + return;
> +}
> +
> +static void ena_rx_queue_release(void *queue)
> +{
> + struct ena_ring *ring = (struct ena_ring*)queue;
> + struct ena_adapter *adapter = ring->adapter;
> + int ena_qid;
> +
> + ena_assert_msg(ring->configured, "API violation. "
> + "Trying to release not configured queue");
> + ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNING,
> + "API violation");
> +
> + /* Destroy HW queue */
> + ena_qid = ENA_IO_RXQ_IDX(ring->id);
> + ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid);
> +
> + /* Free all bufs */
> + ena_rx_queue_release_bufs(ring);
> +
> + /* Free ring resources */
> + if (ring->rx_buffer_info)
> + rte_free(ring->rx_buffer_info);
> + ring->rx_buffer_info = NULL;
> +
> + ring->configured = 0;
> +
> + RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released \n", ring->port_id,
> ring->id);
> + return;
> +}
> +
> +static void ena_tx_queue_release(void *queue)
> +{
> + struct ena_ring *ring = (struct ena_ring*) queue;
> + struct ena_adapter *adapter = ring->adapter;
> + int ena_qid;
> +
> + ena_assert_msg(ring->configured, "API violation. Trying to release
> "
> + "not configured queue");
> + ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNING,
> + "API violation");
> +
> + /* Destroy HW queue */
> + ena_qid = ENA_IO_TXQ_IDX(ring->id);
> + ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid);
> +
> + /* Free all bufs */
> + ena_tx_queue_release_bufs(ring);
> +
> + /* Free ring resources */
> + if (ring->tx_buffer_info)
> + rte_free(ring->tx_buffer_info);
> +
> + if (ring->empty_tx_reqs)
> + rte_free(ring->empty_tx_reqs);
> +
> + ring->empty_tx_reqs = NULL;
> + ring->tx_buffer_info = NULL;
> +
> + ring->configured = 0;
> +
> + RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released \n", ring->port_id,
> ring->id);
> + return;
> +}
> +
> +static void ena_rx_queue_release_bufs(struct ena_ring *ring)
> +{
> + unsigned int ring_mask = ring->ring_size - 1;
> +
> + while (ring->next_to_clean != ring->next_to_use) {
> + struct rte_mbuf *m =
> ring->rx_buffer_info[ring->next_to_clean & ring_mask];
> +
> + if (m)
> + __rte_mbuf_raw_free(m);
> +
> + ring->next_to_clean = ENA_CIRC_INC(ring->next_to_clean, 1,
> ring->ring_size);
> + }
> +
> + return;
> +}
> +
> +static void ena_tx_queue_release_bufs(struct ena_ring *ring)
> +{
> + unsigned int ring_mask = ring->ring_size - 1;
> +
> + while (ring->next_to_clean != ring->next_to_use) {
> + struct ena_tx_buffer* tx_buf =
> + &ring->tx_buffer_info[ring->next_to_clean &
> ring_mask];
> +
> + if (tx_buf->mbuf)
> + rte_pktmbuf_free(tx_buf->mbuf);
> +
> + ring->next_to_clean = ENA_CIRC_INC(ring->next_to_clean, 1,
> ring->ring_size);
> + }
> +
> + return;
> +}
> +
> +static int ena_link_update(struct rte_eth_dev *dev,
> + __attribute__((unused))int wait_to_complete)
> +{
> + struct rte_eth_link *link = &dev->data->dev_link;
> +
> + link->link_status = 1;
> + link->link_speed = ETH_LINK_SPEED_10G;
> + link->link_duplex = ETH_LINK_FULL_DUPLEX;
> +
> + return 0;
> +}
> +
> +static int ena_queue_restart_all(struct rte_eth_dev *dev, enum
> ena_ring_type ring_type)
> +{
> + struct ena_adapter *adapter = (struct
> ena_adapter*)(dev->data->dev_private);
> + struct ena_ring *queues = NULL;
> + int i = 0;
> + int rc = 0;
> +
> + queues = (ring_type == ENA_RING_TYPE_RX) ? adapter->rx_ring :
> adapter->tx_ring;
> +
> + for (i = 0; i < ENA_NUM_QUEUES; i++) {
> + if (queues[i].configured) {
> + if (ring_type == ENA_RING_TYPE_RX) {
> + ena_assert_msg(dev->data->rx_queues[i] ==
> &queues[i],
> + "Inconsistent state of rx
> queues");
> + } else {
> + ena_assert_msg(dev->data->tx_queues[i] ==
> &queues[i],
> + "Inconsistent state of tx
> queues");
> + }
> +
> + rc = ena_queue_restart(&queues[i]);
> +
> + if (rc) {
> + PMD_INIT_LOG(ERR, "failed to restart queue
> %d type (%d) . rc:%d\n",
> + i, ring_type, rc);
> + return -1;
> + }
> + }
> + }
> +
> + return 0;
> +}
> +
> +static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter)
> +{
> +
> + uint32_t max_frame_len = adapter->max_mtu;
> + if (adapter->rte_eth_dev_data->dev_conf.rxmode.jumbo_frame == 1)
> + max_frame_len =
> adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len;
> +
> + return max_frame_len;
> +}
> +
> +static int ena_check_valid_conf(struct ena_adapter *adapter)
> +{
> + uint32_t max_frame_len = ena_get_mtu_conf(adapter);
> +
> + if (max_frame_len > ENA_MAX_FRAME_LEN) {
> + PMD_INIT_LOG(ERR, "Unsupported MTU of %d\n",
> max_frame_len);
> + return -1;
> + }
> +
> + return 0;
> +
> +}
> +
> +static void ena_stats_restart(struct rte_eth_dev *dev)
> +{
> + struct ena_adapter *adapter = (struct
> ena_adapter*)(dev->data->dev_private);
> + rte_atomic64_init(&adapter->drv_stats->ierrors);
> + rte_atomic64_init(&adapter->drv_stats->oerrors);
> + rte_atomic64_init(&adapter->drv_stats->imcasts);
> + rte_atomic64_init(&adapter->drv_stats->rx_nombuf);
> +}
> +
> +static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats
> *stats)
> +{
> + struct ena_admin_basic_stats ena_stats;
> + struct ena_adapter *adapter = (struct
> ena_adapter*)(dev->data->dev_private);
> + struct ena_com_dev *ena_dev = &adapter->ena_dev;
> + int rc;
> +
> + if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> + return;
> +
> + memset(&ena_stats, 0, sizeof(ena_stats));
> + rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats);
> + if (unlikely(rc)) {
> + RTE_LOG(ERR, PMD, "Could not retrieve statistics from
> ENA");
> + return;
> + }
> +
> + /* Set of basic statistics from ENA */
> + stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high,
> ena_stats.rx_pkts_low);
> + stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high,
> ena_stats.tx_pkts_low);
> + stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high,
> ena_stats.rx_bytes_low);
> + stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high,
> ena_stats.tx_bytes_low);
> + stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high,
> ena_stats.rx_drops_low);
> +
> + /* Driver related stats */
> + stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors);
> + stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors);
> + stats->imcasts = rte_atomic64_read(&adapter->drv_stats->imcasts);
> + stats->rx_nombuf =
> rte_atomic64_read(&adapter->drv_stats->rx_nombuf);
> +}
> +
> +static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
> +{
> + struct ena_adapter *adapter;
> + struct ena_com_dev *ena_dev;
> + int rc = 0;
> +
> + ena_assert_msg(dev->data != NULL, "Uninitialized device");
> + ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized
> device");
> + adapter = (struct ena_adapter*)(dev->data->dev_private);
> +
> + ena_dev = &adapter->ena_dev;
> + ena_assert_msg(ena_dev != NULL, "Uninitialized device");
> +
> + if (mtu > ena_get_mtu_conf(adapter)) {
> + RTE_LOG(ERR, PMD, "Given MTU (%d) exceeds maximum MTU
> supported (%d)\n",
> + mtu, ena_get_mtu_conf(adapter));
> + rc = -EINVAL;
> + goto err;
> + }
> +
> + rc = ena_com_set_dev_mtu(ena_dev, mtu);
> + if (rc)
> + RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu);
> + else
> + RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu);
> +
> +err:
> + return rc;
> +}
> +
> +static int ena_start(struct rte_eth_dev *dev)
> +{
> + struct ena_adapter *adapter = (struct
> ena_adapter*)(dev->data->dev_private);
> + int rc = 0;
> +
> + if(!(adapter->state == ENA_ADAPTER_STATE_CONFIG ||
> + adapter->state == ENA_ADAPTER_STATE_STOPPED)) {
> + PMD_INIT_LOG(ERR, "API violation");
> + return -1;
> + }
> +
> + rc = ena_check_valid_conf(adapter);
> + if (rc)
> + return rc;
> +
> + rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX);
> + if (rc)
> + return rc;
> +
> + rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX);
> + if (rc)
> + return rc;
> +
> + ena_stats_restart(dev);
> +
> + adapter->state = ENA_ADAPTER_STATE_RUNING;
> +
> + return 0;
> +}
> +
> +static int ena_queue_restart(struct ena_ring *ring)
> +{
> + int rc;
> +
> + ena_assert_msg(ring->configured == 1, "Trying to restart
> unconfigured queue");
> +
> + ring->next_to_clean = 0;
> + ring->next_to_use = 0;
> +
> + if (ring->type == ENA_RING_TYPE_TX)
> + return 0;
> +
> + rc = ena_populate_rx_queue(ring, ring->ring_size - 1);
> + if ((unsigned int)rc != ring->ring_size - 1) {
> + PMD_INIT_LOG(ERR, "Failed to populate rx ring !\n");
> + return (-1);
> + }
> +
> + return 0;
> +}
> +
> +static int ena_tx_queue_setup(struct rte_eth_dev *dev,
> + uint16_t queue_idx,
> + uint16_t nb_desc,
> + __attribute__((unused))unsigned int
> socket_id,
> + __attribute__((unused))const struct
> rte_eth_txconf *tx_conf)
> +
> +{
> + struct ena_ring *txq = NULL;
> + struct ena_adapter *adapter = (struct
> ena_adapter*)(dev->data->dev_private);
> + unsigned int i;
> + int ena_qid;
> + int rc;
> + struct ena_com_dev *ena_dev = &adapter->ena_dev;
> +
> + txq = &adapter->tx_ring[queue_idx];
> +
> + if (txq->configured) {
> + RTE_LOG(CRIT, PMD, "API violation. Queue %d is already
> configured\n",
> + queue_idx);
> + return -1;
> + }
> +
> + ena_qid = ENA_IO_TXQ_IDX(queue_idx);
> + rc = ena_com_create_io_queue(ena_dev, ena_qid,
> + ENA_COM_IO_QUEUE_DIRECTION_TX,
> + ENA_ADMIN_PLACEMENT_POLICY_HOST,
> + -1 /* admin interrupts is not used */,
> + nb_desc);
> + if (rc) {
> + RTE_LOG(ERR, PMD, "failed to create io TX queue #%d
> (ena_qid: %d)"
> + "rc: %d\n", queue_idx, ena_qid, rc);
> + }
> + txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid];
> + txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid];
> +
> + txq->port_id = dev->data->port_id;
> + txq->next_to_clean = 0;
> + txq->next_to_use = 0;
> + txq->ring_size = nb_desc;
> +
> + txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info",
> + sizeof(struct ena_tx_buffer) *
> txq->ring_size,
> + RTE_CACHE_LINE_SIZE);
> + if (!txq->tx_buffer_info) {
> + RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer
> info\n");
> + return -ENOMEM;
> + }
> +
> + txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs",
> + sizeof(u16) * txq->ring_size,
> RTE_CACHE_LINE_SIZE);
> + if (!txq->empty_tx_reqs) {
> + RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n");
> + rte_free(txq->tx_buffer_info);
> + return -ENOMEM;
> + }
> + for (i = 0; i < txq->ring_size; i++)
> + txq->empty_tx_reqs[i] = i;
> +
> + /* Store pointer to this queue in upper layer */
> + txq->configured = 1;
> + dev->data->tx_queues[queue_idx] = txq;
> +
> + return rc;
> +}
> +
> +static int ena_rx_queue_setup(struct rte_eth_dev *dev,
> + uint16_t queue_idx,
> + uint16_t nb_desc,
> + __attribute__((unused))unsigned int
> socket_id,
> + __attribute__((unused))const struct
> rte_eth_rxconf *rx_conf,
> + struct rte_mempool *mp)
> +{
> + struct ena_adapter *adapter = (struct
> ena_adapter*)(dev->data->dev_private);
> + struct ena_ring *rxq = NULL;
> + uint16_t ena_qid = 0;
> + int rc = 0;
> + struct ena_com_dev *ena_dev = &adapter->ena_dev;
> +
> + rxq = &adapter->rx_ring[queue_idx];
> + if (rxq->configured) {
> + RTE_LOG(CRIT, PMD, "API violation. Queue %d is already
> configured\n",
> + queue_idx);
> + return -1;
> + }
> + ena_qid = ENA_IO_RXQ_IDX(queue_idx);
> + rc = ena_com_create_io_queue(ena_dev, ena_qid,
> + ENA_COM_IO_QUEUE_DIRECTION_RX,
> + ENA_ADMIN_PLACEMENT_POLICY_HOST,
> + -1 /* admin interrupts not used */,
> + nb_desc);
> + if (rc)
> + RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc:
> %d\n",
> + queue_idx, rc);
> +
> + rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid];
> + rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid];
> +
> + rxq->port_id = dev->data->port_id;
> + rxq->next_to_clean = 0;
> + rxq->next_to_use = 0;
> + rxq->ring_size = nb_desc;
> + rxq->mb_pool = mp;
> +
> + rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info",
> + sizeof(struct rte_mbuf*) *
> nb_desc, RTE_CACHE_LINE_SIZE);
> + if (!rxq->rx_buffer_info) {
> + RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer
> info\n");
> + return -ENOMEM;
> + }
> +
> + /* Store pointer to this queue in upper layer */
> + rxq->configured = 1;
> + dev->data->rx_queues[queue_idx] = rxq;
> +
> + return rc;
> +}
> +
> +static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count)
> +{
> + unsigned int i;
> + int rc;
> + unsigned int ring_size = rxq->ring_size;
> + unsigned int ring_mask = ring_size - 1;
> + int next_to_use = rxq->next_to_use & ring_mask;
> + struct rte_mbuf ** mbufs = &rxq->rx_buffer_info[0];
> +
> + if (unlikely(!count)) return 0;
> +
> + ena_assert_msg((((ENA_CIRC_COUNT(rxq->next_to_use,
> rxq->next_to_clean,
> + rxq->ring_size)) +
> + count) < rxq->ring_size), "bad ring state");
> +
> + count = RTE_MIN(count, ring_size - next_to_use);
> +
> + /* get resources for incoming packets */
> + rc = rte_mempool_get_bulk(rxq->mb_pool, (void
> **)(&mbufs[next_to_use]), count);
> + if (unlikely(rc < 0)) {
> + rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf);
> + PMD_RX_LOG(DEBUG,"there are no enough free buffers");
> + return 0;
> + }
> +
> + for (i = 0; i < count; i++) {
> + struct rte_mbuf *mbuf = mbufs[next_to_use];
> + struct ena_com_buf ebuf;
> +
> + rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]);
> + /* prepare physical address for DMA transaction */
> + ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM;
> + ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM;
> + /* pass resource to device */
> + rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, &ebuf,
> next_to_use);
> + if (unlikely(rc)) {
> + RTE_LOG(WARNING, PMD, "failed adding rx desc\n");
> + break;
> + }
> + next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, ring_size);
> + }
> +
> + rte_wmb();
> + rxq->next_to_use = next_to_use;
> + /* let HW know that it can fill buffers with data */
> + ena_com_write_sq_doorbell(rxq->ena_com_io_sq);
> +
> + return i;
> +}
> +
> +static int ena_device_init(struct ena_com_dev *ena_dev,
> + struct ena_com_dev_get_features_ctx
> *get_feat_ctx)
> +{
> + int rc;
> +
> + /* Initialize mmio registers */
> + rc = ena_com_mmio_reg_read_request_init(ena_dev);
> + if (rc) {
> + RTE_LOG(ERR, PMD, "failed to init mmio read less\n");
> + return rc;
> + }
> +
> + /* reset device */
> + rc = ena_com_dev_reset(ena_dev);
> + if (rc) {
> + RTE_LOG(ERR, PMD, "cannot reset device\n");
> + goto err_mmio_read_less;
> + }
> +
> + /* check FW version */
> + rc = ena_com_validate_version(ena_dev);
> + if (rc) {
> + RTE_LOG(ERR, PMD, "device version is too low\n");
> + goto err_mmio_read_less;
> + }
> +
> + ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev);
> +
> + /* ENA device administration layer init */
> + rc = ena_com_admin_init(ena_dev, NULL, true);
> + if (rc) {
> + RTE_LOG(ERR, PMD, "cannot initialize ena admin queue with
> device\n");
> + goto err_mmio_read_less;
> + }
> +
> + /* To enable the msix interrupts the driver needs to know the
> number
> + * of queues. So the driver uses polling mode to retrieve this
> + * information */
> + ena_com_set_admin_polling_mode(ena_dev, true);
> +
> + /* Get Device Attributes and features */
> + rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
> + if (rc) {
> + RTE_LOG(ERR, PMD, "cannot get attribute for ena device rc=
> %d\n", rc);
> + goto err_admin_init;
> + }
> +
> + return 0;
> +
> +err_admin_init:
> + ena_com_admin_destroy(ena_dev);
> +
> +err_mmio_read_less:
> + ena_com_mmio_reg_read_request_destroy(ena_dev);
> +
> + return rc;
> +}
> +
> +static int eth_ena_dev_init(struct rte_eth_dev *eth_dev)
> +{
> + struct rte_pci_device *pci_dev;
> + struct ena_adapter *adapter = (struct
> ena_adapter*)(eth_dev->data->dev_private);
> + struct ena_com_dev *ena_dev = &adapter->ena_dev;
> + struct ena_com_dev_get_features_ctx get_feat_ctx;
> + int rc;
> +
> + static int adapters_found = 0;
> +
> + memset(adapter, 0, sizeof(struct ena_adapter));
> + ena_dev = &adapter->ena_dev;
> +
> + eth_dev->dev_ops = &ena_dev_ops;
> + eth_dev->rx_pkt_burst = ð_ena_recv_pkts;
> + eth_dev->tx_pkt_burst = ð_ena_xmit_pkts;
> + adapter->rte_eth_dev_data = eth_dev->data;
> + adapter->rte_dev = eth_dev;
> +
> + if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
> + return 0;
> + }
> +
> + pci_dev = eth_dev->pci_dev;
> + adapter->pdev = pci_dev;
> +
> + PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n",
> pci_dev->addr.domain,
> + pci_dev->addr.bus, pci_dev->addr.devid,
> pci_dev->addr.function);
> +
> + adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr;
> + /* DPDK is operating only on host memory policy */
> + adapter->dev_mem_base = NULL;
> + ena_dev->reg_bar = adapter->regs;
> + ena_dev->mem_bar = adapter->dev_mem_base;
> + ena_dev->dmadev = adapter->pdev;
> +
> + adapter->id_number = adapters_found;
> +
> + snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d",
> adapter->id_number);
> +
> + /* device specific initialization routine */
> + rc = ena_device_init(ena_dev, &get_feat_ctx);
> + if (rc) {
> + PMD_INIT_LOG(CRIT, "Failed to init ENA device\n");
> + return -1;
> + }
> + /* prepare ring structures */
> + ena_init_rings(adapter);
> +
> + /* Set max MTU for this device */
> + adapter->max_mtu = ENA_MAX_MTU;
> +
> + /* Copy MAC address and point DPDK to it */
> + eth_dev->data->mac_addrs = (struct ether_addr*)adapter->mac_addr;
> + ether_addr_copy((struct ether_addr
> *)get_feat_ctx.dev_attr.mac_addr,
> + (struct ether_addr *)adapter->mac_addr);
> +
> + adapter->drv_stats = rte_zmalloc("adapter stats",
> + sizeof(*adapter->drv_stats),
> + RTE_CACHE_LINE_SIZE);
> + if (!adapter->drv_stats) {
> + RTE_LOG(ERR, PMD, "failed to alloc mem for adapter
> stats\n");
> + return -ENOMEM;
> + }
> +
> + adapters_found++;
> + adapter->state = ENA_ADAPTER_STATE_INIT;
> +
> + return 0;
> +}
> +
> +static int ena_dev_configure(struct rte_eth_dev *dev)
> +{
> +
> + struct ena_adapter *adapter = (struct
> ena_adapter*)(dev->data->dev_private);
> +
> + if (!(adapter->state == ENA_ADAPTER_STATE_INIT ||
> + adapter->state == ENA_ADAPTER_STATE_STOPPED)) {
> + PMD_INIT_LOG(ERR, "Illegal adapter state: %d\n",
> adapter->state);
> + return -1;
> + }
> + switch (adapter->state) {
> + case ENA_ADAPTER_STATE_INIT:
> + case ENA_ADAPTER_STATE_STOPPED:
> + adapter->state = ENA_ADAPTER_STATE_CONFIG;
> + break;
> + case ENA_ADAPTER_STATE_CONFIG:
> + RTE_LOG(WARNING, PMD, "Ivalid driver state while trying to
> configure device\n");
> + break;
> + default:
> + break;
> + }
> +
> + return 0;
> +}
> +
> +static void ena_init_rings(struct ena_adapter *adapter)
> +{
> + int i;
> +
> + for (i = 0; i < ENA_NUM_QUEUES; i++) {
> + struct ena_ring *ring = &adapter->tx_ring[i];
> + ring->configured = 0;
> + ring->type = ENA_RING_TYPE_TX;
> + ring->adapter = adapter;
> + ring->id = i;
> + }
> +
> + for (i = 0; i < ENA_NUM_QUEUES; i++) {
> + struct ena_ring *ring = &adapter->rx_ring[i];
> + ring->configured = 0;
> + ring->type = ENA_RING_TYPE_RX;
> + ring->adapter = adapter;
> + ring->id = i;
> + }
> +}
> +
> +static void ena_infos_get(struct rte_eth_dev *dev,
> + struct rte_eth_dev_info *dev_info)
> +{
> + struct ena_adapter *adapter;
> + struct ena_com_dev *ena_dev;
> + struct ena_com_dev_get_features_ctx feat;
> + uint32_t rx_feat = 0, tx_feat = 0;
> + int rc = 0;
> +
> + ena_assert_msg(dev->data != NULL, "Uninitialized device");
> + ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized
> device");
> + adapter = (struct ena_adapter*)(dev->data->dev_private);
> +
> + ena_dev = &adapter->ena_dev;
> + ena_assert_msg(ena_dev != NULL, "Uninitialized device");
> +
> + /* Get supported features from HW */
> + rc = ena_com_get_dev_attr_feat(ena_dev, &feat);
> + if (unlikely(rc)) {
> + RTE_LOG(ERR, PMD, "Cannot get attribute for ena device rc=
> %d\n", rc);
> + return;
> + }
> +
> + /* Set Tx & Rx features available for device */
> + if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
> + tx_feat |= DEV_TX_OFFLOAD_TCP_TSO;
> +
> + if (feat.offload.tx &
> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
> + tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM |
> DEV_TX_OFFLOAD_UDP_CKSUM |
> + DEV_TX_OFFLOAD_TCP_CKSUM;
> +
> + if (feat.offload.tx &
> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
> + rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM |
> DEV_RX_OFFLOAD_UDP_CKSUM |
> + DEV_RX_OFFLOAD_TCP_CKSUM;
> +
> + /* Inform framework about available features */
> + dev_info->rx_offload_capa = rx_feat;
> + dev_info->tx_offload_capa = tx_feat;
> +
> + dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN;
> + dev_info->max_rx_pktlen = ENA_MAX_FRAME_LEN;
> + dev_info->max_mac_addrs = 1;
> +
> + dev_info->max_rx_queues = ENA_NUM_QUEUES;
> + dev_info->max_tx_queues = ENA_NUM_QUEUES;
> +}
> +
> +static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf
> **rx_pkts,
> + uint16_t nb_pkts)
> +{
> + struct ena_ring *rx_ring = (struct ena_ring*)(rx_queue);
> + unsigned int ring_size = rx_ring->ring_size;
> + unsigned int ring_mask = ring_size - 1;
> + uint16_t next_to_clean = rx_ring->next_to_clean;
> + int desc_in_use = 0;
> + unsigned int recv_idx = 0;
> + struct rte_mbuf *mbuf = NULL;
> + struct rte_mbuf *mbuf_head = NULL;
> + struct rte_mbuf *mbuf_prev = NULL;
> + struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info;
> + unsigned int completed;
> +
> + struct ena_com_rx_ctx ena_rx_ctx;
> + int rc = 0;
> +
> + /* Check adapter state */
> + if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNING))
> {
> + RTE_LOG(ALERT, PMD,
> + "Trying to receive packets while eth device is NOT
> running\n");
> + return 0;
> + }
> +
> + desc_in_use = ENA_CIRC_COUNT(rx_ring->next_to_use, next_to_clean,
> ring_size);
> + if (unlikely(nb_pkts > desc_in_use)) nb_pkts = desc_in_use;
> +
> + for (completed = 0; completed < nb_pkts; completed++) {
> + ena_rx_ctx.max_bufs = rx_ring->ring_size;
> + ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
> + ena_rx_ctx.descs = 0;
> + /* receive packet context */
> + rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
> rx_ring->ena_com_io_sq,
> + &ena_rx_ctx);
> + if (unlikely(rc)) {
> + RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d \n",
> rc);
> + return 0;
> + }
> +
> + if (unlikely(ena_rx_ctx.descs == 0))
> + break;
> +
> + int segments = 0;
> + while (segments < ena_rx_ctx.descs) {
> + mbuf = rx_buff_info[next_to_clean & ring_mask];
> + mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len;
> + mbuf->data_off = RTE_PKTMBUF_HEADROOM;
> + mbuf->refcnt = 1;
> + mbuf->next = NULL;
> + if (segments == 0) {
> + mbuf->nb_segs = ena_rx_ctx.descs;
> + mbuf->port = rx_ring->port_id;
> + mbuf->pkt_len = 0;
> + mbuf_head = mbuf;
> + } else {
> + /* for multi-segment packets create mbuf
> chain */
> + mbuf_prev->next = mbuf;
> + }
> + mbuf_head->pkt_len += mbuf->data_len;
> +
> + mbuf_prev = mbuf;
> + segments++;
> + next_to_clean =
> ENA_RX_RING_IDX_NEXT(next_to_clean, ring_size);
> + }
> +
> + /* fill mbuf attributes if any */
> + ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx);
> +
> + /* pass to DPDK application head mbuf */
> + rx_pkts[recv_idx] = mbuf_head;
> + recv_idx++;
> + }
> +
> + /* Burst refill to save doorbells, memory barriers, const interval
> */
> + if (ring_size - desc_in_use - 1 > ENA_RING_DESCS_RATIO(ring_size))
> + ena_populate_rx_queue(rx_ring, ring_size - desc_in_use -
> 1);
> +
> + rx_ring->next_to_clean = next_to_clean & ring_mask;
> +
> + return recv_idx;
> +}
> +
> +static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf
> **tx_pkts,
> + uint16_t nb_pkts)
> +{
> + struct ena_ring *tx_ring = (struct ena_ring*)(tx_queue);
> + unsigned int next_to_use = tx_ring->next_to_use;
> + struct rte_mbuf *mbuf;
> + unsigned int ring_size = tx_ring->ring_size;
> + unsigned int ring_mask = ring_size - 1;
> + struct ena_com_tx_ctx ena_tx_ctx;
> + struct ena_tx_buffer *tx_info;
> + struct ena_com_buf *ebuf;
> + unsigned int i;
> + uint16_t rc, req_id, total_tx_descs = 0;
> + int sent_idx = 0;
> + int nb_hw_desc;
> +
> + /* Check adapter state */
> + if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNING))
> {
> + RTE_LOG(ALERT, PMD,
> + "Trying to transmit packets while device is NOT
> running\n");
> + return 0;
> + }
> +
> + for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) {
> +
> + mbuf = tx_pkts[sent_idx];
> +
> + req_id = tx_ring->empty_tx_reqs[next_to_use];
> + tx_info = &tx_ring->tx_buffer_info[req_id];
> + tx_info->mbuf = mbuf;
> + tx_info->num_of_bufs = mbuf->nb_segs;
> +
> + ebuf = tx_info->bufs;
> + /* Prepare TX context */
> + memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
> + memset(&ena_tx_ctx.ena_meta, 0x0, sizeof(struct
> ena_com_tx_meta));
> + ena_tx_ctx.ena_bufs = ebuf;
> + ena_tx_ctx.push_header = (void *)tx_info->bufs;
> + ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
> + ena_tx_ctx.req_id = req_id;
> + /* Set TX offloads flags, if applicable */
> + ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx);
> +
> + if (unlikely(mbuf->ol_flags &
> (PKT_RX_L4_CKSUM_BAD|PKT_RX_IP_CKSUM_BAD))) {
> +
> rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors);
> + }
> +
> + rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]);
> +
> + /* Fill bufs for HW */
> + for (i=1; i <= tx_info->num_of_bufs; i++) {
> + ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off;
> + ebuf->len = mbuf->data_len;
> +
> + /* Iterate through all mbuf segments */
> + if (((mbuf = mbuf->next) == NULL) &&
> + (i != tx_info->num_of_bufs)) {
> + break;
> + }
> + ebuf++;
> + }
> +
> + /* Write data to device */
> + rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq,
> &ena_tx_ctx, &nb_hw_desc);
> + if (unlikely(rc)) {
> +
> rte_atomic64_inc(&tx_ring->adapter->drv_stats->oerrors);
> + break;
> + }
> +
> + tx_info->tx_descs = nb_hw_desc;
> +
> + next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, ring_size);
> + }
> +
> + /* Let HW do it's best :-) */
> + rte_wmb();
> + ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
> +
> + /* Clear complete packets */
> + while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id)
> >= 0) {
> + /* Get Tx info & store how many descs were processed */
> + tx_info = &tx_ring->tx_buffer_info[req_id];
> + total_tx_descs += tx_info->tx_descs;
> +
> + /* Free whole mbuf chain */
> + mbuf = tx_info->mbuf;
> + rte_pktmbuf_free(mbuf);
> +
> + /* Put back descriptor to the ring for reuse */
> + tx_ring->empty_tx_reqs[tx_ring->next_to_clean] = req_id;
> + tx_ring->next_to_clean =
> + ENA_TX_RING_IDX_NEXT(tx_ring->next_to_clean,
> tx_ring->ring_size);
> +
> + /* If too many descs to clean, leave it for another run */
> + if (unlikely(total_tx_descs >
> ENA_RING_DESCS_RATIO(ring_size)))
> + break;
> + }
> +
> + /* acknowledge completion of sent packets */
> + ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs);
> + tx_ring->next_to_use = next_to_use;
> + return sent_idx;
> +}
> +
> +static struct eth_driver rte_ena_pmd = {
> + {
> + .name = "rte_ena_pmd",
> + .id_table = pci_id_ena_map,
> + .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
> + },
> + .eth_dev_init = eth_ena_dev_init,
> + .dev_private_size = sizeof(struct ena_adapter),
> +};
> +
> +static int
> +rte_ena_pmd_init(const char *name __rte_unused, const char *params
> __rte_unused)
> +{
> + rte_eth_driver_register(&rte_ena_pmd);
> + return 0;
> +};
> +
> +struct rte_driver ena_pmd_drv = {
> + .name = "ena_driver",
> + .type = PMD_PDEV,
> + .init = rte_ena_pmd_init,
> +};
> +
> +PMD_REGISTER_DRIVER(ena_pmd_drv);
> diff --git a/drivers/net/ena/ena_ethdev.h b/drivers/net/ena/ena_ethdev.h
> new file mode 100755
> index 0000000..ba5b301
> --- /dev/null
> +++ b/drivers/net/ena/ena_ethdev.h
> @@ -0,0 +1,151 @@
> +/*-
> +* BSD LICENSE
> +*
> +* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates.
> +* All rights reserved.
> +*
> +* Redistribution and use in source and binary forms, with or without
> +* modification, are permitted provided that the following conditions
> +* are met:
> +*
> +* * Redistributions of source code must retain the above copyright
> +* notice, this list of conditions and the following disclaimer.
> +* * Redistributions in binary form must reproduce the above copyright
> +* notice, this list of conditions and the following disclaimer in
> +* the documentation and/or other materials provided with the
> +* distribution.
> +* * Neither the name of copyright holder nor the names of its
> +* contributors may be used to endorse or promote products derived
> +* from this software without specific prior written permission.
> +*
> +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +*/
> +
> +#ifndef _ENA_ETHDEV_H_
> +#define _ENA_ETHDEV_H_
> +
> +#include <rte_pci.h>
> +
> +#pragma GCC diagnostic ignored "-Wcast-qual"
> +
> +#include "ena_com.h"
> +
> +#define ENA_REGS_BAR 0
> +
> +#define ENA_NUM_QUEUES 8
> +
> +#define ENA_DEFAULT_TX_SW_DESCS (1024)
> +#define ENA_DEFAULT_TX_HW_DESCS (1024)
> +#define ENA_DEFAULT_RX_DESCS (1024)
> +#define ENA_DEFAULT_TX_DESCS (1024)
> +
> +#define ENA_MIN_FRAME_LEN 64
> +#define ENA_MAX_FRAME_LEN 9216
> +
> +#define ENA_NAME_MAX_LEN 20
> +#define ENA_IRQNAME_SIZE 40
> +
> +#define ENA_PKT_MAX_BUFS 17
> +
> +#define ENA_CIRC_COUNT(head, tail, size) (((uint16_t)((uint16_t)(head) -
> (uint16_t)(tail))) & ((size) - 1))
> +
> +#define ENA_CIRC_INC(index, step, size) ((uint16_t)(index) +
> (uint16_t)(step))
> +#define ENA_CIRC_INC_WRAP(index, step, size) (((uint16_t)(index) +
> (uint16_t)(step)) & ((size) - 1))
> +
> +#define ENA_TX_RING_IDX_NEXT(idx, ring_size) ENA_CIRC_INC_WRAP(idx, 1,
> ring_size)
> +#define ENA_RX_RING_IDX_NEXT(idx, ring_size) ENA_CIRC_INC_WRAP(idx, 1,
> ring_size)
> +
> +
> +struct ena_adapter;
> +
> +enum ena_ring_type {
> + ENA_RING_TYPE_RX = 1,
> + ENA_RING_TYPE_TX = 2,
> +};
> +
> +struct ena_tx_buffer {
> + struct rte_mbuf *mbuf;
> + unsigned int tx_descs;
> + unsigned int num_of_bufs;
> + struct ena_com_buf bufs[ENA_PKT_MAX_BUFS];
> +};
> +
> +struct ena_ring {
> + u16 next_to_use;
> + u16 next_to_clean;
> +
> + enum ena_ring_type type;
> + uint16_t *empty_tx_reqs; /* Holds the empty requests for TX OOO
> completions */
> + union {
> + struct ena_tx_buffer *tx_buffer_info; /* contex of tx
> packet */
> + struct rte_mbuf **rx_buffer_info; /* contex of rx packet */
> + };
> + unsigned int ring_size; /* number of tx/rx_buffer_info's entries */
> +
> + struct ena_com_io_cq *ena_com_io_cq;
> + struct ena_com_io_sq *ena_com_io_sq;
> +
> + struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS]
> __rte_cache_aligned;
> +
> + struct rte_mempool *mb_pool;
> + unsigned int port_id;
> + unsigned int id;
> + int configured;
> + struct ena_adapter *adapter;
> +};
> +
> +enum ena_adapter_state {
> + ENA_ADAPTER_STATE_FREE = 0,
> + ENA_ADAPTER_STATE_INIT = 1,
> + ENA_ADAPTER_STATE_RUNING = 2,
> + ENA_ADAPTER_STATE_STOPPED = 3,
> + ENA_ADAPTER_STATE_CONFIG = 4,
> +};
> +
> +struct ena_driver_stats {
> + rte_atomic64_t ierrors;
> + rte_atomic64_t oerrors;
> + rte_atomic64_t imcasts;
> + rte_atomic64_t rx_nombuf;
> +};
> +
> +/* board specific private data structure */
> +struct ena_adapter {
> + /* OS defined structs */
> + struct rte_pci_device *pdev;
> + struct rte_eth_dev_data *rte_eth_dev_data;
> + struct rte_eth_dev *rte_dev;
> +
> + struct ena_com_dev ena_dev;
> +
> + /* TX */
> + struct ena_ring tx_ring[ENA_NUM_QUEUES] __rte_cache_aligned;
> +
> + /* RX */
> + struct ena_ring rx_ring[ENA_NUM_QUEUES] __rte_cache_aligned;
> +
> + u8 mac_addr[ETHER_ADDR_LEN];
> + u16 max_mtu;
> +
> + int id_number;
> + char name[ENA_NAME_MAX_LEN];
> +
> + void* regs;
> + void *dev_mem_base;
> +
> + struct ena_driver_stats *drv_stats;
> + enum ena_adapter_state state;
> +};
> +
> +
> +#endif
> diff --git a/drivers/net/ena/ena_logs.h b/drivers/net/ena/ena_logs.h
> new file mode 100644
> index 0000000..10e4858
> --- /dev/null
> +++ b/drivers/net/ena/ena_logs.h
> @@ -0,0 +1,76 @@
> +/*-
> +* BSD LICENSE
> +*
> +* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates.
> +* All rights reserved.
> +*
> +* Redistribution and use in source and binary forms, with or without
> +* modification, are permitted provided that the following conditions
> +* are met:
> +*
> +* * Redistributions of source code must retain the above copyright
> +* notice, this list of conditions and the following disclaimer.
> +* * Redistributions in binary form must reproduce the above copyright
> +* notice, this list of conditions and the following disclaimer in
> +* the documentation and/or other materials provided with the
> +* distribution.
> +* * Neither the name of copyright holder nor the names of its
> +* contributors may be used to endorse or promote products derived
> +* from this software without specific prior written permission.
> +*
> +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +*/
> +
> +#ifndef _ENA_LOGS_H_
> +#define _ENA_LOGS_H_
> +
> +#define RTE_LOGTYPE_ENA RTE_LOGTYPE_USER1
> +
> +#ifdef RTE_LIBRTE_ENA_DEBUG_INIT
> +#define PMD_INIT_LOG(level, fmt, args...) \
> + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +#else
> +#define PMD_INIT_LOG(level, fmt, args...) do { } while(0)
> +#endif
> +
> +#ifdef RTE_LIBRTE_ENA_DEBUG_RX
> +#define PMD_RX_LOG(level, fmt, args...) \
> + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +#else
> +#define PMD_RX_LOG(level, fmt, args...) do { } while(0)
> +#endif
> +
> +#ifdef RTE_LIBRTE_ENA_DEBUG_TX
> +#define PMD_TX_LOG(level, fmt, args...) \
> + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +#else
> +#define PMD_TX_LOG(level, fmt, args...) do { } while(0)
> +#endif
> +
> +#ifdef RTE_LIBRTE_ENA_DEBUG_TX_FREE
> +#define PMD_TX_FREE_LOG(level, fmt, args...) \
> + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +#else
> +#define PMD_TX_FREE_LOG(level, fmt, args...) do { } while(0)
> +#endif
> +
> +#ifdef RTE_LIBRTE_ENA_DEBUG_DRIVER
> +#define PMD_DRV_LOG(level, fmt, args...) \
> + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
> +#else
> +#define PMD_DRV_LOG(level, fmt, args...) do { } while(0)
> +#endif
> +
> +
> +
> +#endif /* _ENA_LOGS_H_ */
> diff --git a/drivers/net/ena/ena_platform.h
> b/drivers/net/ena/ena_platform.h
> new file mode 100644
> index 0000000..e1e9dd7
> --- /dev/null
> +++ b/drivers/net/ena/ena_platform.h
> @@ -0,0 +1,58 @@
> +/*-
> +* BSD LICENSE
> +*
> +* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates.
> +* All rights reserved.
> +*
> +* Redistribution and use in source and binary forms, with or without
> +* modification, are permitted provided that the following conditions
> +* are met:
> +*
> +* * Redistributions of source code must retain the above copyright
> +* notice, this list of conditions and the following disclaimer.
> +* * Redistributions in binary form must reproduce the above copyright
> +* notice, this list of conditions and the following disclaimer in
> +* the documentation and/or other materials provided with the
> +* distribution.
> +* * Neither the name of copyright holder nor the names of its
> +* contributors may be used to endorse or promote products derived
> +* from this software without specific prior written permission.
> +*
> +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +*/
> +
> +#ifndef __ENA_PLATFORM_H__
> +#define __ENA_PLATFORM_H__
> +
> +#define swap16_to_le(x) (x)
> +
> +#define swap32_to_le(x) (x)
> +
> +#define swap64_to_le(x) (x)
> +
> +#define swap16_from_le(x) (x)
> +
> +#define swap32_from_le(x) (x)
> +
> +#define swap64_from_le(x) (x)
> +
> +#define ena_assert(cond)
> \
> + if (unlikely(!(cond))) {
> \
> +/* RTE_LOG(ERR, ENA, "Assert failed on %s:%s:%d:
> "__FILE__, __func__, __LINE__);*/ \
> + rte_panic();
> \
> + }
> \
> +
> +#define ena_assert_msg(x,msg) {if(unlikely(!(x))) {rte_panic(msg);}}
> +
> +
> +#endif
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk
> index 8ecab41..791bd48 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -144,6 +144,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_I40E_PMD) +=
> -lrte_pmd_i40e
> _LDLIBS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += -lrte_pmd_fm10k
> _LDLIBS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += -lrte_pmd_ixgbe
> _LDLIBS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += -lrte_pmd_e1000
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += -lrte_pmd_ena
> _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4
> _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5
> _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp
> --
> 1.9.1
>
>
More information about the dev
mailing list