[dpdk-dev] [PATCH v6 10/13] ixgbe: support VMDq RSS in non-SRIOV environment

[Ananyev, Konstantin]

Hi 

> -----Original Message-----
> From: dev [mailto:dev-bounces at dpdk.org] On Behalf Of Yuanhan Liu
> Sent: Friday, October 09, 2015 6:46 AM
> To: dev at dpdk.org
> Cc: Michael S. Tsirkin; marcel at redhat.com
> Subject: [dpdk-dev] [PATCH v6 10/13] ixgbe: support VMDq RSS in non-SRIOV environment
> 
> From: Changchun Ouyang <changchun.ouyang at intel.com>
> 
> In non-SRIOV environment, VMDq RSS could be enabled by MRQC register.
> In theory, the queue number per pool could be 2 or 4, but only 2 queues
> are available due to HW limitation, the same limit also exists in Linux
> ixgbe driver.
> 
> Signed-off-by: Changchun Ouyang <changchun.ouyang at intel.com>
> Signed-off-by: Yuanhan Liu <yuanhan.liu at linux.intel.com>
> ---
>  drivers/net/ixgbe/ixgbe_rxtx.c | 86 +++++++++++++++++++++++++++++++++++-------
>  lib/librte_ether/rte_ethdev.c  | 11 ++++++
>  2 files changed, 84 insertions(+), 13 deletions(-)
> 
> diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c
> index a598a72..e502fe8 100644
> --- a/drivers/net/ixgbe/ixgbe_rxtx.c
> +++ b/drivers/net/ixgbe/ixgbe_rxtx.c
> @@ -3445,16 +3445,16 @@ void ixgbe_configure_dcb(struct rte_eth_dev *dev)
>  	return;
>  }
> 
> -/*
> - * VMDq only support for 10 GbE NIC.
> +/**
> + * Config pool for VMDq on 10 GbE NIC.
>   */
>  static void
> -ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
> +ixgbe_vmdq_pool_configure(struct rte_eth_dev *dev)
>  {
>  	struct rte_eth_vmdq_rx_conf *cfg;
>  	struct ixgbe_hw *hw;
>  	enum rte_eth_nb_pools num_pools;
> -	uint32_t mrqc, vt_ctl, vlanctrl;
> +	uint32_t vt_ctl, vlanctrl;
>  	uint32_t vmolr = 0;
>  	int i;
> 
> @@ -3463,12 +3463,6 @@ ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
>  	cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
>  	num_pools = cfg->nb_queue_pools;
> 
> -	ixgbe_rss_disable(dev);
> -
> -	/* MRQC: enable vmdq */
> -	mrqc = IXGBE_MRQC_VMDQEN;
> -	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
> -
>  	/* PFVTCTL: turn on virtualisation and set the default pool */
>  	vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
>  	if (cfg->enable_default_pool)
> @@ -3534,7 +3528,29 @@ ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
>  	IXGBE_WRITE_FLUSH(hw);
>  }
> 
> -/*
> +/**
> + * VMDq only support for 10 GbE NIC.
> + */
> +static void
> +ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
> +{
> +	struct ixgbe_hw *hw;
> +	uint32_t mrqc;
> +
> +	PMD_INIT_FUNC_TRACE();
> +	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> +
> +	ixgbe_rss_disable(dev);
> +
> +	/* MRQC: enable vmdq */
> +	mrqc = IXGBE_MRQC_VMDQEN;
> +	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
> +	IXGBE_WRITE_FLUSH(hw);
> +
> +	ixgbe_vmdq_pool_configure(dev);
> +}
> +
> +/**
>   * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters
>   * @hw: pointer to hardware structure
>   */
> @@ -3639,6 +3655,41 @@ ixgbe_config_vf_rss(struct rte_eth_dev *dev)
>  }
> 
>  static int
> +ixgbe_config_vmdq_rss(struct rte_eth_dev *dev)
> +{
> +	struct ixgbe_hw *hw;
> +	uint32_t mrqc;
> +
> +	ixgbe_rss_configure(dev);
> +
> +	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> +
> +	/* MRQC: enable VMDQ RSS */
> +	mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
> +	mrqc &= ~IXGBE_MRQC_MRQE_MASK;
> +
> +	switch (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) {
> +	case 2:
> +		mrqc |= IXGBE_MRQC_VMDQRSS64EN;
> +		break;
> +
> +	case 4:
> +		mrqc |= IXGBE_MRQC_VMDQRSS32EN;
> +		break;
> +
> +	default:
> +		PMD_INIT_LOG(ERR, "Invalid pool number in non-IOV mode with VMDQ RSS");
> +		return -EINVAL;
> +	}
> +
> +	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
> +
> +	ixgbe_vmdq_pool_configure(dev);
> +
> +	return 0;
> +}

So ixgbe_config_vmdq_rss() checks nb_q_per_pool value, and might return an error if the value is invalid.
Though this return value seems just ignored by ixgbe_dev_mq_rx_configure() below.
Probably, it is better to move nb_q_per_pool value checking into rte_eth_dev_check_mq_mode(),
as is done for other modes?
I know it is no ideal, as it probably should be HW specific check,
but seems anyway better than just ignoring the error.

Konstantin

> +
> +static int
>  ixgbe_config_vf_default(struct rte_eth_dev *dev)
>  {
>  	struct ixgbe_hw *hw =
> @@ -3694,6 +3745,10 @@ ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
>  				ixgbe_vmdq_rx_hw_configure(dev);
>  				break;
> 
> +			case ETH_MQ_RX_VMDQ_RSS:
> +				ixgbe_config_vmdq_rss(dev);
> +				break;
> +
>  			case ETH_MQ_RX_NONE:
>  				/* if mq_mode is none, disable rss mode.*/
>  			default: ixgbe_rss_disable(dev);
> @@ -4186,6 +4241,8 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
> 
>  	/* Setup RX queues */
>  	for (i = 0; i < dev->data->nb_rx_queues; i++) {
> +		uint32_t psrtype = 0;
> +
>  		rxq = dev->data->rx_queues[i];
> 
>  		/*
> @@ -4213,12 +4270,10 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
>  		if (rx_conf->header_split) {
>  			if (hw->mac.type == ixgbe_mac_82599EB) {
>  				/* Must setup the PSRTYPE register */
> -				uint32_t psrtype;
>  				psrtype = IXGBE_PSRTYPE_TCPHDR |
>  					IXGBE_PSRTYPE_UDPHDR   |
>  					IXGBE_PSRTYPE_IPV4HDR  |
>  					IXGBE_PSRTYPE_IPV6HDR;
> -				IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
>  			}
>  			srrctl = ((rx_conf->split_hdr_size <<
>  				IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
> @@ -4228,6 +4283,11 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
>  #endif
>  			srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
> 
> +		/* Set RQPL for VMDQ RSS according to max Rx queue */
> +		psrtype |= (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool >> 1) <<
> +			IXGBE_PSRTYPE_RQPL_SHIFT;
> +		IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
> +
>  		/* Set if packets are dropped when no descriptors available */
>  		if (rxq->drop_en)
>  			srrctl |= IXGBE_SRRCTL_DROP_EN;
> diff --git a/lib/librte_ether/rte_ethdev.c b/lib/librte_ether/rte_ethdev.c
> index f593f6e..fe9dc5c 100644
> --- a/lib/librte_ether/rte_ethdev.c
> +++ b/lib/librte_ether/rte_ethdev.c
> @@ -1067,6 +1067,17 @@ rte_eth_dev_check_mq_mode(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
>  				return -EINVAL;
>  			}
>  		}
> +
> +		if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_RSS) {
> +			uint32_t nb_queue_pools =
> +				dev_conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools;
> +			struct rte_eth_dev_info dev_info;
> +
> +			rte_eth_dev_info_get(port_id, &dev_info);
> +			dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
> +			RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool =
> +				dev_info.vmdq_queue_num / nb_queue_pools;
> +		}
>  	}
>  	return 0;
>  }
> --
> 1.9.0