[dpdk-dev] [PATCH 06/14] fm10k: add Vector RX function

[Ananyev, Konstantin]

> -----Original Message-----
> From: dev [mailto:dev-bounces at dpdk.org] On Behalf Of Chen Jing D(Mark)
> Sent: Tuesday, September 29, 2015 2:04 PM
> To: dev at dpdk.org
> Subject: [dpdk-dev] [PATCH 06/14] fm10k: add Vector RX function
> 
> From: "Chen Jing D(Mark)" <jing.d.chen at intel.com>
> 
> Add func fm10k_recv_raw_pkts_vec to parse raw packets, in which
> includes possible chained packets.
> Add func fm10k_recv_pkts_vec to receive single mbuf packet.
> 
> Signed-off-by: Chen Jing D(Mark) <jing.d.chen at intel.com>
> ---
>  drivers/net/fm10k/fm10k.h          |    1 +
>  drivers/net/fm10k/fm10k_rxtx_vec.c |  213 ++++++++++++++++++++++++++++++++++++
>  2 files changed, 214 insertions(+), 0 deletions(-)
> 
> diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
> index d924cae..285254f 100644
> --- a/drivers/net/fm10k/fm10k.h
> +++ b/drivers/net/fm10k/fm10k.h
> @@ -327,4 +327,5 @@ uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
>  	uint16_t nb_pkts);
> 
>  int fm10k_rxq_vec_setup(struct fm10k_rx_queue *rxq);
> +uint16_t fm10k_recv_pkts_vec(void *, struct rte_mbuf **, uint16_t);
>  #endif
> diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
> index 581a309..63b34b5 100644
> --- a/drivers/net/fm10k/fm10k_rxtx_vec.c
> +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
> @@ -281,3 +281,216 @@ fm10k_rxq_rearm(struct fm10k_rx_queue *rxq)
>  	/* Update the tail pointer on the NIC */
>  	FM10K_PCI_REG_WRITE(rxq->tail_ptr, rx_id);
>  }
> +
> +/*
> + * vPMD receive routine, now only accept (nb_pkts == RTE_IXGBE_VPMD_RX_BURST)
> + * in one loop
> + *
> + * Notice:
> + * - nb_pkts < RTE_IXGBE_VPMD_RX_BURST, just return no packet
> + * - nb_pkts > RTE_IXGBE_VPMD_RX_BURST, only scan RTE_IXGBE_VPMD_RX_BURST
> + *   numbers of DD bit
> + * - don't support ol_flags for rss and csum err
> + */
> +static inline uint16_t
> +fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
> +		uint16_t nb_pkts, uint8_t *split_packet)
> +{
> +	volatile union fm10k_rx_desc *rxdp;
> +	struct rte_mbuf **mbufp;
> +	uint16_t nb_pkts_recd;
> +	int pos;
> +	struct fm10k_rx_queue *rxq = rx_queue;
> +	uint64_t var;
> +	__m128i shuf_msk;
> +	__m128i dd_check, eop_check;
> +	uint16_t next_dd;
> +
> +	next_dd = rxq->next_dd;
> +
> +	if (unlikely(nb_pkts < RTE_FM10K_MAX_RX_BURST))
> +		return 0;
> +
> +	/* Just the act of getting into the function from the application is
> +	 * going to cost about 7 cycles
> +	 */
> +	rxdp = rxq->hw_ring + next_dd;
> +
> +	_mm_prefetch((const void *)rxdp, _MM_HINT_T0);
> +
> +	/* See if we need to rearm the RX queue - gives the prefetch a bit
> +	 * of time to act
> +	 */
> +	if (rxq->rxrearm_nb > RTE_FM10K_RXQ_REARM_THRESH)
> +		fm10k_rxq_rearm(rxq);
> +
> +	/* Before we start moving massive data around, check to see if
> +	 * there is actually a packet available
> +	 */
> +	if (!(rxdp->d.staterr & FM10K_RXD_STATUS_DD))
> +		return 0;
> +
> +	/* 4 packets DD mask */
> +	dd_check = _mm_set_epi64x(0x0000000100000001LL, 0x0000000100000001LL);
> +
> +	/* 4 packets EOP mask */
> +	eop_check = _mm_set_epi64x(0x0000000200000002LL, 0x0000000200000002LL);
> +
> +	/* mask to shuffle from desc. to mbuf */
> +	shuf_msk = _mm_set_epi8(
> +		7, 6, 5, 4,  /* octet 4~7, 32bits rss */
> +		15, 14,      /* octet 14~15, low 16 bits vlan_macip */
> +		13, 12,      /* octet 12~13, 16 bits data_len */
> +		0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
> +		13, 12,      /* octet 12~13, low 16 bits pkt_len */
> +		0xFF, 0xFF,  /* skip high 16 bits pkt_type */
> +		0xFF, 0xFF   /* Skip pkt_type field in shuffle operation */
> +		);
> +
> +	/* Cache is empty -> need to scan the buffer rings, but first move
> +	 * the next 'n' mbufs into the cache
> +	 */
> +	mbufp = &rxq->sw_ring[next_dd];
> +
> +	/* A. load 4 packet in one loop
> +	 * [A*. mask out 4 unused dirty field in desc]
> +	 * B. copy 4 mbuf point from swring to rx_pkts
> +	 * C. calc the number of DD bits among the 4 packets
> +	 * [C*. extract the end-of-packet bit, if requested]
> +	 * D. fill info. from desc to mbuf
> +	 */
> +	for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
> +			pos += RTE_FM10K_DESCS_PER_LOOP,
> +			rxdp += RTE_FM10K_DESCS_PER_LOOP) {
> +		__m128i descs0[RTE_FM10K_DESCS_PER_LOOP];
> +		__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
> +		__m128i zero, staterr, sterr_tmp1, sterr_tmp2;
> +		__m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */
> +
> +		if (split_packet) {
> +			rte_prefetch0(&rx_pkts[pos]->cacheline1);
> +			rte_prefetch0(&rx_pkts[pos + 1]->cacheline1);
> +			rte_prefetch0(&rx_pkts[pos + 2]->cacheline1);
> +			rte_prefetch0(&rx_pkts[pos + 3]->cacheline1);
> +		}


Same thing as with i40e vPMD:
You are pretching junk addreses here.
Check out Zoltan's patch:
http://dpdk.org/dev/patchwork/patch/7190/
and related conversation:
http://dpdk.org/ml/archives/dev/2015-September/023715.html
I think there is the same issue here.
Konstantin 

> +
> +		/* B.1 load 1 mbuf point */
> +		mbp1 = _mm_loadu_si128((__m128i *)&mbufp[pos]);
> +
> +		/* Read desc statuses backwards to avoid race condition */
> +		/* A.1 load 4 pkts desc */
> +		descs0[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
> +
> +		/* B.2 copy 2 mbuf point into rx_pkts  */
> +		_mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
> +
> +		/* B.1 load 1 mbuf point */
> +		mbp2 = _mm_loadu_si128((__m128i *)&mbufp[pos+2]);
> +
> +		descs0[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
> +		/* B.1 load 2 mbuf point */
> +		descs0[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));
> +		descs0[0] = _mm_loadu_si128((__m128i *)(rxdp));
> +
> +		/* B.2 copy 2 mbuf point into rx_pkts  */
> +		_mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2);
> +
> +		/* avoid compiler reorder optimization */
> +		rte_compiler_barrier();
> +
> +		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
> +		pkt_mb4 = _mm_shuffle_epi8(descs0[3], shuf_msk);
> +		pkt_mb3 = _mm_shuffle_epi8(descs0[2], shuf_msk);
> +
> +		/* C.1 4=>2 filter staterr info only */
> +		sterr_tmp2 = _mm_unpackhi_epi32(descs0[3], descs0[2]);
> +		/* C.1 4=>2 filter staterr info only */
> +		sterr_tmp1 = _mm_unpackhi_epi32(descs0[1], descs0[0]);
> +
> +		/* set ol_flags with vlan packet type */
> +		fm10k_desc_to_olflags_v(descs0, &rx_pkts[pos]);
> +
> +		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
> +		pkt_mb2 = _mm_shuffle_epi8(descs0[1], shuf_msk);
> +		pkt_mb1 = _mm_shuffle_epi8(descs0[0], shuf_msk);
> +
> +		/* C.2 get 4 pkts staterr value  */
> +		zero = _mm_xor_si128(dd_check, dd_check);
> +		staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
> +
> +		/* D.3 copy final 3,4 data to rx_pkts */
> +		_mm_storeu_si128((void *)&rx_pkts[pos+3]->rx_descriptor_fields1,
> +				pkt_mb4);
> +		_mm_storeu_si128((void *)&rx_pkts[pos+2]->rx_descriptor_fields1,
> +				pkt_mb3);
> +
> +		/* C* extract and record EOP bit */
> +		if (split_packet) {
> +			__m128i eop_shuf_mask = _mm_set_epi8(
> +					0xFF, 0xFF, 0xFF, 0xFF,
> +					0xFF, 0xFF, 0xFF, 0xFF,
> +					0xFF, 0xFF, 0xFF, 0xFF,
> +					0x04, 0x0C, 0x00, 0x08
> +					);
> +
> +			/* and with mask to extract bits, flipping 1-0 */
> +			__m128i eop_bits = _mm_andnot_si128(staterr, eop_check);
> +			/* the staterr values are not in order, as the count
> +			 * count of dd bits doesn't care. However, for end of
> +			 * packet tracking, we do care, so shuffle. This also
> +			 * compresses the 32-bit values to 8-bit
> +			 */
> +			eop_bits = _mm_shuffle_epi8(eop_bits, eop_shuf_mask);
> +			/* store the resulting 32-bit value */
> +			*(int *)split_packet = _mm_cvtsi128_si32(eop_bits);
> +			split_packet += RTE_FM10K_DESCS_PER_LOOP;
> +
> +			/* zero-out next pointers */
> +			rx_pkts[pos]->next = NULL;
> +			rx_pkts[pos + 1]->next = NULL;
> +			rx_pkts[pos + 2]->next = NULL;
> +			rx_pkts[pos + 3]->next = NULL;
> +		}
> +
> +		/* C.3 calc available number of desc */
> +		staterr = _mm_and_si128(staterr, dd_check);
> +		staterr = _mm_packs_epi32(staterr, zero);
> +
> +		/* D.3 copy final 1,2 data to rx_pkts */
> +		_mm_storeu_si128((void *)&rx_pkts[pos+1]->rx_descriptor_fields1,
> +				pkt_mb2);
> +		_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
> +				pkt_mb1);
> +
> +		fm10k_desc_to_pktype_v(descs0, &rx_pkts[pos]);
> +
> +		/* C.4 calc avaialbe number of desc */
> +		var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
> +		nb_pkts_recd += var;
> +		if (likely(var != RTE_FM10K_DESCS_PER_LOOP))
> +			break;
> +	}
> +
> +	/* Update our internal tail pointer */
> +	rxq->next_dd = (uint16_t)(rxq->next_dd + nb_pkts_recd);
> +	rxq->next_dd = (uint16_t)(rxq->next_dd & (rxq->nb_desc - 1));
> +	rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
> +
> +	return nb_pkts_recd;
> +}
> +
> +/* vPMD receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP)
> + *
> + * Notice:
> + * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet
> + * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
> + *   numbers of DD bit
> + * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two
> + * - don't support ol_flags for rss and csum err
> + */
> +uint16_t
> +fm10k_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
> +		uint16_t nb_pkts)
> +{
> +	return fm10k_recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);
> +}
> --
> 1.7.7.6