From 3da25b109163d5d0a0600501dcc6d2d913c43663 Mon Sep 17 00:00:00 2001 From: Alexander Kozyrev Date: Wed, 1 Nov 2023 16:57:10 +0200 Subject: [PATCH] net/mlx5: replenish MPRQ buffers for miniCQEs Keep unzipping if the next CQE is the miniCQE array in rxq_cq_decompress_v() routine only for non-MPRQ scenario, MPRQ requires buffer replenishment between the miniCQEs. Restore the check for the initial compressed CQE for SPRQ and check that the current CQE is not compressed before copying it as a possible title CQE. Signed-off-by: Alexander Kozyrev Acked-by: Dariusz Sosnowski --- drivers/net/mlx5/mlx5_rxtx_vec.c | 56 ++++++++++++++++++------ drivers/net/mlx5/mlx5_rxtx_vec_altivec.h | 6 ++- drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 6 ++- drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 6 ++- 4 files changed, 54 insertions(+), 20 deletions(-) diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c index 2363d7ed27a..1872bf310ca 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.c +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c @@ -331,6 +331,15 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, } /* At this point, there shouldn't be any remaining packets. */ MLX5_ASSERT(rxq->decompressed == 0); + /* Go directly to unzipping in case the first CQE is compressed. */ + if (rxq->cqe_comp_layout) { + ret = check_cqe_iteration(cq, rxq->cqe_n, rxq->cq_ci); + if (ret == MLX5_CQE_STATUS_SW_OWN && + (MLX5_CQE_FORMAT(cq->op_own) == MLX5_COMPRESSED)) { + comp_idx = 0; + goto decompress; + } + } /* Process all the CQEs */ nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); /* If no new CQE seen, return without updating cq_db. */ @@ -345,18 +354,23 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, rcvd_pkt += nocmp_n; /* Copy title packet for future compressed sessions. */ if (rxq->cqe_comp_layout) { - next = &(*rxq->cqes)[rxq->cq_ci & q_mask]; - ret = check_cqe_iteration(next, rxq->cqe_n, rxq->cq_ci); - if (ret != MLX5_CQE_STATUS_SW_OWN || - MLX5_CQE_FORMAT(next->op_own) == MLX5_COMPRESSED) - rte_memcpy(&rxq->title_pkt, elts[nocmp_n - 1], - sizeof(struct rte_mbuf)); + ret = check_cqe_iteration(cq, rxq->cqe_n, rxq->cq_ci); + if (ret == MLX5_CQE_STATUS_SW_OWN && + (MLX5_CQE_FORMAT(cq->op_own) != MLX5_COMPRESSED)) { + next = &(*rxq->cqes)[rxq->cq_ci & q_mask]; + ret = check_cqe_iteration(next, rxq->cqe_n, rxq->cq_ci); + if (MLX5_CQE_FORMAT(next->op_own) == MLX5_COMPRESSED || + ret != MLX5_CQE_STATUS_SW_OWN) + rte_memcpy(&rxq->title_pkt, elts[nocmp_n - 1], + sizeof(struct rte_mbuf)); + } } +decompress: /* Decompress the last CQE if compressed. */ if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], - &elts[nocmp_n]); + &elts[nocmp_n], true); rxq->cq_ci += rxq->decompressed; /* Return more packets if needed. */ if (nocmp_n < pkts_n) { @@ -482,6 +496,15 @@ rxq_burst_mprq_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, } /* At this point, there shouldn't be any remaining packets. */ MLX5_ASSERT(rxq->decompressed == 0); + /* Go directly to unzipping in case the first CQE is compressed. */ + if (rxq->cqe_comp_layout) { + ret = check_cqe_iteration(cq, rxq->cqe_n, rxq->cq_ci); + if (ret == MLX5_CQE_STATUS_SW_OWN && + (MLX5_CQE_FORMAT(cq->op_own) == MLX5_COMPRESSED)) { + comp_idx = 0; + goto decompress; + } + } /* Process all the CQEs */ nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); /* If no new CQE seen, return without updating cq_db. */ @@ -495,18 +518,23 @@ rxq_burst_mprq_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, rcvd_pkt += cp_pkt; /* Copy title packet for future compressed sessions. */ if (rxq->cqe_comp_layout) { - next = &(*rxq->cqes)[rxq->cq_ci & q_mask]; - ret = check_cqe_iteration(next, rxq->cqe_n, rxq->cq_ci); - if (ret != MLX5_CQE_STATUS_SW_OWN || - MLX5_CQE_FORMAT(next->op_own) == MLX5_COMPRESSED) - rte_memcpy(&rxq->title_pkt, elts[nocmp_n - 1], - sizeof(struct rte_mbuf)); + ret = check_cqe_iteration(cq, rxq->cqe_n, rxq->cq_ci); + if (ret == MLX5_CQE_STATUS_SW_OWN && + (MLX5_CQE_FORMAT(cq->op_own) != MLX5_COMPRESSED)) { + next = &(*rxq->cqes)[rxq->cq_ci & q_mask]; + ret = check_cqe_iteration(next, rxq->cqe_n, rxq->cq_ci); + if (MLX5_CQE_FORMAT(next->op_own) == MLX5_COMPRESSED || + ret != MLX5_CQE_STATUS_SW_OWN) + rte_memcpy(&rxq->title_pkt, elts[nocmp_n - 1], + sizeof(struct rte_mbuf)); + } } +decompress: /* Decompress the last CQE if compressed. */ if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], - &elts[nocmp_n]); + &elts[nocmp_n], false); /* Return more packets if needed. */ if (nocmp_n < pkts_n) { uint16_t n = rxq->decompressed; diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h index cccfa7f2d31..b2bbc4ba17a 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h @@ -68,13 +68,15 @@ rxq_copy_mbuf_v(struct rte_mbuf **elts, struct rte_mbuf **pkts, uint16_t n) * @param elts * Pointer to SW ring to be filled. The first mbuf has to be pre-built from * the title completion descriptor to be copied to the rest of mbufs. + * @param keep + * Keep unzipping if the next CQE is the miniCQE array. * * @return * Number of mini-CQEs successfully decompressed. */ static inline uint16_t rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, - struct rte_mbuf **elts) + struct rte_mbuf **elts, bool keep) { volatile struct mlx5_mini_cqe8 *mcq = (void *)&(cq + !rxq->cqe_comp_layout)->pkt_info; @@ -507,7 +509,7 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, } } - if (rxq->cqe_comp_layout) { + if (rxq->cqe_comp_layout && keep) { int ret; /* Keep unzipping if the next CQE is the miniCQE array. */ cq = &cq[mcqe_n]; diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h index 3ed688191f1..510f60b25d1 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h @@ -63,13 +63,15 @@ rxq_copy_mbuf_v(struct rte_mbuf **elts, struct rte_mbuf **pkts, uint16_t n) * @param elts * Pointer to SW ring to be filled. The first mbuf has to be pre-built from * the title completion descriptor to be copied to the rest of mbufs. + * @param keep + * Keep unzipping if the next CQE is the miniCQE array. * * @return * Number of mini-CQEs successfully decompressed. */ static inline uint16_t rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, - struct rte_mbuf **elts) + struct rte_mbuf **elts, bool keep) { volatile struct mlx5_mini_cqe8 *mcq = (void *)&(cq + !rxq->cqe_comp_layout)->pkt_info; @@ -372,7 +374,7 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, } } } - if (rxq->cqe_comp_layout) { + if (rxq->cqe_comp_layout && keep) { int ret; /* Keep unzipping if the next CQE is the miniCQE array. */ cq = &cq[mcqe_n]; diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h index 93d6d1b5f09..e71d6c303fd 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h @@ -65,13 +65,15 @@ rxq_copy_mbuf_v(struct rte_mbuf **elts, struct rte_mbuf **pkts, uint16_t n) * @param elts * Pointer to SW ring to be filled. The first mbuf has to be pre-built from * the title completion descriptor to be copied to the rest of mbufs. + * @param keep + * Keep unzipping if the next CQE is the miniCQE array. * * @return * Number of mini-CQEs successfully decompressed. */ static inline uint16_t rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, - struct rte_mbuf **elts) + struct rte_mbuf **elts, bool keep) { volatile struct mlx5_mini_cqe8 *mcq = (void *)(cq + !rxq->cqe_comp_layout); /* Title packet is pre-built. */ @@ -361,7 +363,7 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, } } } - if (rxq->cqe_comp_layout) { + if (rxq->cqe_comp_layout && keep) { int ret; /* Keep unzipping if the next CQE is the miniCQE array. */ cq = &cq[mcqe_n];