diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c index 2363d7ed27a..1872bf310ca 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.c +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c @@ -331,6 +331,15 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, } /* At this point, there shouldn't be any remaining packets. */ MLX5_ASSERT(rxq->decompressed == 0); + /* Go directly to unzipping in case the first CQE is compressed. */ + if (rxq->cqe_comp_layout) { + ret = check_cqe_iteration(cq, rxq->cqe_n, rxq->cq_ci); + if (ret == MLX5_CQE_STATUS_SW_OWN && + (MLX5_CQE_FORMAT(cq->op_own) == MLX5_COMPRESSED)) { + comp_idx = 0; + goto decompress; + } + } /* Process all the CQEs */ nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); /* If no new CQE seen, return without updating cq_db. */ @@ -345,18 +354,23 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, rcvd_pkt += nocmp_n; /* Copy title packet for future compressed sessions. */ if (rxq->cqe_comp_layout) { - next = &(*rxq->cqes)[rxq->cq_ci & q_mask]; - ret = check_cqe_iteration(next, rxq->cqe_n, rxq->cq_ci); - if (ret != MLX5_CQE_STATUS_SW_OWN || - MLX5_CQE_FORMAT(next->op_own) == MLX5_COMPRESSED) - rte_memcpy(&rxq->title_pkt, elts[nocmp_n - 1], - sizeof(struct rte_mbuf)); + ret = check_cqe_iteration(cq, rxq->cqe_n, rxq->cq_ci); + if (ret == MLX5_CQE_STATUS_SW_OWN && + (MLX5_CQE_FORMAT(cq->op_own) != MLX5_COMPRESSED)) { + next = &(*rxq->cqes)[rxq->cq_ci & q_mask]; + ret = check_cqe_iteration(next, rxq->cqe_n, rxq->cq_ci); + if (MLX5_CQE_FORMAT(next->op_own) == MLX5_COMPRESSED || + ret != MLX5_CQE_STATUS_SW_OWN) + rte_memcpy(&rxq->title_pkt, elts[nocmp_n - 1], + sizeof(struct rte_mbuf)); + } } +decompress: /* Decompress the last CQE if compressed. */ if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], - &elts[nocmp_n]); + &elts[nocmp_n], true); rxq->cq_ci += rxq->decompressed; /* Return more packets if needed. */ if (nocmp_n < pkts_n) { @@ -482,6 +496,15 @@ rxq_burst_mprq_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, } /* At this point, there shouldn't be any remaining packets. */ MLX5_ASSERT(rxq->decompressed == 0); + /* Go directly to unzipping in case the first CQE is compressed. */ + if (rxq->cqe_comp_layout) { + ret = check_cqe_iteration(cq, rxq->cqe_n, rxq->cq_ci); + if (ret == MLX5_CQE_STATUS_SW_OWN && + (MLX5_CQE_FORMAT(cq->op_own) == MLX5_COMPRESSED)) { + comp_idx = 0; + goto decompress; + } + } /* Process all the CQEs */ nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); /* If no new CQE seen, return without updating cq_db. */ @@ -495,18 +518,23 @@ rxq_burst_mprq_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, rcvd_pkt += cp_pkt; /* Copy title packet for future compressed sessions. */ if (rxq->cqe_comp_layout) { - next = &(*rxq->cqes)[rxq->cq_ci & q_mask]; - ret = check_cqe_iteration(next, rxq->cqe_n, rxq->cq_ci); - if (ret != MLX5_CQE_STATUS_SW_OWN || - MLX5_CQE_FORMAT(next->op_own) == MLX5_COMPRESSED) - rte_memcpy(&rxq->title_pkt, elts[nocmp_n - 1], - sizeof(struct rte_mbuf)); + ret = check_cqe_iteration(cq, rxq->cqe_n, rxq->cq_ci); + if (ret == MLX5_CQE_STATUS_SW_OWN && + (MLX5_CQE_FORMAT(cq->op_own) != MLX5_COMPRESSED)) { + next = &(*rxq->cqes)[rxq->cq_ci & q_mask]; + ret = check_cqe_iteration(next, rxq->cqe_n, rxq->cq_ci); + if (MLX5_CQE_FORMAT(next->op_own) == MLX5_COMPRESSED || + ret != MLX5_CQE_STATUS_SW_OWN) + rte_memcpy(&rxq->title_pkt, elts[nocmp_n - 1], + sizeof(struct rte_mbuf)); + } } +decompress: /* Decompress the last CQE if compressed. */ if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], - &elts[nocmp_n]); + &elts[nocmp_n], false); /* Return more packets if needed. */ if (nocmp_n < pkts_n) { uint16_t n = rxq->decompressed; diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h index cccfa7f2d31..b2bbc4ba17a 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h @@ -68,13 +68,15 @@ rxq_copy_mbuf_v(struct rte_mbuf **elts, struct rte_mbuf **pkts, uint16_t n) * @param elts * Pointer to SW ring to be filled. The first mbuf has to be pre-built from * the title completion descriptor to be copied to the rest of mbufs. + * @param keep + * Keep unzipping if the next CQE is the miniCQE array. * * @return * Number of mini-CQEs successfully decompressed. */ static inline uint16_t rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, - struct rte_mbuf **elts) + struct rte_mbuf **elts, bool keep) { volatile struct mlx5_mini_cqe8 *mcq = (void *)&(cq + !rxq->cqe_comp_layout)->pkt_info; @@ -507,7 +509,7 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, } } - if (rxq->cqe_comp_layout) { + if (rxq->cqe_comp_layout && keep) { int ret; /* Keep unzipping if the next CQE is the miniCQE array. */ cq = &cq[mcqe_n]; diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h index 3ed688191f1..510f60b25d1 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h @@ -63,13 +63,15 @@ rxq_copy_mbuf_v(struct rte_mbuf **elts, struct rte_mbuf **pkts, uint16_t n) * @param elts * Pointer to SW ring to be filled. The first mbuf has to be pre-built from * the title completion descriptor to be copied to the rest of mbufs. + * @param keep + * Keep unzipping if the next CQE is the miniCQE array. * * @return * Number of mini-CQEs successfully decompressed. */ static inline uint16_t rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, - struct rte_mbuf **elts) + struct rte_mbuf **elts, bool keep) { volatile struct mlx5_mini_cqe8 *mcq = (void *)&(cq + !rxq->cqe_comp_layout)->pkt_info; @@ -372,7 +374,7 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, } } } - if (rxq->cqe_comp_layout) { + if (rxq->cqe_comp_layout && keep) { int ret; /* Keep unzipping if the next CQE is the miniCQE array. */ cq = &cq[mcqe_n]; diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h index 93d6d1b5f09..e71d6c303fd 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h @@ -65,13 +65,15 @@ rxq_copy_mbuf_v(struct rte_mbuf **elts, struct rte_mbuf **pkts, uint16_t n) * @param elts * Pointer to SW ring to be filled. The first mbuf has to be pre-built from * the title completion descriptor to be copied to the rest of mbufs. + * @param keep + * Keep unzipping if the next CQE is the miniCQE array. * * @return * Number of mini-CQEs successfully decompressed. */ static inline uint16_t rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, - struct rte_mbuf **elts) + struct rte_mbuf **elts, bool keep) { volatile struct mlx5_mini_cqe8 *mcq = (void *)(cq + !rxq->cqe_comp_layout); /* Title packet is pre-built. */ @@ -361,7 +363,7 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, } } } - if (rxq->cqe_comp_layout) { + if (rxq->cqe_comp_layout && keep) { int ret; /* Keep unzipping if the next CQE is the miniCQE array. */ cq = &cq[mcqe_n];