From 075c734e0f5db59a9c87364062632a864c57eeba Mon Sep 17 00:00:00 2001 From: Jessie Yang Date: Mon, 30 Sep 2024 13:04:34 -0700 Subject: [PATCH] prov/efa: Remove tx_size and rx_size from efa_rdm_ep tx_size is a duplication of info->tx_attr->size. rx_size is a duplication of info->rx_attr->size. Access them via fi_info to save the memory of efa_rdm_ep. Signed-off-by: Jessie Yang --- prov/efa/src/rdm/efa_rdm_ep.h | 11 ++--------- prov/efa/src/rdm/efa_rdm_ep_fiops.c | 10 ++++------ prov/efa/src/rdm/efa_rdm_srx.c | 2 +- prov/efa/test/efa_unit_test_ep.c | 8 +++++--- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/prov/efa/src/rdm/efa_rdm_ep.h b/prov/efa/src/rdm/efa_rdm_ep.h index 75ac34857e1..eba552d39ad 100644 --- a/prov/efa/src/rdm/efa_rdm_ep.h +++ b/prov/efa/src/rdm/efa_rdm_ep.h @@ -55,13 +55,6 @@ struct efa_rdm_ep { /* shm provider fid */ struct fid_ep *shm_ep; - /* - * EFA RDM endpoint rx/tx queue sizes. These may be different from the core - * provider's rx/tx size and will either limit the number of possible - * receives/sends or allow queueing. - */ - size_t rx_size; - size_t tx_size; size_t mtu_size; size_t inject_size; @@ -227,12 +220,12 @@ void efa_rdm_ep_record_tx_op_completed(struct efa_rdm_ep *ep, struct efa_rdm_pke static inline size_t efa_rdm_ep_get_rx_pool_size(struct efa_rdm_ep *ep) { - return MIN(ep->efa_max_outstanding_rx_ops, ep->rx_size); + return MIN(ep->efa_max_outstanding_rx_ops, ep->base_ep.info->rx_attr->size); } static inline size_t efa_rdm_ep_get_tx_pool_size(struct efa_rdm_ep *ep) { - return MIN(ep->efa_max_outstanding_tx_ops, ep->tx_size); + return MIN(ep->efa_max_outstanding_tx_ops, ep->base_ep.info->tx_attr->size); } static inline int efa_rdm_ep_need_sas(struct efa_rdm_ep *ep) diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c index 15a20256c87..abc4f89c720 100644 --- a/prov/efa/src/rdm/efa_rdm_ep_fiops.c +++ b/prov/efa/src/rdm/efa_rdm_ep_fiops.c @@ -236,7 +236,7 @@ int efa_rdm_ep_create_buffer_pools(struct efa_rdm_ep *ep) ret = ofi_bufpool_create(&ep->user_rx_pkt_pool, sizeof(struct efa_rdm_pke), EFA_RDM_BUFPOOL_ALIGNMENT, - 0,ep->rx_size,0); + 0, ep->base_ep.info->rx_attr->size, 0); if (ret) goto err_free; @@ -285,7 +285,7 @@ int efa_rdm_ep_create_buffer_pools(struct efa_rdm_ep *ep) sizeof(struct efa_rdm_rxe_map_entry), EFA_RDM_BUFPOOL_ALIGNMENT, 0, /* no limit for max_cnt */ - ep->rx_size, 0); + ep->base_ep.info->rx_attr->size, 0); if (ret) goto err_free; @@ -301,7 +301,7 @@ int efa_rdm_ep_create_buffer_pools(struct efa_rdm_ep *ep) sizeof(struct efa_rdm_ope), EFA_RDM_BUFPOOL_ALIGNMENT, 0, /* no limit for max_cnt */ - ep->tx_size + ep->rx_size, 0); + ep->base_ep.info->tx_attr->size + ep->base_ep.info->rx_attr->size, 0); if (ret) goto err_free; @@ -309,7 +309,7 @@ int efa_rdm_ep_create_buffer_pools(struct efa_rdm_ep *ep) sizeof(struct efa_rdm_peer_overflow_pke_list_entry), EFA_RDM_BUFPOOL_ALIGNMENT, 0, /* no limit for max_cnt */ - ep->rx_size, 0); + ep->base_ep.info->rx_attr->size, 0); if (ret) goto err_free; @@ -557,8 +557,6 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info, EFA_INFO(FI_LOG_EP_CTRL, "efa_rdm_ep->host_id: i-%017lx\n", efa_rdm_ep->host_id); } - efa_rdm_ep->rx_size = info->rx_attr->size; - efa_rdm_ep->tx_size = info->tx_attr->size; efa_rdm_ep->inject_size = info->tx_attr->inject_size; efa_rdm_ep->efa_max_outstanding_tx_ops = efa_domain->device->rdm_info->tx_attr->size; efa_rdm_ep->efa_max_outstanding_rx_ops = efa_domain->device->rdm_info->rx_attr->size; diff --git a/prov/efa/src/rdm/efa_rdm_srx.c b/prov/efa/src/rdm/efa_rdm_srx.c index 47919dc5667..4efbe01da72 100644 --- a/prov/efa/src/rdm/efa_rdm_srx.c +++ b/prov/efa/src/rdm/efa_rdm_srx.c @@ -151,7 +151,7 @@ int efa_rdm_peer_srx_construct(struct efa_rdm_ep *ep) { int ret; ret = util_ep_srx_context(&efa_rdm_ep_domain(ep)->util_domain, - ep->rx_size, EFA_RDM_IOV_LIMIT, + ep->base_ep.info->rx_attr->size, EFA_RDM_IOV_LIMIT, ep->min_multi_recv_size, &efa_rdm_srx_update_mr, &efa_rdm_ep_domain(ep)->srx_lock, diff --git a/prov/efa/test/efa_unit_test_ep.c b/prov/efa/test/efa_unit_test_ep.c index 75f100a80f0..f3a70183c64 100644 --- a/prov/efa/test/efa_unit_test_ep.c +++ b/prov/efa/test/efa_unit_test_ep.c @@ -1126,6 +1126,7 @@ void test_efa_rdm_ep_post_handshake_error_handling_pke_exhaustion(struct efa_res int err, numaddr; struct efa_rdm_pke **pkt_entry_vec; int i; + size_t tx_size; /* disable shm to force using efa device to send */ efa_unit_test_resource_construct_rdm_shm_disabled(resource); @@ -1139,6 +1140,7 @@ void test_efa_rdm_ep_post_handshake_error_handling_pke_exhaustion(struct efa_res assert_int_equal(numaddr, 1); efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); + tx_size = efa_rdm_ep->base_ep.info->tx_attr->size; /* set peer->flag to EFA_RDM_PEER_REQ_SENT will make efa_rdm_atomic() think * a REQ packet has been sent to the peer (so no need to send again) @@ -1148,11 +1150,11 @@ void test_efa_rdm_ep_post_handshake_error_handling_pke_exhaustion(struct efa_res peer->flags = EFA_RDM_PEER_REQ_SENT; peer->is_local = false; - pkt_entry_vec = calloc(efa_rdm_ep->tx_size, sizeof(struct efa_rdm_pke *)); + pkt_entry_vec = calloc(tx_size, sizeof(struct efa_rdm_pke *)); assert_non_null(pkt_entry_vec); /* Exhaust the tx pkt pool */ - for (i = 0; i < efa_rdm_ep->tx_size; i++) { + for (i = 0; i < tx_size; i++) { pkt_entry_vec[i] = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_tx_pkt_pool, EFA_RDM_PKE_FROM_EFA_TX_POOL); assert_non_null(pkt_entry_vec[i]); } @@ -1162,7 +1164,7 @@ void test_efa_rdm_ep_post_handshake_error_handling_pke_exhaustion(struct efa_res assert_int_equal(efa_rdm_ep_post_handshake(efa_rdm_ep, peer), -FI_EAGAIN); assert_true(dlist_empty(&efa_rdm_ep->txe_list)); - for (i = 0; i < efa_rdm_ep->tx_size; i++) + for (i = 0; i < tx_size; i++) efa_rdm_pke_release_tx(pkt_entry_vec[i]); free(pkt_entry_vec);