Skip to content

Commit

Permalink
prov/efa: Remove tx_size and rx_size from efa_rdm_ep
Browse files Browse the repository at this point in the history 
tx_size is a duplication of info->tx_attr->size.
rx_size is a duplication of info->rx_attr->size.
Access them via fi_info to save the memory of efa_rdm_ep.

Signed-off-by: Jessie Yang <jiaxiyan@amazon.com>
  • Loading branch information
@jiaxiyan @shijin-aws
jiaxiyan authored and shijin-aws committed Oct 1, 2024
1 parent 1ef91c8 commit 075c734
Show file tree
Hide file tree
Showing 4 changed files with 12 additions and 19 deletions.
11 changes: 2 additions & 9 deletions prov/efa/src/rdm/efa_rdm_ep.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,6 @@ struct efa_rdm_ep {
/* shm provider fid */
struct fid_ep *shm_ep;

/*
* EFA RDM endpoint rx/tx queue sizes. These may be different from the core
* provider's rx/tx size and will either limit the number of possible
* receives/sends or allow queueing.
*/
size_t rx_size;
size_t tx_size;
size_t mtu_size;
size_t inject_size;

Expand Down Expand Up @@ -227,12 +220,12 @@ void efa_rdm_ep_record_tx_op_completed(struct efa_rdm_ep *ep, struct efa_rdm_pke

static inline size_t efa_rdm_ep_get_rx_pool_size(struct efa_rdm_ep *ep)
{
return MIN(ep->efa_max_outstanding_rx_ops, ep->rx_size);
return MIN(ep->efa_max_outstanding_rx_ops, ep->base_ep.info->rx_attr->size);
}

static inline size_t efa_rdm_ep_get_tx_pool_size(struct efa_rdm_ep *ep)
{
return MIN(ep->efa_max_outstanding_tx_ops, ep->tx_size);
return MIN(ep->efa_max_outstanding_tx_ops, ep->base_ep.info->tx_attr->size);
}

static inline int efa_rdm_ep_need_sas(struct efa_rdm_ep *ep)
Expand Down
10 changes: 4 additions & 6 deletions prov/efa/src/rdm/efa_rdm_ep_fiops.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ int efa_rdm_ep_create_buffer_pools(struct efa_rdm_ep *ep)
ret = ofi_bufpool_create(&ep->user_rx_pkt_pool,
sizeof(struct efa_rdm_pke),
EFA_RDM_BUFPOOL_ALIGNMENT,
0,ep->rx_size,0);
0, ep->base_ep.info->rx_attr->size, 0);
if (ret)
goto err_free;

Expand Down Expand Up @@ -285,7 +285,7 @@ int efa_rdm_ep_create_buffer_pools(struct efa_rdm_ep *ep)
sizeof(struct efa_rdm_rxe_map_entry),
EFA_RDM_BUFPOOL_ALIGNMENT,
0, /* no limit for max_cnt */
ep->rx_size, 0);
ep->base_ep.info->rx_attr->size, 0);

if (ret)
goto err_free;
Expand All @@ -301,15 +301,15 @@ int efa_rdm_ep_create_buffer_pools(struct efa_rdm_ep *ep)
sizeof(struct efa_rdm_ope),
EFA_RDM_BUFPOOL_ALIGNMENT,
0, /* no limit for max_cnt */
ep->tx_size + ep->rx_size, 0);
ep->base_ep.info->tx_attr->size + ep->base_ep.info->rx_attr->size, 0);
if (ret)
goto err_free;

ret = ofi_bufpool_create(&ep->overflow_pke_pool,
sizeof(struct efa_rdm_peer_overflow_pke_list_entry),
EFA_RDM_BUFPOOL_ALIGNMENT,
0, /* no limit for max_cnt */
ep->rx_size, 0);
ep->base_ep.info->rx_attr->size, 0);
if (ret)
goto err_free;

Expand Down Expand Up @@ -557,8 +557,6 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
EFA_INFO(FI_LOG_EP_CTRL, "efa_rdm_ep->host_id: i-%017lx\n", efa_rdm_ep->host_id);
}

efa_rdm_ep->rx_size = info->rx_attr->size;
efa_rdm_ep->tx_size = info->tx_attr->size;
efa_rdm_ep->inject_size = info->tx_attr->inject_size;
efa_rdm_ep->efa_max_outstanding_tx_ops = efa_domain->device->rdm_info->tx_attr->size;
efa_rdm_ep->efa_max_outstanding_rx_ops = efa_domain->device->rdm_info->rx_attr->size;
Expand Down
2 changes: 1 addition & 1 deletion prov/efa/src/rdm/efa_rdm_srx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ int efa_rdm_peer_srx_construct(struct efa_rdm_ep *ep)
{
int ret;
ret = util_ep_srx_context(&efa_rdm_ep_domain(ep)->util_domain,
ep->rx_size, EFA_RDM_IOV_LIMIT,
ep->base_ep.info->rx_attr->size, EFA_RDM_IOV_LIMIT,
ep->min_multi_recv_size,
&efa_rdm_srx_update_mr,
&efa_rdm_ep_domain(ep)->srx_lock,
Expand Down
8 changes: 5 additions & 3 deletions prov/efa/test/efa_unit_test_ep.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1126,6 +1126,7 @@ void test_efa_rdm_ep_post_handshake_error_handling_pke_exhaustion(struct efa_res
int err, numaddr;
struct efa_rdm_pke **pkt_entry_vec;
int i;
size_t tx_size;

/* disable shm to force using efa device to send */
efa_unit_test_resource_construct_rdm_shm_disabled(resource);
Expand All @@ -1139,6 +1140,7 @@ void test_efa_rdm_ep_post_handshake_error_handling_pke_exhaustion(struct efa_res
assert_int_equal(numaddr, 1);

efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
tx_size = efa_rdm_ep->base_ep.info->tx_attr->size;

/* set peer->flag to EFA_RDM_PEER_REQ_SENT will make efa_rdm_atomic() think
* a REQ packet has been sent to the peer (so no need to send again)
Expand All @@ -1148,11 +1150,11 @@ void test_efa_rdm_ep_post_handshake_error_handling_pke_exhaustion(struct efa_res
peer->flags = EFA_RDM_PEER_REQ_SENT;
peer->is_local = false;

pkt_entry_vec = calloc(efa_rdm_ep->tx_size, sizeof(struct efa_rdm_pke *));
pkt_entry_vec = calloc(tx_size, sizeof(struct efa_rdm_pke *));
assert_non_null(pkt_entry_vec);

/* Exhaust the tx pkt pool */
for (i = 0; i < efa_rdm_ep->tx_size; i++) {
for (i = 0; i < tx_size; i++) {
pkt_entry_vec[i] = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_tx_pkt_pool, EFA_RDM_PKE_FROM_EFA_TX_POOL);
assert_non_null(pkt_entry_vec[i]);
}
Expand All @@ -1162,7 +1164,7 @@ void test_efa_rdm_ep_post_handshake_error_handling_pke_exhaustion(struct efa_res
assert_int_equal(efa_rdm_ep_post_handshake(efa_rdm_ep, peer), -FI_EAGAIN);
assert_true(dlist_empty(&efa_rdm_ep->txe_list));

for (i = 0; i < efa_rdm_ep->tx_size; i++)
for (i = 0; i < tx_size; i++)
efa_rdm_pke_release_tx(pkt_entry_vec[i]);

free(pkt_entry_vec);
Expand Down

0 comments on commit 075c734

Please sign in to comment.