From 904729fd748c762600ef32b195f5f61caee834db Mon Sep 17 00:00:00 2001 From: Eric Bolten Date: Fri, 11 Nov 2022 13:05:44 -0800 Subject: [PATCH 1/5] Try each message individually if the chunk fails --- orchestrator/cosmos_gravity/src/send.rs | 49 +++++++++++++++++++------ 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/orchestrator/cosmos_gravity/src/send.rs b/orchestrator/cosmos_gravity/src/send.rs index 957dafdbc..3653b4633 100644 --- a/orchestrator/cosmos_gravity/src/send.rs +++ b/orchestrator/cosmos_gravity/src/send.rs @@ -175,6 +175,7 @@ pub async fn send_main_loop( ) { while let Some(messages) = rx.recv().await { for msg_chunk in messages.chunks(msg_batch_size) { + let batch = msg_chunk.to_vec(); match send_messages( contact, cosmos_key, @@ -184,21 +185,45 @@ pub async fn send_main_loop( ) .await { - Ok(res) => trace!("okay: {:?}", res), + Ok(res) => debug!("message batch sent: {:?}", res), Err(err) => { - let msg_types = msg_chunk - .iter() - .map(|msg| prost_types::Any::from(msg.clone()).type_url) - .collect::>(); - - error!( - "Error during gRPC call to Cosmos containing {} messages of types {:?}: {:?}", - msg_chunk.len(), - msg_types, - err - ); + log_send_error(&batch, err); + + // multiple messages in a single Cosmos transaction will be rejected + // atomically if that transaction cannot be delivered, so retry each + // element separately + info!("Trying each message in batch individually"); + for msg in batch { + let msg_vec = vec![msg]; + match send_messages( + contact, + cosmos_key, + gas_price.to_owned(), + msg_vec.clone(), + gas_adjustment, + ) + .await + { + Ok(res) => debug!("message sent: {:?}", res), + Err(err) => {log_send_error(&msg_vec, err)} + } + } } } } } } + +fn log_send_error(messages: &Vec, err: GravityError) { + let msg_types = messages + .iter() + .map(|msg| prost_types::Any::from(msg.clone()).type_url) + .collect::>(); + + error!( + "Error during gRPC call to Cosmos containing {} messages of types {:?}: {:?}", + messages.len(), + msg_types, + err + ); +} \ No newline at end of file From 406a9cd36e5df3d2f7abdbbf5c40a96bd3e1d21a Mon Sep 17 00:00:00 2001 From: Eric Bolten Date: Fri, 11 Nov 2022 12:47:29 -0800 Subject: [PATCH 2/5] Fix validator out race condition --- integration_tests/setup_test.go | 2 +- integration_tests/validator_out.go | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/integration_tests/setup_test.go b/integration_tests/setup_test.go index 370153bd2..b7a9a9b66 100644 --- a/integration_tests/setup_test.go +++ b/integration_tests/setup_test.go @@ -344,7 +344,7 @@ func (s *IntegrationTestSuite) initGenesis() { s.Require().NoError(cdc.UnmarshalJSON(appGenState[gravitytypes.ModuleName], &gravityGenState)) gravityGenState.Params.GravityId = "gravitytest" gravityGenState.Params.BridgeEthereumAddress = gravityContract.String() - gravityGenState.Params.SignedBatchesWindow = 15 + gravityGenState.Params.SignedBatchesWindow = 35 bz, err = cdc.MarshalJSON(&gravityGenState) s.Require().NoError(err) diff --git a/integration_tests/validator_out.go b/integration_tests/validator_out.go index 89a8a5918..9762bcc09 100644 --- a/integration_tests/validator_out.go +++ b/integration_tests/validator_out.go @@ -16,8 +16,6 @@ import ( // Start the chain with validators func (s *IntegrationTestSuite) TestValidatorOut() { s.Run("Bring up chain, and test the valset update", func() { - s.dockerPool.RemoveContainerByName("orchestrator3") - s.T().Logf("approving Gravity to spend ERC 20") err := s.approveERC20() s.Require().NoError(err, "error approving spending balance for the gravity contract") @@ -83,6 +81,9 @@ func (s *IntegrationTestSuite) TestValidatorOut() { sdk.Coin{Denom: gravityDenom, Amount: sdk.NewInt(1)}, ) + s.dockerPool.RemoveContainerByName("orchestrator3") + s.dockerPool.RemoveContainerByName("orchestrator2") + // Send NewMsgSendToEthereum Message s.Require().Eventuallyf(func() bool { val := s.chain.validators[1] @@ -149,6 +150,7 @@ func (s *IntegrationTestSuite) TestValidatorOut() { // Check jail status of validators s.Require().Eventuallyf(func() bool { + observed_jailing := true orchKey := s.chain.validators[3] keyring, err := orchKey.keyring() s.Require().NoError(err) @@ -161,14 +163,20 @@ func (s *IntegrationTestSuite) TestValidatorOut() { s.T().Logf("error: %s", err) return false } - s.Require().True(valThree.GetValidator().IsJailed()) + if !valThree.GetValidator().IsJailed() { + observed_jailing = false + s.T().Logf("validator 3 not jailed yet") + } valTwo, err := newQ.Validator(context.Background(), &stakingtypes.QueryValidatorRequest{ValidatorAddr: sdk.ValAddress(s.chain.validators[2].keyInfo.GetAddress()).String()}) if err != nil { s.T().Logf("error: %s", err) return false } - s.Require().False(valTwo.GetValidator().IsJailed()) + if !valTwo.GetValidator().IsJailed() { + observed_jailing = false + s.T().Logf("validator 2 not jailed yet") + } valOne, err := newQ.Validator(context.Background(), &stakingtypes.QueryValidatorRequest{ValidatorAddr: sdk.ValAddress(s.chain.validators[1].keyInfo.GetAddress()).String()}) if err != nil { @@ -183,7 +191,8 @@ func (s *IntegrationTestSuite) TestValidatorOut() { return false } s.Require().False(valZero.GetValidator().IsJailed()) - return true - }, 5*time.Minute, 1*time.Minute, "can't find slashing info") + + return observed_jailing + }, 10*time.Minute, 1*time.Minute, "can't confirm jailing status") }) } From a89d067a1ec041f969a319af544eba04271e857c Mon Sep 17 00:00:00 2001 From: Eric Bolten Date: Fri, 11 Nov 2022 12:47:42 -0800 Subject: [PATCH 3/5] Remove broken hook --- module/app/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/app/app.go b/module/app/app.go index 86792a172..d6494e8a3 100644 --- a/module/app/app.go +++ b/module/app/app.go @@ -358,7 +358,7 @@ func NewGravityApp( stakingtypes.NewMultiStakingHooks( app.distrKeeper.Hooks(), app.slashingKeeper.Hooks(), - app.gravityKeeper.Hooks(), + //app.gravityKeeper.Hooks(), TODO(bolten): this hook is broken, do not set it, to be fixed ), ) From 4ecac09f7d9f981252b50bacaff797b65c30b8c8 Mon Sep 17 00:00:00 2001 From: Eric Bolten Date: Fri, 11 Nov 2022 12:48:00 -0800 Subject: [PATCH 4/5] Fix incorrect proto type in batch confirms --- orchestrator/cosmos_gravity/src/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchestrator/cosmos_gravity/src/build.rs b/orchestrator/cosmos_gravity/src/build.rs index 764b8a2c9..1d5df67f1 100644 --- a/orchestrator/cosmos_gravity/src/build.rs +++ b/orchestrator/cosmos_gravity/src/build.rs @@ -67,9 +67,9 @@ pub async fn batch_tx_confirmation_messages( ethereum_signer: format_eth_address(ethereum_address), signature: signature.into(), }; - let msg = proto::MsgSubmitEthereumEvent { + let msg = proto::MsgSubmitEthereumTxConfirmation { signer: cosmos_address.to_string(), - event: confirmation.to_any(), + confirmation: confirmation.to_any(), }; let msg = Msg::new("/gravity.v1.MsgSubmitEthereumTxConfirmation", msg); msgs.push(msg); From ee6cd29fdba0c63a449e31f6ef9110f2f3329eae Mon Sep 17 00:00:00 2001 From: Eric Bolten Date: Fri, 11 Nov 2022 14:46:09 -0800 Subject: [PATCH 5/5] Remove unnecessary brackets --- orchestrator/cosmos_gravity/src/send.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestrator/cosmos_gravity/src/send.rs b/orchestrator/cosmos_gravity/src/send.rs index 3653b4633..ba520a6b5 100644 --- a/orchestrator/cosmos_gravity/src/send.rs +++ b/orchestrator/cosmos_gravity/src/send.rs @@ -205,7 +205,7 @@ pub async fn send_main_loop( .await { Ok(res) => debug!("message sent: {:?}", res), - Err(err) => {log_send_error(&msg_vec, err)} + Err(err) => log_send_error(&msg_vec, err), } } }