Fix historical TX requests for previous epochs during recovery (#6507)

Co-authored-by: Amaury Chamayou <amaury@xargs.fr>
Co-authored-by: Amaury Chamayou <amchamay@microsoft.com>

src/node/historical_queries_adapter.cpp

@@ -509,6 +509,29 @@ namespace ccf::historical
         }
       }
 
+      // If recovery in progress, prohibit any historical queries for previous
+      // epochs, because the service does not yet have access to the
+      // ledger secrets necessary to produce commit evidence.
+      auto service = args.tx.template ro<ccf::Service>(Tables::SERVICE);
+      auto active_service = service->get();
+      if (active_service && active_service->status != ServiceStatus::OPEN)
+      {
+        if (
+          active_service->current_service_create_txid &&
+          target_tx_id.view < active_service->current_service_create_txid->view)
+        {
+          auto reason = fmt::format(
+            "Historical transaction {} is not signed by the current service "
+            "identity key and cannot be retrieved until recovery is complete.",
+            target_tx_id.to_str());
+          ehandler(
+            HistoricalQueryErrorCode::TransactionInvalid,
+            std::move(reason),
+            args);
+          return;
+        }
+      }
+
       // We need a handle to determine whether this request is the 'same' as a
       // previous one. For simplicity we use target_tx_id.seqno. This means we
       // keep a lot of state around for old requests! It should be cleaned up

tests/e2e_logging.py

@@ -77,7 +77,12 @@ def verify_endorsements_openssl(service_cert, receipt):
 
 
 def verify_receipt(
-    receipt, service_cert, claims=None, generic=True, skip_endorsement_check=False
+    receipt,
+    service_cert,
+    claims=None,
+    generic=True,
+    skip_endorsement_check=False,
+    is_signature_tx=False,
 ):
     """
     Raises an exception on failure
@@ -115,7 +120,7 @@ def verify_receipt(
             .digest()
             .hex()
         )
-    else:
+    elif not is_signature_tx:
         assert "leaf_components" in receipt, receipt
         assert "write_set_digest" in receipt["leaf_components"]
         write_set_digest = bytes.fromhex(receipt["leaf_components"]["write_set_digest"])
@@ -133,6 +138,10 @@ def verify_receipt(
             .digest()
             .hex()
         )
+    else:
+        assert is_signature_tx
+        leaf = receipt["leaf"]
+
     root = ccf.receipt.root(leaf, receipt["proof"])
     ccf.receipt.verify(root, receipt["signature"], node_cert)
 

tests/recovery.py

@@ -25,6 +25,10 @@
 from loguru import logger as LOG
 
 
+def shifted_tx(tx, view_diff, seq_dif):
+    return ccf.tx_id.TxID(tx.view + view_diff, tx.seqno + seq_dif)
+
+
 def get_and_verify_historical_receipt(network, ref_msg):
     primary, _ = network.find_primary()
     if not ref_msg:
@@ -165,6 +169,14 @@ def test_recover_service_with_wrong_identity(network, args):
     network.save_service_identity(args)
     first_service_identity_file = args.previous_service_identity_file
 
+    with old_primary.client() as c:
+        before_recovery_tx_id = ccf.tx_id.TxID.from_str(
+            c.get("/node/commit").body.json()["transaction_id"]
+        )
+        previous_service_created_tx_id = ccf.tx_id.TxID.from_str(
+            c.get("/node/network").body.json()["current_service_create_txid"]
+        )
+
     network.stop_all_nodes()
 
     current_ledger_dir, committed_ledger_dirs = old_primary.get_ledger()
@@ -254,8 +266,65 @@ def test_recover_service_with_wrong_identity(network, args):
         snapshots_dir=snapshots_dir,
     )
 
+    # Must fail with a dedicated error message if requesting a receipt for a TX
+    # from past epochs, since ledger secrets are not yet available,
+    # therefore no receipt can be generated.
+    primary, _ = recovered_network.find_primary()
+    with primary.client() as cli:
+        curr_tx_id = ccf.tx_id.TxID.from_str(
+            cli.get("/node/commit").body.json()["transaction_id"]
+        )
+
+        response = cli.get(f"/node/receipt?transaction_id={str(before_recovery_tx_id)}")
+        assert response.status_code == http.HTTPStatus.NOT_FOUND, response
+        assert (
+            "not signed by the current service"
+            in response.body.json()["error"]["message"]
+        ), response
+
+        current_service_created_tx_id = ccf.tx_id.TxID.from_str(
+            cli.get("/node/network").body.json()["current_service_create_txid"]
+        )
+
+    # TX from the current epoch though can be verified, as soon as the caller
+    # trusts the current service identity.
+    receipt = primary.get_receipt(curr_tx_id.view, curr_tx_id.seqno).json()
+    verify_receipt(receipt, recovered_network.cert, is_signature_tx=True)
+
     recovered_network.recover(args)
 
+    # Needs refreshing, recovery has completed.
+    with primary.client() as cli:
+        curr_tx_id = ccf.tx_id.TxID.from_str(
+            cli.get("/node/commit").body.json()["transaction_id"]
+        )
+
+    # Check receipts for transactions after multiple recoveries
+    txids = [
+        # Last TX before previous recovery
+        shifted_tx(previous_service_created_tx_id, -2, -1),
+        # First after previous recovery
+        previous_service_created_tx_id,
+        # Random TX before previous and last recovery
+        shifted_tx(current_service_created_tx_id, -2, -5),
+        # Last TX before last recovery
+        shifted_tx(current_service_created_tx_id, -2, -1),
+        # First TX after last recovery
+        current_service_created_tx_id,
+        # Random TX after last recovery
+        shifted_tx(curr_tx_id, 0, -3),
+    ]
+
+    for tx in txids:
+        receipt = primary.get_receipt(tx.view, tx.seqno).json()
+
+        try:
+            verify_receipt(receipt, recovered_network.cert)
+        except AssertionError:
+            # May fail due to missing leaf components if it's a signature TX,
+            # try again with a flag to force skip leaf components verification.
+            verify_receipt(receipt, recovered_network.cert, is_signature_tx=True)
+
     return recovered_network