Skip to content

Commit

Permalink
#3967 Option<Epoch> for everybody (#4007)
Browse files Browse the repository at this point in the history 
* 3967 Option<Epoch> for everybody

* First check if epochs enabled than divide by epoch_height

---------

Co-authored-by: Lukasz Rzasik <lukasz.rzasik@gmail.com>
  • Loading branch information
@pls148 @lukaszrzasik
pls148 and lukaszrzasik authored Jan 17, 2025
1 parent cd4adbc commit fa68e1b
Show file tree
Hide file tree
Showing 68 changed files with 1,478 additions and 900 deletions.
5 changes: 2 additions & 3 deletions crates/example-types/src/node_types.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,6 @@ impl Versions for EpochUpgradeTestVersions {
mod tests {
use committable::{Commitment, Committable};
use hotshot_types::{
data::EpochNumber,
impl_has_epoch,
message::UpgradeLock,
simple_vote::{HasEpoch, VersionedVoteData},
Expand All @@ -333,7 +332,7 @@ mod tests {
/// Dummy data used for test
struct TestData<TYPES: NodeType> {
data: u64,
epoch: TYPES::Epoch,
epoch: Option<TYPES::Epoch>,
}

impl<TYPES: NodeType> Committable for TestData<TYPES> {
Expand All @@ -353,7 +352,7 @@ mod tests {

let data = TestData {
data: 10,
epoch: EpochNumber::new(0),
epoch: None,
};

let view_0 = <TestTypes as NodeType>::View::new(0);
Expand Down
51 changes: 40 additions & 11 deletions crates/example-types/src/storage_types.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ use async_trait::async_trait;
use hotshot_types::{
consensus::CommitmentMap,
data::{
DaProposal, DaProposal2, Leaf, Leaf2, QuorumProposal, QuorumProposal2, VidDisperseShare,
VidDisperseShare2,
DaProposal, DaProposal2, Leaf, Leaf2, QuorumProposal, QuorumProposal2,
QuorumProposalWrapper, VidDisperseShare, VidDisperseShare2,
},
event::HotShotAction,
message::Proposal,
Expand Down Expand Up @@ -50,12 +50,13 @@ pub struct TestStorageState<TYPES: NodeType> {
da2s: HashMap<TYPES::View, Proposal<TYPES, DaProposal2<TYPES>>>,
proposals: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposal<TYPES>>>,
proposals2: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposal2<TYPES>>>,
proposals_wrapper: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposalWrapper<TYPES>>>,
high_qc: Option<hotshot_types::simple_certificate::QuorumCertificate<TYPES>>,
high_qc2: Option<hotshot_types::simple_certificate::QuorumCertificate2<TYPES>>,
next_epoch_high_qc2:
Option<hotshot_types::simple_certificate::NextEpochQuorumCertificate2<TYPES>>,
action: TYPES::View,
epoch: TYPES::Epoch,
epoch: Option<TYPES::Epoch>,
}

impl<TYPES: NodeType> Default for TestStorageState<TYPES> {
Expand All @@ -67,11 +68,12 @@ impl<TYPES: NodeType> Default for TestStorageState<TYPES> {
da2s: HashMap::new(),
proposals: BTreeMap::new(),
proposals2: BTreeMap::new(),
proposals_wrapper: BTreeMap::new(),
high_qc: None,
next_epoch_high_qc2: None,
high_qc2: None,
action: TYPES::View::genesis(),
epoch: TYPES::Epoch::genesis(),
epoch: None,
}
}
}
Expand Down Expand Up @@ -109,22 +111,27 @@ impl<TYPES: NodeType> TestableDelay for TestStorage<TYPES> {
impl<TYPES: NodeType> TestStorage<TYPES> {
pub async fn proposals_cloned(
&self,
) -> BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposal2<TYPES>>> {
self.inner.read().await.proposals2.clone()
) -> BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposalWrapper<TYPES>>> {
self.inner.read().await.proposals_wrapper.clone()
}

pub async fn high_qc_cloned(&self) -> Option<QuorumCertificate2<TYPES>> {
self.inner.read().await.high_qc2.clone()
}

pub async fn next_epoch_high_qc_cloned(&self) -> Option<NextEpochQuorumCertificate2<TYPES>> {
self.inner.read().await.next_epoch_high_qc2.clone()
}

pub async fn decided_upgrade_certificate(&self) -> Option<UpgradeCertificate<TYPES>> {
self.decided_upgrade_certificate.read().await.clone()
}

pub async fn last_actioned_view(&self) -> TYPES::View {
self.inner.read().await.action
}
pub async fn last_actioned_epoch(&self) -> TYPES::Epoch {

pub async fn last_actioned_epoch(&self) -> Option<TYPES::Epoch> {
self.inner.read().await.epoch
}
pub async fn vids_cloned(&self) -> VidShares2<TYPES> {
Expand Down Expand Up @@ -171,7 +178,7 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
_vid_commit: <VidSchemeType as VidScheme>::Commit,
) -> Result<()> {
if self.should_return_err {
bail!("Failed to append VID proposal to storage");
bail!("Failed to append DA proposal to storage");
}
Self::run_delay_settings_from_config(&self.delay_config).await;
let mut inner = self.inner.write().await;
Expand All @@ -180,13 +187,14 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
.insert(proposal.data.view_number, proposal.clone());
Ok(())
}

async fn append_da2(
&self,
proposal: &Proposal<TYPES, DaProposal2<TYPES>>,
_vid_commit: <VidSchemeType as VidScheme>::Commit,
) -> Result<()> {
if self.should_return_err {
bail!("Failed to append VID proposal to storage");
bail!("Failed to append DA proposal (2) to storage");
}
Self::run_delay_settings_from_config(&self.delay_config).await;
let mut inner = self.inner.write().await;
Expand All @@ -195,12 +203,13 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
.insert(proposal.data.view_number, proposal.clone());
Ok(())
}

async fn append_proposal(
&self,
proposal: &Proposal<TYPES, QuorumProposal<TYPES>>,
) -> Result<()> {
if self.should_return_err {
bail!("Failed to append VID proposal to storage");
bail!("Failed to append Quorum proposal (1) to storage");
}
Self::run_delay_settings_from_config(&self.delay_config).await;
let mut inner = self.inner.write().await;
Expand All @@ -209,12 +218,13 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
.insert(proposal.data.view_number, proposal.clone());
Ok(())
}

async fn append_proposal2(
&self,
proposal: &Proposal<TYPES, QuorumProposal2<TYPES>>,
) -> Result<()> {
if self.should_return_err {
bail!("Failed to append VID proposal to storage");
bail!("Failed to append Quorum proposal (2) to storage");
}
Self::run_delay_settings_from_config(&self.delay_config).await;
let mut inner = self.inner.write().await;
Expand All @@ -224,6 +234,21 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
Ok(())
}

async fn append_proposal_wrapper(
&self,
proposal: &Proposal<TYPES, QuorumProposalWrapper<TYPES>>,
) -> Result<()> {
if self.should_return_err {
bail!("Failed to append Quorum proposal (wrapped) to storage");
}
Self::run_delay_settings_from_config(&self.delay_config).await;
let mut inner = self.inner.write().await;
inner
.proposals_wrapper
.insert(proposal.data.view_number(), proposal.clone());
Ok(())
}

async fn record_action(
&self,
view: <TYPES as NodeType>::View,
Expand Down Expand Up @@ -277,6 +302,7 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
}
Ok(())
}

async fn update_next_epoch_high_qc2(
&self,
new_next_epoch_high_qc: hotshot_types::simple_certificate::NextEpochQuorumCertificate2<
Expand All @@ -297,6 +323,7 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
}
Ok(())
}

async fn update_undecided_state(
&self,
_leaves: CommitmentMap<Leaf<TYPES>>,
Expand All @@ -308,6 +335,7 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
Self::run_delay_settings_from_config(&self.delay_config).await;
Ok(())
}

async fn update_undecided_state2(
&self,
_leaves: CommitmentMap<Leaf2<TYPES>>,
Expand All @@ -319,6 +347,7 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
Self::run_delay_settings_from_config(&self.delay_config).await;
Ok(())
}

async fn update_decided_upgrade_certificate(
&self,
decided_upgrade_certificate: Option<UpgradeCertificate<TYPES>>,
Expand Down
11 changes: 8 additions & 3 deletions crates/examples/infra/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ use hotshot_types::{
node_implementation::{ConsensusTime, NodeType, Versions},
states::TestableState,
},
utils::genesis_epoch_from_version,
HotShotConfig, PeerConfig, ValidatorConfig,
};
use libp2p_networking::network::{
Expand Down Expand Up @@ -524,11 +525,15 @@ pub trait RunDa<
.memberships
.read()
.await
.committee_leaders(TYPES::View::genesis(), TYPES::Epoch::genesis())
.committee_leaders(
TYPES::View::genesis(),
genesis_epoch_from_version::<V, TYPES>(),
)
.len();
let consensus_lock = context.hotshot.consensus();
let consensus = consensus_lock.read().await;
let total_num_views = usize::try_from(consensus.locked_view().u64()).unwrap();
let consensus_reader = consensus_lock.read().await;
let total_num_views = usize::try_from(consensus_reader.locked_view().u64()).unwrap();
drop(consensus_reader);
// `failed_num_views` could include uncommitted views
let failed_num_views = total_num_views - num_successful_commits;
// When posting to the orchestrator, note that the total number of views also include un-finalized views.
Expand Down
53 changes: 33 additions & 20 deletions crates/hotshot/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ pub mod traits;
pub mod types;

pub mod tasks;
use hotshot_types::data::QuorumProposalWrapper;

/// Contains helper functions for the crate
pub mod helpers;
Expand Down Expand Up @@ -62,13 +63,14 @@ use hotshot_types::{
states::ValidatedState,
storage::Storage,
},
utils::epoch_from_block_number,
utils::{genesis_epoch_from_version, option_epoch_from_block_number},
HotShotConfig,
};
/// Reexport rand crate
pub use rand;
use tokio::{spawn, time::sleep};
use tracing::{debug, instrument, trace};

// -- Rexports
// External
use crate::{
Expand Down Expand Up @@ -121,7 +123,7 @@ pub struct SystemContext<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versi
start_view: TYPES::View,

/// The epoch to enter when first starting consensus
start_epoch: TYPES::Epoch,
start_epoch: Option<TYPES::Epoch>,

/// Access to the output event stream.
output_event_stream: (Sender<Event<TYPES>>, InactiveReceiver<Event<TYPES>>),
Expand Down Expand Up @@ -235,6 +237,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
internal_chan,
external_chan,
)
.await
}

/// Creates a new [`Arc<SystemContext>`] with the given configuration options.
Expand All @@ -245,7 +248,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
/// Use this function if you want to use some preexisting channels and to spin up the tasks
/// and start consensus manually. Mostly useful for tests
#[allow(clippy::too_many_arguments, clippy::type_complexity)]
pub fn new_from_channels(
pub async fn new_from_channels(
public_key: TYPES::SignatureKey,
private_key: <TYPES::SignatureKey as SignatureKey>::PrivateKey,
nonce: u64,
Expand Down Expand Up @@ -290,10 +293,15 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
)),
};

let epoch = TYPES::Epoch::new(epoch_from_block_number(
// #3967 REVIEW NOTE: Should this actually be Some()? How do we know?
let epoch = option_epoch_from_block_number::<TYPES>(
upgrade_lock
.epochs_enabled(anchored_leaf.view_number())
.await,
anchored_leaf.height(),
config.epoch_height,
));
);

// Insert the validated state to state map.
let mut validated_state_map = BTreeMap::default();
validated_state_map.insert(
Expand Down Expand Up @@ -322,18 +330,11 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
saved_payloads.insert(anchored_leaf.view_number(), Arc::new(payload));
}

let anchored_epoch = if config.epoch_height == 0 {
TYPES::Epoch::new(0)
} else if anchored_leaf.height() % config.epoch_height == 0 {
TYPES::Epoch::new(anchored_leaf.height() / config.epoch_height)
} else {
TYPES::Epoch::new(anchored_leaf.height() / config.epoch_height + 1)
};
let consensus = Consensus::new(
validated_state_map,
initializer.saved_vid_shares,
anchored_leaf.view_number(),
anchored_epoch,
epoch,
anchored_leaf.view_number(),
anchored_leaf.view_number(),
initializer.actioned_view,
Expand Down Expand Up @@ -415,7 +416,10 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> SystemContext<T
async move {
sleep(Duration::from_millis(next_view_timeout)).await;
broadcast_event(
Arc::new(HotShotEvent::Timeout(start_view + 1, start_epoch + 1)),
Arc::new(HotShotEvent::Timeout(
start_view + 1,
start_epoch.map(|x| x + 1),
)),
&event_stream,
)
.await;
Expand Down Expand Up @@ -1002,26 +1006,35 @@ pub struct HotShotInitializer<TYPES: NodeType> {

/// Starting view number that should be equivalent to the view the node shut down with last.
start_view: TYPES::View,

/// Starting epoch number that should be equivalent to the epoch the node shut down with last.
start_epoch: TYPES::Epoch,
start_epoch: Option<TYPES::Epoch>,

/// The view we last performed an action in. An action is Proposing or voting for
/// Either the quorum or DA.
actioned_view: TYPES::View,

/// Highest QC that was seen, for genesis it's the genesis QC. It should be for a view greater
/// than `inner`s view number for the non genesis case because we must have seen higher QCs
/// to decide on the leaf.
high_qc: QuorumCertificate2<TYPES>,

/// Next epoch highest QC that was seen. This is needed to propose during epoch transition after restart.
next_epoch_high_qc: Option<NextEpochQuorumCertificate2<TYPES>>,

/// Previously decided upgrade certificate; this is necessary if an upgrade has happened and we are not restarting with the new version
decided_upgrade_certificate: Option<UpgradeCertificate<TYPES>>,

/// Undecided leaves that were seen, but not yet decided on. These allow a restarting node
/// to vote and propose right away if they didn't miss anything while down.
undecided_leaves: Vec<Leaf2<TYPES>>,

/// Not yet decided state
undecided_state: BTreeMap<TYPES::View, View<TYPES>>,

/// Proposals we have sent out to provide to others for catchup
saved_proposals: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposal2<TYPES>>>,
saved_proposals: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposalWrapper<TYPES>>>,

/// Saved VID shares
saved_vid_shares: Option<VidShares<TYPES>>,
}
Expand All @@ -1037,11 +1050,11 @@ impl<TYPES: NodeType> HotShotInitializer<TYPES> {
let high_qc = QuorumCertificate2::genesis::<V>(&validated_state, &instance_state).await;

Ok(Self {
inner: Leaf2::genesis(&validated_state, &instance_state).await,
inner: Leaf2::genesis::<V>(&validated_state, &instance_state).await,
validated_state: Some(Arc::new(validated_state)),
state_delta: Some(Arc::new(state_delta)),
start_view: TYPES::View::new(0),
start_epoch: TYPES::Epoch::new(0),
start_epoch: genesis_epoch_from_version::<V, TYPES>(),
actioned_view: TYPES::View::new(0),
saved_proposals: BTreeMap::new(),
high_qc,
Expand All @@ -1067,9 +1080,9 @@ impl<TYPES: NodeType> HotShotInitializer<TYPES> {
instance_state: TYPES::InstanceState,
validated_state: Option<Arc<TYPES::ValidatedState>>,
start_view: TYPES::View,
start_epoch: TYPES::Epoch,
start_epoch: Option<TYPES::Epoch>,
actioned_view: TYPES::View,
saved_proposals: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposal2<TYPES>>>,
saved_proposals: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposalWrapper<TYPES>>>,
high_qc: QuorumCertificate2<TYPES>,
next_epoch_high_qc: Option<NextEpochQuorumCertificate2<TYPES>>,
decided_upgrade_certificate: Option<UpgradeCertificate<TYPES>>,
Expand Down
Loading

0 comments on commit fa68e1b

Please sign in to comment.