Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WEEKLY RELEASE] Bump HotShot #743

Merged
merged 14 commits into from
Dec 6, 2024
Merged

[WEEKLY RELEASE] Bump HotShot #743

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
8695018
Bump HotShot
QuentinI Nov 27, 2024
1dcab37
Downgrade primitive-types
QuentinI Dec 2, 2024
4974e42
Merge branch 'main' into hotshot/0.5.82
QuentinI Dec 5, 2024
d3e1397
Update lockfile
QuentinI Dec 5, 2024
e0887c3
tag
QuentinI Dec 5, 2024
a5a8292
Fix version
QuentinI Dec 5, 2024
3d694f0
Back to leaf1
QuentinI Dec 5, 2024
faf78b9
Discard changes to src/availability/query_data.rs
QuentinI Dec 5, 2024
c707270
Discard changes to src/data_source.rs
QuentinI Dec 5, 2024
d0d4391
Discard changes to src/data_source/storage/sql/queries.rs
QuentinI Dec 5, 2024
5e2c984
Discard changes to src/lib.rs
QuentinI Dec 5, 2024
31ab748
Merge branch 'main' into hotshot/0.5.82
QuentinI Dec 5, 2024
ab55db8
log qc2 data leaf commit and leaf2commit
imabdulbasit Dec 5, 2024
0d75e20
Revert "log qc2 data leaf commit and leaf2commit"
QuentinI Dec 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
428 changes: 167 additions & 261 deletions Cargo.lock
View file
Open in desktop

Large diffs are not rendered by default.

48 changes: 24 additions & 24 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

[package]
name = "hotshot-query-service"
version = "0.1.62"
version = "0.1.63"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we haven't updated this in a while. The version should be 0.1.75

authors = ["Espresso Systems <hello@espressosys.com>"]
edition = "2021"
license = "GPL-3.0-or-later"
Expand Down Expand Up @@ -43,12 +43,12 @@ sql-data-source = ["include_dir", "refinery", "refinery-core", "sqlx", "log"]

# Enable extra features useful for writing tests with a query service.
testing = [
"espresso-macros",
"hotshot-example-types",
"portpicker",
"rand",
"spin_sleep",
"tempfile",
"espresso-macros",
"hotshot-example-types",
"portpicker",
"rand",
"spin_sleep",
"tempfile",
]

[[example]]
Expand All @@ -69,16 +69,16 @@ derivative = "2.2"
derive_more = "0.99"
either = "1.12"
futures = "0.3"
hotshot = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.81" }
hotshot-testing = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.81" }
hotshot-types = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.81" }
hotshot = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.82" }
hotshot-testing = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.82" }
hotshot-types = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.82" }
itertools = "0.12.1"
jf-merkle-tree = { version = "0.1.0", git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5", features = [
"std",
"std",
] }
jf-vid = { version = "0.1.0", git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5", features = [
"std",
"parallel",
"std",
"parallel",
] }
prometheus = "0.13"
serde = { version = "1.0", features = ["derive"] }
Expand All @@ -89,10 +89,10 @@ tagged-base64 = "0.4"
tide-disco = "0.9"
time = "0.3"
tokio = { version = "1", default-features = false, features = [
"rt-multi-thread",
"macros",
"parking_lot",
"sync",
"rt-multi-thread",
"macros",
"parking_lot",
"sync",
] }
toml = "0.8"
tracing = "0.1"
Expand All @@ -112,16 +112,16 @@ log = { version = "0.4", optional = true }
refinery = { version = "0.8", features = ["tokio-postgres"], optional = true }
refinery-core = { version = "0.8", optional = true }
sqlx = { version = "0.8", features = [
"bit-vec",
"postgres",
"runtime-tokio",
"sqlite",
"tls-native-tls",
"bit-vec",
"postgres",
"runtime-tokio",
"sqlite",
"tls-native-tls",
], optional = true }

# Dependencies enabled by feature "testing".
espresso-macros = { git = "https://github.com/EspressoSystems/espresso-macros.git", tag = "0.1.0", optional = true }
hotshot-example-types = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.81", optional = true }
hotshot-example-types = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.82", optional = true }
portpicker = { version = "0.1", optional = true }
rand = { version = "0.8", optional = true }
spin_sleep = { version = "1.2", optional = true }
Expand All @@ -142,7 +142,7 @@ backtrace-on-stack-overflow = { version = "0.3", optional = true }
clap = { version = "4.5", features = ["derive", "env"] }
espresso-macros = { git = "https://github.com/EspressoSystems/espresso-macros.git", tag = "0.1.0" }
generic-array = "0.14"
hotshot-example-types = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.81" }
hotshot-example-types = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.82" }
portpicker = "0.1"
rand = "0.8"
reqwest = "0.12.3"
Expand Down
18 changes: 4 additions & 14 deletions examples/simple-server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use futures::future::{join_all, try_join_all};
use hotshot::{
traits::implementations::{MasterMap, MemoryNetwork},
types::{SignatureKey, SystemContextHandle},
HotShotInitializer, MarketplaceConfig, Memberships, SystemContext,
HotShotInitializer, MarketplaceConfig, SystemContext,
};
use hotshot_example_types::{
auction_results_provider_types::TestAuctionResultsProvider, state_types::TestInstanceState,
Expand Down Expand Up @@ -169,20 +169,10 @@ async fn init_consensus(
// Get the number of nodes with stake
let num_nodes_with_stake = NonZeroUsize::new(pub_keys.len()).unwrap();

let da_membership = MockMembership::new(
let membership = MockMembership::new(
known_nodes_with_stake.clone(),
known_nodes_with_stake.clone(),
Topic::Da,
);
let non_da_membership = MockMembership::new(
known_nodes_with_stake.clone(),
known_nodes_with_stake.clone(),
Topic::Global,
);
let memberships = Memberships {
quorum_membership: non_da_membership.clone(),
da_membership: da_membership.clone(),
};

// Pick a random, unused port for the builder server
let builder_port = portpicker::pick_unused_port().expect("No ports available");
Expand Down Expand Up @@ -234,7 +224,7 @@ async fn init_consensus(
let config = config.clone();
let master_map = master_map.clone();

let memberships = memberships.clone();
let membership = membership.clone();
async move {
let network = Arc::new(MemoryNetwork::new(
&pub_keys[node_id],
Expand All @@ -250,7 +240,7 @@ async fn init_consensus(
priv_key,
node_id as u64,
config,
memberships,
membership,
network,
HotShotInitializer::from_genesis::<MockVersions>(TestInstanceState::default())
.await
Expand Down
17 changes: 4 additions & 13 deletions flake.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,13 @@
inputs.pre-commit-hooks.inputs.flake-utils.follows = "flake-utils";
inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs";

inputs.poetry2nixFlake = {
url = "github:nix-community/poetry2nix";
inputs.nixpkgs.follows = "nixpkgs";
};

outputs = { self, nixpkgs, flake-utils, rust-overlay, pre-commit-hooks, poetry2nixFlake, ... }:
outputs = { self, nixpkgs, flake-utils, rust-overlay, pre-commit-hooks, ... }:
flake-utils.lib.eachDefaultSystem (system:
let
overlays = [
(import rust-overlay)
];
pkgs = import nixpkgs { inherit system overlays; };
poetry2nix = poetry2nixFlake.lib.mkPoetry2Nix { inherit pkgs; };
rustToolchain = pkgs.rust-bin.stable.latest.minimal.override {
extensions = [ "rustfmt" "clippy" "llvm-tools-preview" "rust-src" ];
};
Expand Down Expand Up @@ -102,8 +96,6 @@
license = with licenses; [ mit asl20 ];
};
};
pythonEnv = poetry2nix.mkPoetryEnv { projectDir = ./.; };
myPython = with pkgs; [ poetry pythonEnv ];
shellHook = ''
# Prevent cargo aliases from using programs in `~/.cargo` to avoid conflicts with rustup
# installations.
Expand Down Expand Up @@ -182,7 +174,7 @@
git
mdbook # make-doc, documentation generation
rustToolchain
] ++ myPython ++ rustDeps;
] ++ rustDeps;

inherit RUST_SRC_PATH RUST_BACKTRACE RUST_LOG RUSTFLAGS CARGO_TARGET_DIR;
};
Expand All @@ -194,12 +186,11 @@
nixWithFlakes
git
nightlyToolchain
] ++ myPython ++ rustDeps;
] ++ rustDeps;
};
perfShell = pkgs.mkShell {
shellHook = shellHook;
buildInputs = with pkgs;
[ nixWithFlakes cargo-llvm-cov rustToolchain ] ++ rustDeps;
buildInputs = [ nixWithFlakes cargo-llvm-cov rustToolchain ] ++ rustDeps;

inherit RUST_SRC_PATH RUST_BACKTRACE RUST_LOG RUSTFLAGS CARGO_TARGET_DIR;
};
Expand Down
7 changes: 5 additions & 2 deletions src/availability.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -914,10 +914,13 @@ mod test {
);

// mock up some consensus data.
let leaf = Leaf::<MockTypes>::genesis(&Default::default(), &Default::default()).await;
let leaf = Leaf::<MockTypes>::genesis(&Default::default(), &Default::default())
.await
.into();
let qc =
QuorumCertificate::genesis::<TestVersions>(&Default::default(), &Default::default())
.await;
.await
.to_qc2();
let leaf = LeafQueryData::new(leaf, qc).unwrap();
let block = BlockQueryData::new(leaf.header().clone(), MockPayload::genesis());
data_source
Expand Down
32 changes: 18 additions & 14 deletions src/availability/query_data.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
use crate::{types::HeightIndexed, Header, Metadata, Payload, Transaction, VidCommon, VidShare};
use committable::{Commitment, Committable};
use hotshot_types::{
data::Leaf,
simple_certificate::QuorumCertificate,
data::{Leaf, Leaf2},
simple_certificate::{QuorumCertificate, QuorumCertificate2},
traits::{
self,
block_contents::{BlockHeader, GENESIS_VID_NUM_STORAGE_NODES},
Expand All @@ -28,7 +28,7 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
use snafu::{ensure, Snafu};
use std::fmt::Debug;

pub type LeafHash<Types> = Commitment<Leaf<Types>>;
pub type LeafHash<Types> = Commitment<Leaf2<Types>>;
/// A block hash is the hash of the block header.
///
/// A block consists of a header and a payload. But the header itself contains a commitment to the
Expand Down Expand Up @@ -190,8 +190,8 @@ pub trait QueryablePayload<Types: NodeType>: traits::BlockPayload<Types> {
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(bound = "")]
pub struct LeafQueryData<Types: NodeType> {
pub(crate) leaf: Leaf<Types>,
pub(crate) qc: QuorumCertificate<Types>,
pub(crate) leaf: Leaf2<Types>,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm I am not too sure about this change. We should be inserting Leaf1 for now or migrate the exisitng leaves to Leaf2 and start inserting leaf2

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but in the sequencer, we are inserting V1 of everything so I think we want to keep it as Leaf1?

pub(crate) qc: QuorumCertificate2<Types>,
}

#[derive(Clone, Debug, Snafu)]
Expand All @@ -210,13 +210,13 @@ impl<Types: NodeType> LeafQueryData<Types> {
///
/// Fails with an [`InconsistentLeafError`] if `qc` does not reference `leaf`.
pub fn new(
mut leaf: Leaf<Types>,
qc: QuorumCertificate<Types>,
mut leaf: Leaf2<Types>,
qc: QuorumCertificate2<Types>,
) -> Result<Self, InconsistentLeafError<Types>> {
// TODO: Replace with the new `commit` function in HotShot. Add an `upgrade_lock` parameter
// and a `HsVer: Versions` bound, then call `leaf.commit(upgrade_lock).await`. This will
// require updates in callers and relevant types as well.
let leaf_commit = <Leaf<Types> as Committable>::commit(&leaf);
let leaf_commit = <Leaf2<Types> as Committable>::commit(&leaf);
ensure!(
qc.data.leaf_commit == leaf_commit,
InconsistentLeafSnafu {
Expand All @@ -237,16 +237,18 @@ impl<Types: NodeType> LeafQueryData<Types> {
instance_state: &Types::InstanceState,
) -> Self {
Self {
leaf: Leaf::genesis(validated_state, instance_state).await,
qc: QuorumCertificate::genesis::<HsVer>(validated_state, instance_state).await,
leaf: Leaf::genesis(validated_state, instance_state).await.into(),
qc: QuorumCertificate::genesis::<HsVer>(validated_state, instance_state)
.await
.to_qc2(),
}
}

pub fn leaf(&self) -> &Leaf<Types> {
pub fn leaf(&self) -> &Leaf2<Types> {
&self.leaf
}

pub fn qc(&self) -> &QuorumCertificate<Types> {
pub fn qc(&self) -> &QuorumCertificate2<Types> {
&self.qc
}

Expand All @@ -258,7 +260,7 @@ impl<Types: NodeType> LeafQueryData<Types> {
// TODO: Replace with the new `commit` function in HotShot. Add an `upgrade_lock` parameter
// and a `HsVer: Versions` bound, then call `leaf.commit(upgrade_lock).await`. This will
// require updates in callers and relevant types as well.
<Leaf<Types> as Committable>::commit(&self.leaf)
<Leaf2<Types> as Committable>::commit(&self.leaf)
}

pub fn block_hash(&self) -> BlockHash<Types> {
Expand Down Expand Up @@ -457,7 +459,9 @@ impl<Types: NodeType> VidCommonQueryData<Types> {
validated_state: &Types::ValidatedState,
instance_state: &Types::InstanceState,
) -> Self {
let leaf = Leaf::<Types>::genesis(validated_state, instance_state).await;
let leaf: Leaf2<Types> = Leaf::<Types>::genesis(validated_state, instance_state)
.await
.into();
let payload = leaf.block_payload().unwrap();
let bytes = payload.encode();
let disperse = vid_scheme(GENESIS_VID_NUM_STORAGE_NODES)
Expand Down
Loading
Loading