From 1c608259379c50bbf01c9484ba9962e24b8fbdea Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Tue, 16 Jan 2024 06:32:10 +0000 Subject: [PATCH] Update Substrate Module for Hong Kong (Kian's Lectures Only) (#921) * updates to intro to substrate * new tx-pool lecure and other changes * a few more touches * fix great comments from joshy * small updates * final recap of all lectures I have --- assets/img/5-Substrate/4.4-pool-context.svg | 1 - .../img/5-Substrate/dev-4-1-comms-format.svg | 2 +- assets/img/5-Substrate/dev-4-1-comms.svg | 2 +- assets/img/5-Substrate/dev-4-1-contracts.svg | 2 +- .../5-Substrate/dev-4-1-smoldot-browser.svg | 1 + assets/img/5-Substrate/dev-4-1-smoldot.svg | 2 +- assets/img/5-Substrate/dev-4-1-state-code.svg | 2 +- .../img/5-Substrate/dev-4-1-state-opaque.svg | 2 +- .../dev-4-1-substrate-meta-substrate.svg | 2 +- .../dev-4-1-substrate-meta-version.svg | 2 +- .../5-Substrate/dev-4-1-substrate-meta.svg | 2 +- assets/img/5-Substrate/dev-4-1-substrate.svg | 2 +- .../img/5-Substrate/dev-4-3-block-opaqueu.svg | 1 + assets/img/5-Substrate/dev-4-3-full-comm.svg | 2 +- assets/img/5-Substrate/dev-4-3-full.svg | 2 +- assets/img/5-Substrate/dev-pool-context.svg | 1 + .../1-Intro-to-Substrate_Slides.md | 643 ++++++++++-------- .../2-WASM-Meta-Protocol-Slides.md | 460 ++++++------- .../5-Substrate/4-Transaction-Pool_Slides.md | 526 ++++++++++---- 19 files changed, 984 insertions(+), 673 deletions(-) delete mode 100644 assets/img/5-Substrate/4.4-pool-context.svg create mode 100644 assets/img/5-Substrate/dev-4-1-smoldot-browser.svg create mode 100644 assets/img/5-Substrate/dev-4-3-block-opaqueu.svg create mode 100644 assets/img/5-Substrate/dev-pool-context.svg diff --git a/assets/img/5-Substrate/4.4-pool-context.svg b/assets/img/5-Substrate/4.4-pool-context.svg deleted file mode 100644 index 16b51ba06..000000000 --- a/assets/img/5-Substrate/4.4-pool-context.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-comms-format.svg b/assets/img/5-Substrate/dev-4-1-comms-format.svg index b1729fb61..b15273c83 100644 --- a/assets/img/5-Substrate/dev-4-1-comms-format.svg +++ b/assets/img/5-Substrate/dev-4-1-comms-format.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-comms.svg b/assets/img/5-Substrate/dev-4-1-comms.svg index 555b35631..09bffc168 100644 --- a/assets/img/5-Substrate/dev-4-1-comms.svg +++ b/assets/img/5-Substrate/dev-4-1-comms.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-contracts.svg b/assets/img/5-Substrate/dev-4-1-contracts.svg index 02dd86415..9a07c0cbb 100644 --- a/assets/img/5-Substrate/dev-4-1-contracts.svg +++ b/assets/img/5-Substrate/dev-4-1-contracts.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-smoldot-browser.svg b/assets/img/5-Substrate/dev-4-1-smoldot-browser.svg new file mode 100644 index 000000000..08611eed2 --- /dev/null +++ b/assets/img/5-Substrate/dev-4-1-smoldot-browser.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-smoldot.svg b/assets/img/5-Substrate/dev-4-1-smoldot.svg index 9cbbe065c..05ed1d19a 100644 --- a/assets/img/5-Substrate/dev-4-1-smoldot.svg +++ b/assets/img/5-Substrate/dev-4-1-smoldot.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-state-code.svg b/assets/img/5-Substrate/dev-4-1-state-code.svg index b6aabd9cd..3a4f8d8fc 100644 --- a/assets/img/5-Substrate/dev-4-1-state-code.svg +++ b/assets/img/5-Substrate/dev-4-1-state-code.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-state-opaque.svg b/assets/img/5-Substrate/dev-4-1-state-opaque.svg index 8129e0aca..e3b915bef 100644 --- a/assets/img/5-Substrate/dev-4-1-state-opaque.svg +++ b/assets/img/5-Substrate/dev-4-1-state-opaque.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-substrate-meta-substrate.svg b/assets/img/5-Substrate/dev-4-1-substrate-meta-substrate.svg index 056c8d12f..80bf0ed33 100644 --- a/assets/img/5-Substrate/dev-4-1-substrate-meta-substrate.svg +++ b/assets/img/5-Substrate/dev-4-1-substrate-meta-substrate.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-substrate-meta-version.svg b/assets/img/5-Substrate/dev-4-1-substrate-meta-version.svg index 7b53aeec5..dee986dc7 100644 --- a/assets/img/5-Substrate/dev-4-1-substrate-meta-version.svg +++ b/assets/img/5-Substrate/dev-4-1-substrate-meta-version.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-substrate-meta.svg b/assets/img/5-Substrate/dev-4-1-substrate-meta.svg index b4c2d8305..e743ac833 100644 --- a/assets/img/5-Substrate/dev-4-1-substrate-meta.svg +++ b/assets/img/5-Substrate/dev-4-1-substrate-meta.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-1-substrate.svg b/assets/img/5-Substrate/dev-4-1-substrate.svg index 0598f4e22..53b0fd09e 100644 --- a/assets/img/5-Substrate/dev-4-1-substrate.svg +++ b/assets/img/5-Substrate/dev-4-1-substrate.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-3-block-opaqueu.svg b/assets/img/5-Substrate/dev-4-3-block-opaqueu.svg new file mode 100644 index 000000000..fa13f5e2a --- /dev/null +++ b/assets/img/5-Substrate/dev-4-3-block-opaqueu.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-3-full-comm.svg b/assets/img/5-Substrate/dev-4-3-full-comm.svg index dfe59f764..6eff82b9c 100644 --- a/assets/img/5-Substrate/dev-4-3-full-comm.svg +++ b/assets/img/5-Substrate/dev-4-3-full-comm.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-4-3-full.svg b/assets/img/5-Substrate/dev-4-3-full.svg index 80e018f10..c6e77e76c 100644 --- a/assets/img/5-Substrate/dev-4-3-full.svg +++ b/assets/img/5-Substrate/dev-4-3-full.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/assets/img/5-Substrate/dev-pool-context.svg b/assets/img/5-Substrate/dev-pool-context.svg new file mode 100644 index 000000000..407fe67af --- /dev/null +++ b/assets/img/5-Substrate/dev-pool-context.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/syllabus/5-Substrate/1-Intro-to-Substrate_Slides.md b/syllabus/5-Substrate/1-Intro-to-Substrate_Slides.md index 41846b51f..089ef918e 100644 --- a/syllabus/5-Substrate/1-Intro-to-Substrate_Slides.md +++ b/syllabus/5-Substrate/1-Intro-to-Substrate_Slides.md @@ -8,20 +8,15 @@ duration: 60 minutes --- -## Before Going Any Further πŸ›‘ - -While I speak, please clone `substrate`, and run `cargo build && cargo build --release`. - -> https://github.com/paritytech/substrate - ---- - ## About These Lectures and Lecturer - Ground-up, low-level, but hands-on. - Intentionally avoiding FRAME, but giving you the tools to be successful at it. -- Narratives above facts all. - Interrupts and questions are always welcome. +- Narratives > facts. + + +Your feedback is highly appreciated! --- @@ -29,39 +24,41 @@ While I speak, please clone `substrate`, and run `cargo build && cargo build --r Substrate is a **Rust framework** for **building blockchains**. ----v - -### Why Substrate? - - +Note: -Notes: - -Highlight the multi-chain part. +like a blockchain making factory. ---v ### Why Substrate? - + Notes: -Polkadot is the biggest bet in this ecosystem against chain maximalism, and Substrate plays a big -role in this scenario. +Highlight the multi-chain part. ---v ### Why Substrate? - ⛓️ Future is multi-chain. - - 😭 Building a blockchain is hard. Upgrading it even harder. - πŸ’‘ Framework! -- 🧐 But which attitude to take? - + +---v + +### Why Substrate? + + + +Notes: + +Substrate is a pure technological investment against chain maximalism, even outside of Polkadot. You +can create sovereign chains with substrate, or Polkadot Parachains, or more hybrid things. More on +this when you learn about Substrate's role in the Polkadot ecosystem. --- @@ -97,6 +94,9 @@ meet the demands of today. https://en.wikipedia.org/wiki/Bitcoin_scalability_problem https://ycharts.com/indicators/ethereum_average_gas_price +It is interesting to checkout some projects that are now trying to be "Substrate for L2s": +https://github.com/Sovereign-Labs/sovereign-sdk / https://arbitrum.io/orbit + ---v ### Core Philosophies of Substrate πŸ’­ @@ -230,7 +230,7 @@ Also, this is a good time to talk about how we use "Runtime" in a different way. ## 🀩 Generic, Modular and Extensible Design - Second line of defense. -- Our _execution_ (possibly thanks to Rust) is perfect, but we can't predict the future. +- Whatever code we write is (thanks to Rust) perfectly executed, *but what should we write*? Notes: @@ -291,7 +291,7 @@ particular implementation. - What use is governance, if the upgrade cannot be enacted? -- (trustlessly) Upgradeability! +- (trustless and forkless) Upgradeability! Notes: @@ -340,7 +340,7 @@ whole thing to be updated. ### 🏦 Governance + Upgradeability -_The way to make a protocol truly upgradeable is to design a meta-protocol that is not upgradeable._ +_The way to make a protocol truly upgradeable is to design a meta-protocol._ ---v @@ -350,8 +350,8 @@ _The way to make a protocol truly upgradeable is to design a meta-protocol that Note: -In this figure, the meta-protocol, the substrate client, is not forklessly upgrade-able. It can only -be upgraded with a fork. The WASM protocol, though, can be upgraded forklessly. +In this figure, the meta-protocol, the substrate node, is not forklessly upgrade-able. It can only +be upgraded with a fork. ---v @@ -359,18 +359,22 @@ be upgraded with a fork. The WASM protocol, though, can be upgraded forklessly. +Note: + +This is very similar to treating your own runtime as a smart contract. + ---v ### 🏦 Governance + Upgradeability -- Fixed meta-protocol? -- ­ "_State machine as stored Wasm_" in the Substrate client. -- inherently upgradeable protocol? -- Substrate Wasm Runtime +- Meta-protocol? +- ­ "*State machine as stored WASM* " in the Substrate node. +- Inherently upgradeable protocol? +- WASM Runtime --- -### Substrate Architecture +## Substrate Architecture @@ -391,12 +395,12 @@ be upgraded with a fork. The WASM protocol, though, can be upgraded forklessly. -

Client (Meta-protocol)

+

Node (Meta-protocol)

- Native Binary - Executes the Wasm runtime - Everything else: Database, Networking, Mempool, Consensus.. -- Also known as: Host +- Also known as: Host, Client
@@ -404,105 +408,6 @@ be upgraded with a fork. The WASM protocol, though, can be upgraded forklessly. --- -## The Runtime - -
- -- Runtime -> **Application Logic**. - -
- -
- -- A _fancy_ term: Runtime -> **State Transition Function**. - -
- -
- -- A _technical_ term: Runtime -> **how to execute blocks**. - -
- - -Notes: - -- I would personally call the runtime STF to avoid confusion with the "runtime" that is the general - programming runtime, but kinda too late for that now. -- Among the definitions of the Wasm runtime, let's recap what the state transition is. -- The block execution definition will be described more in the Wasm-meta lecture. - ---- - -## State Transition Function - -**State** - - - -Notes: - -entire set of data upon which we want to maintain a consensus. -key value. -associated with each block. - ----v - -### State Transition Function - -**Transition Function** - - - ----v - -### State Transition Function - -$$STF = F(block_{N}, state_{N}, code_{N}): state_{N+1}$$ - ----v - -### State Transition Function - - - -Notes: - -The Wasm runtime in this figure is in fact obtained from the state (see `0x123`) - ----v - -### State Transition Function - - - ----v - -### State Transition Function - - - -Notes: - -THIS IS HOW A META-PROTOCOL MAKES A SYSTEM UPGRADE-ABLE. - -could we have updated the code in N+1? By default, no because we load the wasm before you even look -into the block. - -IMPORTANT: State is not IN THE BLOCK, each state has AN ASSOCIATED BLOCK. - -Keeping the state is 100% optional. You can always re-create the state of block `N` by re-executing -block `[0, .., N-1]`. - -ofc, changing the Wasm code cannot be done by anyone. That's up to governance. - ---- - -## Full Substrate Architecture - - - ---- ## Positive Consequences of _Wasm_ Runtime πŸ”₯ @@ -510,7 +415,7 @@ ofc, changing the Wasm code cannot be done by anyone. That's up to governance. ### πŸ€– Deterministic Execution -- Portable, yet deterministic. +- Portable, yet deterministic bytecode. Notes: @@ -527,21 +432,21 @@ Wasm's instruction set is deterministic, so all good. Notes: -howe can we guarantee that neither enter an infinite loop, or try to access the filesystem? +how can we guarantee that neither enter an infinite loop, or try to access the filesystem? ---v -### 🌈 Easier (light)Client Development + ### 🌈 Easier (light)Node Development Notes: -for the case of client, your client only needs to implement a set of host environments, and NOT -re-implement the business logic. +for the case of node, your node only needs to implement the host/client, and NOT re-implement the +business logic. -Simply compare the process to create an alternative client for Ethereum, where you need to +Simply compare the process to create an alternative node for Ethereum, where you need to re-implement the EVM. -Same applies to light client, as they do not need to deal with the state transition function. +Same applies to light node, as they do not need to deal with the state transition function. ---v @@ -566,8 +471,8 @@ This update was: Notes: -take a moment to establish that this upgrade is forkless. The runtime is upgraded, but the client is -not. In fact, the client didn't need to know about this at all. +take a moment to establish that this upgrade is forkless. The runtime is upgraded, but the node is +not. In fact, the node didn't need to know about this at all. This is what the meta-protocol achieves. @@ -575,8 +480,11 @@ This is what the meta-protocol achieves. ## Negative Consequences of _Wasm_ Runtime -- 😩 Constrained resources (memory, speed, host access). -- 🌈 Client diversification != state-transition diversification +---v + +### 😩 Constrained Resources + +πŸ‘Ύ memory, speed, host access Notes: @@ -585,91 +493,42 @@ Notes: - Can be slower than native, depending on the executor/execution method. - Limited access to the host host env, all needs to be done through syscalls. -Less state-transition diversification, because the runtime is the same for all clients. If there a -bug in it, everyone is affected. - ---- - -## Consensus <> Runtime πŸ€” +---v -- Yes, consensus is not a core part of a blockchain runtime. Why? -- Not part of your STF! - -- The consensus protocol is to your runtime what HTTP is to Facebook. - +### 🌈 Node diversification != runtime diversification Notes: -comments from Joshy: - -I think this is important. The runtime is the application you want to run. - -Consensus is outside of this application helping us agree what the official state of this runtime -is. Last wave I used this analogy. - -Imagine a writers room for some TV show. Writers are sitting around coming up with potential plot -points for future episodes. Any of their ideas could work. But eventually they need to agree what -the next episode to air actually will be. - ---- - -## Database <> State πŸ€” +Less state-transition diversification, because the runtime is the same for all clients. If there a +bug in it, everyone is affected. +---v -- State is the entire set of key value data that is associated with one block. -- Database is the component that allows this be stored in disk. May or may not be key-value. +### πŸ”„ Metering - +- Is not so great in WASM... +- New experimental work on PolkaVM, based on RiscV. Notes: -state is sometimes called "storage" asd well. +It is possible to meter wasm, but it has overhead, and it is still not perfect. Goes back to the instruction set being relatively large. ---- - -## Database <> Runtime πŸ€” - -- Yes, data is stored outside of the runtime. Why? -- Wasm runtime does not have the means to store it. - -- Yet, the interpretation of the data is up to the runtime. - - ----v +Both of the following are amazing reads: -### Database <> Runtime πŸ€” +https://forum.polkadot.network/t/eliminating-pre-dispatch-weight/400 - - ----v - -## The Client: Database πŸ€” - -- The database, from the client's PoV, is a _untyped_, key-value storage. -- The runtime knows which key/value means what. +https://forum.polkadot.network/t/announcing-polkavm-a-new-risc-v-based-vm-for-smart-contracts-and-possibly-more/3811/32 --- -## State of Light Clients +## State of Light Client/Nodes -- Client that follows headers, therefore knows state roots and can ask for state-proofs to do more. +- Node that follows headers, therefore knows state roots and can ask for state-proofs to do more. ---v ### State of Light Clients - - - -- Not only possible, but they can also run as Wasm, in the browser! -- "Substrate Connect" / SMOLDOT - - - - - - - - + Notes: @@ -684,6 +543,12 @@ This has to do with the fact that consensus and a few other bits of the client a client. Now, a client that is configured with GRANDPA can only work with runtimes that are also configured with GRANDPA. +---v + +### State of Light Clients + + + --- ## Communication Paths @@ -758,12 +623,6 @@ fn main() { --- -## Substrate and Polkadot - - - ---- - ## Substrate and Smart Contracts @@ -772,7 +631,7 @@ fn main() { ### Substrate and Smart Contracts -> A Substrate-Connect extension is syncing a chain who's runtime is executing wasm contracts. +> SMOLDOT is syncing a chain who's runtime is executing wasm contracts. Question: How many nested Wasm blobs are executing one another? @@ -789,7 +648,7 @@ Question: How many nested Wasm blobs are executing one another? - The browser is executing: -- a Wasm blob (substrate-connect) +- a Wasm blob (SMOLDOT in extension) - that executes a Wasm blob (runtime) - that executes a Wasm blob (contract) @@ -800,18 +659,14 @@ Question: How many nested Wasm blobs are executing one another? ### Substrate and Smart Contracts - - ----v - -### Substrate and Smart Contracts - - So when should you write with a smart contract (Ink!) and when a Runtime (FRAME)? Notes: +https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/reference_docs/runtime_vs_smart_contract/index.html + I was asked this yesterday as well. My latest answer is: if you don't need any of the customizations -that a blockchain client/runtime gives to you, and the performance of a shared platform is okay for +that a blockchain node/runtime gives to you, and the performance of a shared platform is okay for you, then go with a smart contract. If you need more, you need a "runtime" (some kind of chian, parachain or solo) @@ -822,22 +677,70 @@ Also, a contract can not have fee-less transactions. Also, a contract usually depends on a token for gas, while a runtime can be in principle token-less fee-less. +---v + +### Substrate and Smart Contracts + + + + --- ## Technical Freedom vs Ease +Notes: + +OpenZepplin is working on templates for substrate now: + +https://github.com/OpenZeppelin/polkadot-runtime-template + +We also have some of our own: + +https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/polkadot_sdk/templates/index.html + --- -### Substrate: The Gaming Console of Blockchains! +## Substrate and Polkadot + + + +> ..is the biggest bet against chain maximalism + +---v + +### Repository Structure + +#### [`polkadot-sdk`](https://github.com/paritytech/polkadot-sdk) + +- Substrate + FRAME + XCM +- Cumulus +- Bridges +- Polkadot Node + +**This is a big repo, please clone and run `cargo build` on it after this lecture** + +
+ +#### [`polkadot-fellowship/runtime`](https://github.com/polkadot-fellows/runtimes) + +- Polkadot Runtimes + +Notes: + +https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/polkadot_sdk/index.html + +--- + +## Substrate: The Gaming Console of Blockchains! -Substrate Client +Substrate Node @@ -851,7 +754,179 @@ Substrate's Wasm Runtime Notes: -Another good analogy: Client is the FPGA, and FRAME/Wasm is the VHDL. +Another good analogy: Node is the FPGA, and FRAME/Wasm is the VHDL. + +--- + +## Substrate Architecture + +So far we covered high level facts about Substrate. Now let's dive deeper into its architecture using the Runtime/STF. + +--- + +## State Transition Function + +Let's explore this definition of the runtime a bit more. + +---v + +### State Transition Function + +**State** + + + +Notes: + +entire set of data upon which we want to maintain a consensus. +key value. +associated with each block. + +---v + +### State Transition Function + +**Transition Function** + + + +---v + +### State Transition Function + +$$STF = F(block_{N+1}, state_{N}, code_{N}): state_{N+1}$$ + +Note: + +Who can find a small nitpick issue in this? + +Code is part of state. + +---v + +### State Transition Function + + + +Notes: + +This diagram should really be 100000% clear to everyone. Pause. + +The Wasm runtime in this figure is in fact obtained from the state (see `0x123`) + +---v + +### State Transition Function + + + +Note: + +now what would happen when we want to execute block N+2? + +---v + +### State Transition Function + + + +Notes: + +THIS IS HOW A META-PROTOCOL MAKES A SYSTEM UPGRADE-ABLE. + +Q: could we have updated the code in N+1? By default, no because we load the wasm before you even look +into the block. + +IMPORTANT: State is not IN THE BLOCK, each state has AN ASSOCIATED BLOCK. + +Keeping the state is 100% optional. You can always re-create the state of block `N` by re-executing +block `[0, .., N-1]`. + +ofc, changing the Wasm code cannot be done by anyone. That's up to governance. + +---v + +### The Runtime + +
+ +- Blockchain -> **State Machine** +- Runtime -> **State Transition Function** + - ­Or, **application logic**. + - ­Or, **how to execute blocks** + +Notes: + +- I would personally call the runtime STF to avoid confusion with the "runtime" that is the general + programming runtime, but kinda too late for that now. +- Among the definitions of the Wasm runtime, let's recap what the state transition is. +- The block execution definition will be described more in the Wasm-meta lecture. + + +--- + +## Full Substrate Architecture + + + +--- + +## Consensus <> Runtime πŸ€” + +- Yes, consensus is not a core part of a blockchain runtime. Why? +- Not part of your STF! + +- The consensus protocol is to your runtime what HTTP is to Facebook. + + +Notes: + +comments from Joshy: + +I think this is important. The runtime is the application you want to run. + +Consensus is outside of this application helping us agree what the official state of this runtime +is. Last wave I used this analogy. + +Imagine a writers room for some TV show. Writers are sitting around coming up with potential plot +points for future episodes. Any of their ideas could work. But eventually they need to agree what +the next episode to air actually will be. + +--- + +## Database <> State πŸ€” + +- State is the entire set of key value data that is associated with one block. +- Database is the component that allows this be stored in disk. May or may not be key-value. + + + +Notes: + +state is sometimes called "storage" asd well. + +--- + +### Database <> Runtime πŸ€” + +- Yes, data is stored outside of the runtime. Why? +- Wasm runtime does not have the means to store it. + +- Yet, the interpretation of the data is up to the runtime. + + +---v + +### Database <> Runtime πŸ€” + + + +---v + +### The Node: Database πŸ€” + +- The database, from the client's PoV, is a _untyped_, key-value storage. +- The runtime knows which key/value means what. --- @@ -859,11 +934,12 @@ Another good analogy: Client is the FPGA, and FRAME/Wasm is the VHDL. - Substrate's design stems from 3 core principles: - **Rust**, **Generic Design**, **Upgradeability/Governance** -- Client / Runtime architecture -- State Transition - Positive and negative consequences of Wasm - Substrate next to Polkadot and other chains. - Substrate for Smart Contracts. +- Light Nodes/Clients. +- Node / Runtime architecture +- State Transition Deep Dive -> Forkless Upgrade ---v @@ -873,7 +949,7 @@ Another good analogy: Client is the FPGA, and FRAME/Wasm is the VHDL. ---v -## Recap: 🏦 Governance and Upgradeability +### Recap: 🏦 Governance and Upgradeability A timeless system must be: @@ -889,7 +965,7 @@ Notes: Question: how would you put the meta-protocol of Substrate into words? -The client is basically a wasm meta-protocol that does only one thing. This meta-protocol is +The node is basically a wasm meta-protocol that does only one thing. This meta-protocol is hardcoded, but the protocol itself is flexible. --- @@ -898,19 +974,18 @@ hardcoded, but the protocol itself is flexible. #### Track: Main Lectures -- Wasm Meta Protocol -- Substrate Storage +- Wasm Meta Protocol (Kian) +- Substrate Storage (Ankan) #### Track: Aux Lecture -- TX-Pool -- Substrate: Show Me The Code -- Substrate Interactions -- SCALE +- Tx Pool (Kian) +- Interacting with Substrate (Ankan) +- SCALE Codec (Ankan) #### Track: Graded Activity -- FRAME-Less +- FRAMELess Runtime ---v @@ -918,42 +993,37 @@ hardcoded, but the protocol itself is flexible. #### Day 0 -- Introduction βœ… (60m) -- Wasm Meta Protocol (120+m) - - Activity: Finding Runtime APIs and Host Functions in Substrate -- 🌭 _Lunch Break_ -- Show Me The Code (60m) -- Substrate Interactions (60m) -- FRAME-Less Activity (60m) - -Notes: - -We are aware that the module is highly skewed in terms of lecture time, but is is intentional and we -want to see how it works. This allows you to kickstart with your assignment earlier. +- βœ… Introduction +- SCALE Codec +- Interacting with Substrate +- πŸ“š FRAMELess Assignment ---v -## Rest of This Module! 😈 +### Rest of This Module! 😈 #### Day 1 -- Transaction Pool (60m) -- SCALE (60m) -- Substrate/FRAME Tips and Tricks -- 🌭 _Lunch Break_ -- FRAME-Less Activity +- Wasm Meta Protocol +- Tx Pool +- πŸ“š FRAMELess Activity ---v -## Rest of This Module! 😈 +### Rest of This Module! 😈 #### Day 2 -- Substrate Storage (90m) -- FRAME-Less Activity -- 🌭 _Lunch Break_ -- End of Module πŸŽ‰ +- Substrate Storage +- (Substrate/FRAME Tips and Tricks) +- πŸ“š FRAMELess Activity + +--- + +## Homework πŸ“š +Highly suggested reading to ingest this lecture better: + - [`polkadot-sdk-docs`](https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/polkadot_sdk/index.html) --- ## Additional Resources! πŸ˜‹ @@ -1006,6 +1076,33 @@ exception](https://www.gnu.org/software/classpath/license.html). --- + +## Appendix: More Diagrams of Substrate and Polkadot + +Notes: + +I made these figures recently to explain the relationship between Substrate, Cumulus and Polkadot. +They use the most generic term for client and runtime, namely "Host" and "STF" respectively. + +---v + +Substrate + + + +---v + +Polkadot + + +---v + +A Parachain + + + +--- + ## Appendix: What is Wasm Anyways? > WebAssembly (abbreviated Wasm) is a _binary instruction format_ for a _stack-based virtual @@ -1052,27 +1149,3 @@ People actually tried sticking things like JVM into the browser (_Java Applets_) - ... But, of course, Substrate comes with a framework to make this developer-friendly, **FRAMEℒ️**. --- - -## Appendix: More Diagrams of Substrate and Polkadot - -Notes: - -I made these figures recently to explain the relationship between Substrate, Cumulus and Polkadot. -They use the most generic term for client and runtime, namely "Host" and "STF" respectively. - ----v - -Substrate - - - ----v - -Polkadot - - ----v - -A Parachain - - diff --git a/syllabus/5-Substrate/2-WASM-Meta-Protocol-Slides.md b/syllabus/5-Substrate/2-WASM-Meta-Protocol-Slides.md index 5e8e51781..c9cbf7883 100644 --- a/syllabus/5-Substrate/2-WASM-Meta-Protocol-Slides.md +++ b/syllabus/5-Substrate/2-WASM-Meta-Protocol-Slides.md @@ -33,9 +33,9 @@ Notes: ---v -## It All Began With a Runtime.. +### It All Began With a Runtime.. -- The Client / Runtime division is one of the most important design decisions in Substrate. +- The Node / Runtime division is one of the most important design decisions in Substrate. - πŸ‘Ώ Bad: Fixed opinion. - πŸ˜‡ Good: Enables countless other things to not be fixed. @@ -53,7 +53,7 @@ Recall that the boundary for this division is the **state transition** ### Substrate: a short recap -- **Host Functions**: Means of a runtime communicating with its host environment, i.e. the Substrate client. +- **Host Functions**: Means of a runtime communicating with its host environment, i.e. the Substrate node. ---v @@ -69,13 +69,13 @@ Building a Wasm module's activity was building something akin to runtime-apis ### Substrate: a short recap -- Database is on the client side, storing an opaque key-value state per block. +- **Database** is on the node side, storing an opaque key-value state per block. ---v ### Substrate: a short recap -- Communication language of client/runtime is SCALE: +- Communication language of node/runtime is **SCALE**: flowchart LR @@ -136,14 +136,14 @@ fn set(key: Vec, value: Vec); Notes: -ofc the IO to these functions is all opaque bytes, because the client does not know the state +ofc the IO to these functions is all opaque bytes, because the node does not know the state layout. ---v ### Example #1: State -- could we have communicated with the client like this? +- could we have communicated with the node like this? ```rust fn set_balance(who: AccountId, amount: u128) @@ -151,17 +151,17 @@ fn set_balance(who: AccountId, amount: u128) Notes: -This would imply that the client would have to know, indefinitely, the types needed for account id +This would imply that the node would have to know, indefinitely, the types needed for account id and balance. Also, it would have to know the final key for someone's balance. ---v ### Example #1: State -- Exceptions: +- [Exceptions](https://paritytech.github.io/polkadot-sdk/master/sp_storage/well_known_keys/index.html): ```rust -/// The keys known to the client. +/// The keys known to the node. mod well_known_keys { const CODE: &[u8] = b":code"; } @@ -185,29 +185,30 @@ See https://paritytech.github.io/substrate/master/sp_storage/well_known_keys/ind ### Example #2: Block Import -- Client's view of the state -> Opaque. -- Client's view of the transactions? πŸ€” +- Node's view of the state -> Opaque. +- Node's view of the transactions? πŸ€” Notes: -Short answer is: anything that is part of the STF definition must be opaque to the client, and is +Short answer is: anything that is part of the STF definition must be opaque to the node, and is upgradeable, but we will learn this later. ---v ### Example #2: Block Import -- Transactions format is by definition part of the state transition function as well. -- What about header, and other fields in a typical block? +- Transactions format is by definition part of the state transition function as well -> Opaque Notes: +What about header, and other fields in a typical block? + as in, do we want to able to update our transactions format as well in a forkless manner? we want the runtime to be able to change its transactions format as well, in a forkless manner. The answer to the latter is more involved. The short answer is that these fields like header must be -known and established between client and runtime. If you want to alter the header format, that's a +known and established between node and runtime. If you want to alter the header format, that's a hard fork. The concept of `digest` is a means through which additional data can be put in the header without @@ -234,7 +235,7 @@ https://paritytech.github.io/substrate/master/sp_runtime/generic/index.html ```rust -struct ClientBlock { +struct NodeBlock { header: Header, transactions: Vec> } @@ -265,7 +266,7 @@ this slide is intentionally using the keyword transaction instead of extrinsic. ```rust [1-100|1-2|4-6|8-9|1-100] // fetch the block from the outer world. -let opaque_block: ClientBlock = networking::import_queue::next_block(); +let opaque_block: NodeBlock = import_queue::next_block(); // initialize a wasm runtime. let code = database::get(well_known_keys::CODE); @@ -279,10 +280,10 @@ runtime.execute_block(opaque_block); ### Example #2: Block Import -- πŸ’‘ The client needs a runtime API to ask the runtime to execute the block. +- πŸ’‘ The node needs a runtime API to ask the runtime to execute the block. ```rust -fn execute_block(opaque_block: ClientBlock) -> Result<_, _> { .. } +fn execute_block(opaque_block: NodeBlock) -> Result<_, _> { .. } ``` Notes: @@ -315,7 +316,7 @@ Notes: ```rust [1-100|1-2|4-6|8-10|12-15|17-100] // fetch the block from the outer world. -let block: ClientBlock = networking::import_queue::next_block(); +let block: NodeBlock = import_queue::next_block(); // get the parent block's state. let parent = block.header.parent_hash; @@ -343,11 +344,10 @@ Notes: ### Example #2: Block Import -- A state key is only meaningful at a given block. -- A :code is only meaningful at at given block. - -- πŸ’‘ A runtime (API) is only meaningful when executed at a give block. - +- *A state key is only meaningful at a given block* +- ­*A `:code` / Runtime is only meaningful at at given block* +- ­*A runtime (API) is only meaningful when executed at a give block* + Notes: @@ -365,38 +365,16 @@ Notes: ### Example #2: Block Import -- I can add one more small touch to this to make it more accurate.. 🀌 - ----v - -### Example #2: Block Import - -```rust [14-20] -// fetch the block from the outer world. -let block: ClientBlock = networking::import_queue::next_block(); - -// get the parent hash. Note that `sp_runtime::traits::Header` provides this. -let parent = block.header.parent_hash; -let mut state = database::state_at(parent); - -// initialize a wasm runtime FROM THE PARENT `state`! -let code = state::get(well_known_keys::CODE); -let runtime = wasm::Executor::new(code); - -// call into this runtime, update `state`. +```rust state.execute(|| { - // within this, we probably call into `host_functions::set` a lot. + // What is the most important check that a runtime must do + // inside `execute_block` πŸ€”? runtime.execute_block(block); - - let new_state_root = host_functions::state_root(); - let claimed_state_root = block.header.state_root; - assert_eq!(new_state_root, claimed_state_root); }); - -// create the state of the next_block -database::store_state(block.header.hash, state) ``` +Note: make sure roots in the header match! + ---v ### Example #2: Block Import: Recap @@ -428,20 +406,23 @@ database::store_state(block.header.hash, state) - An Extrinsic is data that come from outside of the runtime. - ­Inherents are data that is put into the block by the block author, directly. -- ­ Yes, transactions are **a type of extrinsic**, but not all extrinsics are transactions. -- ­ So, why is it called _Transaction Pool_ and not _Extrinsic Pool_? +- ­Yes, transactions are **a type of extrinsic**, but not all extrinsics are transactions. +- ­So, why is it called _Transaction Pool_ and not _Extrinsic Pool_? Notes: extrinsics are just blobs of data which can be included in a block. inherents are types of extrinsic which are crafted by the block builder itself in the production process. they are unsigned because the assertion is that they are "inherently true" by virtue of getting past all validators. + notionally the origin can be said to be a plurality of validators. take for example the timestamp set inherent. if the data were sufficiently incorrect (i.e. the wrong time), then the block would not be accepted by enough validators and would not become canonicalised. so the "nobody" origin is -actually the tacit approval of the validators. transactions are generally statements of opinion -which are valuable to the chain to have included (because fees are paid or some other good is done). -the transaction pool filters out which of these are indeed valuable and nodes share them. +actually the tacit approval of the validators. + +transactions are generally statements of opinion which are valuable to the chain to have included +(because fees are paid or some other good is done). the transaction pool filters out which of these +are indeed valuable and nodes share them. --- @@ -501,7 +482,7 @@ The point being, eventually the pool builds a list of "ready transactions". ### Example #3: Block Authoring -```rust [1-100|1-2|4-5|7-9|11-12|14-20|21-100] +```rust [1-100|1-2|4-5|7-9|11-12|14-21|23-100] // get the best-block, based on whatever consensus rule we have. let (best_number, best_hash) = consensus::best_block(); @@ -512,15 +493,16 @@ let mut state = database::state_at(best_hash); let code = state::get(well_known_keys::CODE); let runtime = wasm::Executor::new(code); -// get an empty client block. -let mut block: ClientBlock = Default::default(); +// get an empty node block. +let mut block: NodeBlock = Default::default(); // repeatedly apply transactions. while let Some(next_transaction) = transaction_pool_iter::next() { state.execute(|| { - runtime.apply_extrinsic(next_transaction); + if runtime.apply_extrinsic(next_ext).is_ok() { + block.extrinsics.push(next_ext); + } }); - block.extrinsics.push(next_transaction); } // set the new state root. @@ -529,63 +511,22 @@ block.header.state_root = state.root(); Notes: -- What is the type of `next_ext`? `Vec` +- What is the type of `next_transaction`? `Vec` - Do we actually loop forever until the tx-pool is empty? probably not! ---v ### Example #3: Block Authoring -- Substrate based runtimes are allowed to perform some operations at the beginning and end of each block. +- Substrate based runtimes are allowed to perform some operations at the beginning and end of each + block. - βœ‹πŸ» And recall that a smart contract could not do this. ---v ### Example #3: Block Authoring -```rust [14-15,25-26] -// get the best-block, based on whatever consensus rule we have. -let (best_number, best_hash) = consensus::best_block(); - -// get the latest state. -let mut state = database::state_at(best_hash); - -// initialize a wasm runtime. -let code = state::get(well_known_keys::CODE); -let runtime = wasm::Executor::new(code); - -// get an empty client block. -let mut block: ClientBlock = Default::default(); - -// tell this runtime that you wish to start a new block. -runtime.initialize_block(); - -// repeatedly apply transactions. -while let Some(next_ext) = transaction_pool_iter::next() { - state.execute(|| { - runtime.apply_extrinsic(next_ext); - }); - block.extrinsics.push(next_ext); -} - -// tell the runtime that we are done. -runtime.finalize_block(); - -// set the new state root. -block.header.state_root = state.root(); -``` - ----v - -### Example #3: Block Authoring - -- What about Inherents? - ----v - -### Example #3: Block Authoring - -```rust [14-26] +```rust [14-15,25-100] // get the best-block, based on whatever consensus rule we have. let (best_number, best_hash) = consensus::best_block(); @@ -596,53 +537,42 @@ let mut state = database::state_at(best_hash); let code = state::get(well_known_keys::CODE); let runtime = wasm::Executor::new(code); -// get an empty client block. -let mut block: ClientBlock = Default::default(); - -// tell this runtime that you wish to start a new block. -runtime.initialize_block(); +// get an empty node block. +let mut block: NodeBlock = Default::default(); -let inherents: Vec> = block_builder::inherents(); -block.extrinsics = inherents; +// tell this runtime that you wish to start a new block, and pass in a raw header. +runtime.initialize_block(&block.header); // repeatedly apply transactions. while let Some(next_ext) = transaction_pool_iter::next() { state.execute(|| { - runtime.apply_extrinsic(next_ext); + if runtime.apply_extrinsic(next_ext).is_ok() { + block.extrinsics.push(next_ext); + } }); - block.extrinsics.push(next_ext); } -// tell the runtime that we are done. -runtime.finalize_block(); - -// set the new state root. -block.header.state_root = state.root(); +// finalize and set the final header. +block.header = runtime.finalize_block(); ``` Notes: -while inherents can in principle come at any point in the block, since FRAME restricts them to -come first, we also keep our example aligned. +The header interactions here are a substrate implementation detail, not a golden rule. I guess it +could have been done otherwise. I am mainly covering it because it is relevant to the assignment. -Should you wish to see the real version of this, check this crate: -https://paritytech.github.io/substrate/master/sc_basic_authorship/index.html +Good question to ask: what about inherents? see appendix slides. ---v ### Example #3: Block Authoring ```rust -fn initialize_block(..) { ... } -// note the opaque extrinsic type. +fn initialize_block(raw_header: Header) { ... } fn apply_extrinsic(extrinsic: Vec) { ... } -fn finalize_block(..) { ... } +fn finalize_block(..) -> Header { ... } ``` -Notes: - -in fact, the client also builds its inherent list with the help of the runtime. - --- ## BUT WAIT A MINUTE 😱 @@ -673,7 +603,7 @@ In order the address the mentioned issue, metadata must be a runtime API.
- Metadata contains all the basic information to know about all storage items, all extrinsics, and so - on. It will also help a client/app decode them into the right type. + on. It will also help a node/app decode them into the right type. - Substrate itself doesn't impose what the metadata should be. It is `Vec`. - FRAME based runtime expose a certain format, which is extensively adopted in the ecosystem. @@ -691,12 +621,12 @@ In order the address the mentioned issue, metadata must be a runtime API. Notes: -By Applications/Clients I really mean anyone/anything. Substrate client doesn't really use metadata +By Applications I really mean anyone/anything. Substrate node doesn't really use metadata because it is dynamically typed, but if needed, it could. --- -### Radical Upgradeability +## Radical Upgradeability Comes at the cost of radical opaque/dynamic typing. @@ -705,7 +635,7 @@ Notes: I wish you could have both, but not so easy. Some personal rant: radical upgrade-ability is the biggest advantage, and arguably one of the main -develop-ability problems of the substrate ecosystem. Writing clients, such as block explorers, +develop-ability problems of the substrate ecosystem. Writing nodes, such as block explorers, scanners, and even exchange integration are orders of magnitude harder than a blockchain that has a fixed format and only changes every 18 months at most. That being said, this is a battle that is to me obvious: we simply HAVE to win. When ethereum first introduced smart contracts, everyone @@ -715,18 +645,20 @@ also, as noted in an earlier slide, once you make it work for one chain, it work --- -## Oblivious Client πŸ™ˆπŸ™‰ +## Oblivious Node πŸ™ˆπŸ™‰ -- The underlying reason why the client is "**kept in the dark**" is so that it wouldn't need to care +- The underlying reason why the node is "**kept in the dark**" is so that it wouldn't need to care about the runtime upgrading from one block to the other. + + ---v -## Oblivious Client πŸ™ˆπŸ™‰ +### Oblivious Node πŸ™ˆπŸ™‰ -$$STF = F(blockBody_{N}, state_{N}) > state_{N+1}$$ +$$STF = F(blockBody_{N+1}, state_{N}) > state_{N+1}$$ -_Anything that is part of the STF is opaque to the client, but it can change forklessly!_ +_Anything that is part of the STF is opaque to the node, but it can change forklessly!_ - The `F` itself (your Wasm blob)? It can change! - Extrinsic format? It can change! @@ -734,16 +666,16 @@ _Anything that is part of the STF is opaque to the client, but it can change for Notes: -In essence, all components of the STF must be opaque to the client. `Vec`. +In essence, all components of the STF must be opaque to the node. `Vec`. Metadata is there to assist where needed. This is why forkless upgrades are possible in substrate. ---v -## Oblivious Client πŸ™ˆπŸ™‰ +### Oblivious Node πŸ™ˆπŸ™‰ - What about new host functions? -- What about a new header filed*? +- What about a new header field*? - What about a new Hashing primitive? πŸ₯Ί No longer forkless. @@ -778,12 +710,72 @@ time to ask any missing questions. ### Finding APIs and Host Functions - look for `impl_runtime_apis! {...}` and `decl_runtime_apis! {...}` macro calls. - - Try and find the corresponding the client code calling a given api as well. + - Try and find the corresponding the node code calling a given api as well. - Look for `#[runtime_interface]` macro, and try and find usage of the host functions! - You have 15 minutes! ---v +### Defining a Runtime API + +```rust [1-7|9-15|17-100|1-100] +// somewhere in common between node/runtime => substrate-primitive. +decl_runtime_apis! { + pub trait Core { + fn version() -> RuntimeVersion; + fn execute_block(block: Block) -> bool; + } +} + +// somewhere in the runtime code. +impl_runtime_apis! { + impl sp_api::Core for Runtime { + fn version() -> RuntimeVersion { /* stuff */ } + fn execute_block(block: Block) -> bool { /* stuff */ } + } +} + +// somewhere in the node code.. +let block_hash = "0xffff..."; +let block = Block { ... }; +let outcome: Vec = api.execute_block(block, block_hash).unwrap(); +``` + +Notes: + +- All runtime APIs are generic over a `` by default. +- All runtime APIs are executed on top of a **specific block**. This is the implicit _at_ parameter. +- Going over the API, everything is SCALE encoded both ways, but abstractions like + `impl_runtime_apis` hide that away from you. + +---v + +### Defining a Host Function + +```rust +// somewhere in substrate primitives, almost always `sp_io`. +#[runtime_interface] +pub trait Storage { + fn get(&self, key: &[u8]) -> Option> {...} + fn get(&self, key: &[u8], value: &[u8]) -> Option> {...} + fn root() -> Vec {...} +} + +#[runtime_interface] +pub trait Hashing { + fn blake2_128(data: &[u8]) -> [u8; 16] { + sp_core::hashing::blake2_128(data) + } +} + +// somewhere in substrate runtime +let hashed_value = sp_io::storage::get(b"key") + .and_then(sp_io::hashing::blake2_128) + .unwrap(); +``` + +---v + ### Finding APIs and Host Functions Activity Outcomes: @@ -843,14 +835,15 @@ sp_io::storage::set(..); sp_io::storage::root(); ``` + --- ## Lecture Recap (Part 1) - Revise "**Runtime API**" and "**Host Function**" concepts. - Deep look at block import and authoring. -- Oblivious client. -- Metadata +- Oblivious node. STF has to be opaque, but can be forklessly upgraded. +- Metadata to the rescue! --- @@ -858,70 +851,6 @@ sp_io::storage::root(); --- -## Defining a Runtime API - -```rust [1-7|9-15|17-100|1-100] -// somewhere in common between client/runtime => substrate-primitive. -decl_runtime_apis! { - pub trait Core { - fn version() -> RuntimeVersion; - fn execute_block(block: Block) -> bool; - } -} - -// somewhere in the runtime code. -impl_runtime_apis! { - impl sp_api::Core for Runtime { - fn version() -> RuntimeVersion { /* stuff */ } - fn execute_block(block: Block) -> bool { /* stuff */ } - } -} - -// somewhere in the client code.. -let block_hash = "0xffff..."; -let block = Block { ... }; -let outcome: Vec = api.execute_block(block, block_hash).unwrap(); -``` - -Notes: - -- All runtime APIs are generic over a `` by default. -- All runtime APIs are executed on top of a **specific block**. This is the implicit _at_ parameter. -- Going over the API, everything is SCALE encoded both ways, but abstractions like - `impl_runtime_apis` hide that away from you. - ---- - -## Defining a Host Function - -```rust -// somewhere in substrate primitives, almost always `sp_io`. -#[runtime_interface] -pub trait Storage { - fn get(&self, key: &[u8]) -> Option> {...} - fn get(&self, key: &[u8], value: &[u8]) -> Option> {...} - fn root() -> Vec {...} -} - -#[runtime_interface] -pub trait Hashing { - fn blake2_128(data: &[u8]) -> [u8; 16] { - sp_core::hashing::blake2_128(data) - } -} - -// somewhere in substrate runtime -let hashed_value = sp_io::storage::get(b"key") - .and_then(sp_io::hashing::blake2_128) - .unwrap(); -``` - ---- - -## Considerations - ---- - ## Considerations: Speed - (new) Wasmtime is near-native 🏎️. @@ -975,7 +904,7 @@ wasm can be bigger than the actual hashing cost. --- -### Consideration: Native Runtime +## Consideration: Native Runtime @@ -1031,7 +960,7 @@ fn execute_native_else_wasm() { ### Consideration: Native Runtime - Question: what happens if you upgrade your runtime, but forget to bump the spec version? -- ­ Question: What if a runtime upgrade is only tweaking implementation details, but not the specification? +- ­Question: What if a runtime upgrade is only tweaking implementation details, but not the specification? Notes: @@ -1040,10 +969,10 @@ But, if some are executing native, then you will have a consensus error. ---v -## Speaking of Versions.. +### Speaking of Versions.. - Make sure you understand the difference! πŸ‘ - - Client Version + - Node Version - Runtime Version ---v @@ -1068,7 +997,7 @@ But, if some are executing native, then you will have a consensus error. ### Speaking of Versions.. -- What happens when Parity release a new `parity-polkadot` client binary? +- What happens when Parity release a new `parity-polkadot` node binary? - What happens when the Polkadot fellowship wants to update the runtime? --- @@ -1082,7 +1011,7 @@ But, if some are executing native, then you will have a consensus error. ### Considerations: Panic -- In a more broad sense, all validators never want to avoid wasting their time. +- All validators/nodes want to avoid wasting their time. - While building a block, sometimes it is unavoidable (when?). - While importing a block, nodes will not tolerate this. @@ -1093,7 +1022,6 @@ But, if some are executing native, then you will have a consensus error. - Panic is a (free) form of wasting a validator's time. - Practically Wasm instance killed; State changes reverted. - Any fee payment also reverted. -- Transactions that consume resources but fail to pay fees are similar. Notes: @@ -1157,13 +1085,13 @@ See the documentation of `ApplyExtrinsicResult` in Substrate for more info about --- -### Consideration: Altering Host Function +## Consideration: Altering Host Function - A runtime upgrade now requires a new `sp_io::new_stuff::foo()`. Can we do a normal runtime upgrade?
-- Clients need to upgrade first. No more fully forkless upgrade 😒 +- Nodes need to upgrade first. No more fully forkless upgrade 😒
@@ -1185,7 +1113,7 @@ fn root(&mut self, version: StateVersion) -> Vec { .. }
-- For some period of time, the client needs to support both..πŸ€” +- For some period of time, the node needs to support both..πŸ€”
@@ -1203,7 +1131,7 @@ fn root(&mut self, version: StateVersion) -> Vec { .. } ### Host Functions.. -## NEED TO BE KEPT FOREVER 😈 +#### NEED TO BE KEPT FOREVER 😈 @@ -1358,7 +1286,7 @@ fn root(&mut self, version: StateVersion) -> Vec { .. } --- -### Activity: Expected Panics In The Runtime +## Activity: Expected Panics In The Runtime - Look into the `frame-executive` crate's code. See instances of `panic!()`, and see if you can make sense out of it. @@ -1369,17 +1297,25 @@ graph LR TransactionPool --"😈"--> Authoring --"πŸ˜‡"--> Import
+Note: + +you should for example find that a real FRAME-based runtime, in `execute_block`, we certainly do +panic if any transaction fails to apply. + --- ## Lecture Recap (Part 2) -- Recap the syntax of host functions and runtime APIs. - Considerations: - Speed - Native Execution and Versioning - Panics - Altering Host Functions +- Activity: + - Inspecting a wasm blob for import and export + - Finding panics in `FRAME` runtimes. + --- ## Additional Resources! πŸ˜‹ @@ -1390,13 +1326,10 @@ graph LR Notes: -- Some very recent change the the block building API set: https://github.com/paritytech/substrate/pull/14414 - -- New runtime API for building genesis config: https://github.com/paritytech/substrate/pull/14310 +TODO: update links, most are outdated. +- Some very recent change the the block building API set: https://github.com/paritytech/substrate/pull/14414 / https://github.com/paritytech/polkadot-sdk/pull/1781 -- All Substrate PRs that have added new host functions: https://github.com/paritytech/substrate/issues?q=label%3AE4-newhostfunctions+is%3Aclosed - -- All substrate PRs that have required the client to be update first: https://github.com/paritytech/substrate/issues?q=is%3Aclosed+label%3A%22E10-client-update-first+%F0%9F%91%80%22 +- New runtime API for building genesis config: https://github.com/paritytech/substrate/pull/14310 / https://github.com/paritytech/polkadot-sdk/pull/1492 - New metadata version, including types for the runtime API: https://github.com/paritytech/substrate/issues/12939 @@ -1421,9 +1354,9 @@ SomeExternalities.execute_with(|| { Content that is not covered, but is relevant. ----v +--- -### Consideration: Runtime API Versioning +## Consideration: Runtime API Versioning - Same principle, but generally easier to deal with. - Metadata is part of the runtime, known **per block**. @@ -1431,7 +1364,7 @@ Content that is not covered, but is relevant. Notes: -Also, it is arguable to say that the runtime is the boss here. The client must serve the runtime +Also, it is arguable to say that the runtime is the boss here. The node must serve the runtime fully, but the runtime may or may not want to support certain APIs for certain applications. Recall from another slide: @@ -1442,7 +1375,7 @@ Recall from another slide: ### Consideration: Runtime API Versioning -- The Rust code (which is **statically** typed) in substrate client does care if the change _is breaking_. +- The Rust code (which is **statically** typed) in substrate node does care if the change _is breaking_. - For example, input/output types change. Rust code cannot deal with that! ---v @@ -1481,6 +1414,61 @@ But what you have to do is dependent on the scenario. ### Activity: API Versioning - Look into substrate and find all instances of `#[changed_in(_)]` macro to detect runtime api version. -- Then see if/how this is being used in the client code. +- Then see if/how this is being used in the node code. - Find all the `#[version]` macros in `sp-io` to find all the versioned host functions. + + +--- + +## Inherents and Block Authoring + +- What about Inherents? + +---v + +### Inherents and Block Authoring + +```rust [14-26] +// get the best-block, based on whatever consensus rule we have. +let (best_number, best_hash) = consensus::best_block(); + +// get the latest state. +let mut state = database::state_at(best_hash); + +// initialize a wasm runtime. +let code = state::get(well_known_keys::CODE); +let runtime = wasm::Executor::new(code); + + +// get an empty node block. +let mut block: NodeBlock = Default::default(); + +// tell this runtime that you wish to start a new block, and pass in a raw header. +runtime.initialize_block(&block.header); + +let inherents: Vec> = block_builder::inherents(); +block.extrinsics = inherents; + +// repeatedly apply transactions. +while let Some(next_ext) = transaction_pool_iter::next() { + state.execute(|| { + runtime.apply_extrinsic(next_ext); + }); + block.extrinsics.push(next_ext); +} + +// finalize and set the final header. +block.header = runtime.finalize_block(); +``` + +Notes: + +In reality, the node can either get the inherents itself or ask the runtime about them. There is +actually a runtime API for this too, but we won't cover it. + +while inherents can in principle come at any point in the block, since FRAME restricts them to +come first, we also keep our example aligned. + +Should you wish to see the real version of this, check this crate: +https://paritytech.github.io/substrate/master/sc_basic_authorship/index.html diff --git a/syllabus/5-Substrate/4-Transaction-Pool_Slides.md b/syllabus/5-Substrate/4-Transaction-Pool_Slides.md index e5f21d1a0..389f2c313 100644 --- a/syllabus/5-Substrate/4-Transaction-Pool_Slides.md +++ b/syllabus/5-Substrate/4-Transaction-Pool_Slides.md @@ -1,19 +1,18 @@ --- -title: Substrate's Transaction Pool and its Runtime API -duration: 30 minutes +title: Substrate Transaction Pool --- -# Substrate's Transaction Pool +# Substrate Transaction Pool --- -## Transaction Pools +## Why Transaction Pool? + +Because it is a (⛓️ blockspace) market. - - @@ -47,232 +46,481 @@ Second, it is more accurate to think of the transactions themselves waiting in l Let's take a closer look. ----v +--- -### Paths of a Transaction +## Context + + - Notes: -Previously, in the blockchain module, we saw this figure. -It points out that each node has its own view of the blockchain. -Now I'll show you another layer of detail which is that each node also has its own transaction pool -CLICK +- Gossip +- Queue of all transactions, queue of ready transactions. +- Then given to the block author, only READY ones. ---v -### Paths of a Transaction +### Context - +Any Transaction Pool has two main objectives: -Notes: +1. Validate Transactions + - Ready, Future, or πŸ—‘οΈ? +2. Order them + - Within each list, who is first? + +--- -There are many paths a transaction can take from the user who signed it to a finalized block. -Let's talk through some. -Directly to user's authoring node and into chain is simplest. -Could also be gossiped to other author. -Could even go in a block, get orphaned off, back to tx pool, and then in a new block +## 1. Transaction Validation + +Moving transactions from one list to the other. + + +graph LR + W["🀠 Wild West"] --"😈"--> T["πŸ—‘οΈ"] + W --"πŸ˜‡ βŒ›οΈ"--> R["βœ… Ready"] + W --"πŸ˜‡ ⏳"--> F["⏰ Future"] + ---v -### Pool Validation +### 1. Transaction Validation -- Check signature -- Check that sender can afford fees -- Make sure state is ready for application +* Transaction validity is exclusively outside of the transaction pool, and is **100% determined by the Runtime**. +* Transaction validation should be **cheap** to perform. +* Transaction pool is entirely an **offchain operation**. + * No state change Notes: -When a node gets a transaction, it does some light pre-validation sometimes known as pool validation. -This checking determines whether the transactions is {valid now, valid in some potential future, invalid}. -There is periodic re-validation if transactions have been in the pool for a long time. +Important, must pause and ask! + +- Why is it from the runtime? because the transaction format is opaque and the node doesn't even know what to do with it. +- Why does it have to be cheap? wild west, unpaid, DoS! +- Pesky question: but be aware that from the runtime's perspective, the node could be malicious. The runtime cannot trust the node to obey. ---v -### Pool Prioritization +### 1. Transaction Validation -- Priority Queue -- Prioritized by... - - Fee - - Bribe - - Fee per blockspace -- This is all off-chain +The runtime API. -Notes: +```rust[1-100|6] +impl TaggedTransactionQueue for Runtime { + fn validate_transaction( + source: TransactionSource, + tx: ::Extrinsic, + block_hash: ::Hash, + ) -> TransactionValidity { + .. + } +} +``` -There are a few more things the Substrate tx pool does too, and we will look at them in detail soon. +---v ---- +### 1. Transaction Validation -## Tx Pool Runtime Api +```rust[1-100|5-6|8-9|11-12|14-100|1-100] +pub type TransactionValidity = Result; -```rust -pub trait TaggedTransactionQueue: Core { - fn validate_transaction( - &self, - __runtime_api_at_param__: ::Hash, - source: TransactionSource, - tx: ::Extrinsic, - ) -> Result { ... } +/// This is going into `Ready` or `Future` +pub struct ValidTransaction { + /// If in "Ready", what is the priority? + pub priority: u64, + + /// For how long is this validity? + pub longevity: u64, + + /// Should be propagate it? + pub propagate: bool, + + /// Does this require any other tag to be present in ready? + /// + /// This determines "Ready" or "Future". + pub requires: Vec, + /// Does this provide any tags? + pub provides: Vec, } + +type Tag = Vec ``` -[`TaggedTransactionQueue` Rustdocs](https://paritytech.github.io/substrate/master/sp_transaction_pool/runtime_api/trait.TaggedTransactionQueue.html) +---v + +### 1. Transaction Validation: Banning -Introduced in [paritytech/substrate#728](https://github.com/paritytech/substrate/issues/728) +- Once certain transaction is discovered to be invalid, **its hash** is banned for a fixed duration of time. +- Default in substrate is `Duration::from_secs(60 * 30)`, can be configured via CLI. Notes: -This is another runtime api, similar to the block builder and the core that are used for creating and importing blocks. -Like most others, it requires that the Core api also be implemented. +See: https://github.com/paritytech/substrate/pull/11786 -This one is slightly different in that it is actually called from off-chain, and is not part of your STF. -So let's talk about that for a little bit. +we probably also ban the peer who sent us that transaction? but have to learn. ----v +--- -### Runtime vs STF +## 2. Transaction Ordering - +- `provides` and `requires` is a very flexible mechanism; it allows you to: + - Specify if a transaction is "Ready" or "Future" + - Within each, what transactions should ge before which. -Notes: -It is commonly said that the runtime is basically your STF. -This is a good first order approximation. -It is nearly true. +Note: it essentially forms a graph. + +Order mostly matters within the ready pool. I am not sure if the code maintains an order in `future` as well. In any +case, not a big big deal. ---v -### Runtime vs STF +### 2. Transaction Ordering: Quiz Time. - + + -Notes: +``` +( + A, + provides: vec![], + requires: vec![] +) +``` -But as we can see here, when we put our glasses on, actually only some of the apis are part of the stf. + + + + + + + + + + + + + + + + +
ReadyFuture
+
(A, pr: vec![], rq: vec![])
+
+
+ +
---v -## Why is pool logic in the runtime? +### 2. Transaction Ordering: Quiz Time. + + + -- Transaction type is Opaque -- Runtime logic is opaque -- You must understand the transaction to prioritize it +``` +( + B, + provides: vec![1], + requires: vec![2] +) +``` -Notes: + -So if this is not part of the STF why is it in the runtime at all? -This logic is tightly related to the runtime application logic. -The types are opaque outside of the runtime. -So this logic must go in the runtime. + + + + + + + + + + + + + + +
ReadyFuture
+
(A, pr: vec![], rq: vec![])
+
+
(B, pr: vec![1], rq: vec![2])
+
+
-But if it is not on-chain, can individual validators customize it. -In short yes. -There is a mechanism for this. -We won't go deeply into the mechanism, but validators can specify alternate wasm blocs to use instead of the official one. +
---- +---v -## Jobs of the API +### 2. Transaction Ordering: Quiz Time. -- Make fast pre-checks -- Give the transaction a priority -- Determine whether the transaction is ready now or may be ready in the future -- Determine a dependency graph among the transactions + + -Notes: +``` +( + C, + provides: vec![2], + requires: vec![3] +) +``` -So we spoke earlier about the jobs of a transaction pool in general. -Specifically the pre-checks and the priority -Here is a more specific list of tasks that Substrate's TaggedTransactionPool does. + -The second two points are the new additions, and they are the duty of the "tags" after which the tagged transaction queue is named. + + + + + + + + + + + + + + + + + + +
ReadyFuture
+
(A, pr: vec![], rq: vec![])
+
+
(B, pr: vec![1], rq: vec![2])
+
+ +
(C, pr: vec![2], rq: vec![3])
+
+
-The results of all of this are returned to the client side through a shared type `ValidTransaction` or `InvalidTransaction` +
---v -### `ValidTransaction` +### 2. Transaction Ordering: Quiz Time. -```rust -pub struct ValidTransaction { - pub priority: TransactionPriority, - pub requires: Vec, - pub provides: Vec, - pub longevity: TransactionLongevity, - pub propagate: bool, -} + + + +``` +( + D, + provides: vec![1], + requires: vec![0] +) ``` -[`ValidTransaction` Rustdocs](https://paritytech.github.io/substrate/master/sp_runtime/transaction_validity/struct.ValidTransaction.html) + -Notes: + + + + + + + + + + + + + + + + + + + + + + + + + + +
ReadyFuture
+
(A, pr: vec![], rq: vec![])
+
+
+
(D, pr: vec![1], rq: vec![])
+
+
+
(B, pr: vec![1], rq: vec![2])
+
+
+
(C, pr: vec![2], rq: vec![3])
+
+
+
-We indicate that the transaction passes the prechecks at all by returning this valid transaction struct. -If it weren't even valid, we would return a different, `InvalidTransaction` struct. -You learned yesterday how to navigate the rustdocs to find the docs on that one. +
+ +Note: The oder in this slide matters and it is top to bottom. + +---v -Priority we've discussed. -It is worth noting that the notion of priority is intentionally opaque to the client. -The runtime may assign this value however it sees fit. +### 2. Transaction Ordering: `priority` -Provides and requires all forming a dependency graph between the transactions. -Requires is a list of currently unmet dependency transactions. -This transaction will be ready in a future where these dependencies are met so it is kept in the pool. +From the **Ready pool**, when all requirements are met, then `priority` dictates the order. -A simple intuitive example of this is payments. -Image alice pays bob some tokens in transaction1. -Then bob pays those same tokes to charlie in transaction2. -trasnaction2 will be valid only after transaction1 has been applied. -It is a dependency. +Further tie breakers: -Longevity is a field I'm not so familiar with. -It is how long the transaction should stay in the pool before being dropped or re-validated. -TODO what are the units? How does one set it? +2. ttl: shortest `longevity` goes first +3. time in the queue: longest to have waited goes first -And finally whether the transaction should be gossiped. -This is usually true. -Only in special edge cases would this be false. + + +Note: + +https://github.com/paritytech/polkadot-sdk/blob/bc53b9a03a742f8b658806a01a7bf853cb9a86cd/substrate/client/transaction-pool/src/graph/ready.rs#L146 ---v -### Example 1: UTXO System +### 2. Transaction Ordering: `priority` - +> How can the pool be a pure FIFO? Notes: -Prioritize by implicit tip (difference of inputs and outputs) -Requires all missing input transactions -provides this input +All priorities set to 0. + +---v + +### 2. Transaction Ordering: `nonce` + +Purposes of a nonce: + +1. Ordering +2. Replay protection +3. Double spend protection ---v -### Example 2: Nonced Account System +### 2. Transaction Ordering: `nonce` + +- βœ… You will implement a nonce system using the above primitives as a part of your assignment. +- General idea: `require -> (account, nonce - 1).encode()`, provide: `provides -> (account, nonce).encode()` + - Notes: -Prioritize by explicit tip -Requires all previous nonces for this account -provides this nonce for this account +Transaction Ordering: Each time a transaction is sent from an account, the nonce increases by one. This sequential +numbering ensures that transactions are processed in the order they are sent. + +Double Spending Prevention: Since each transaction has a unique nonce, it's impossible for two transactions with the +same nonce to both be valid. This stops attackers from attempting to send the same funds twice. + +Replay Attack Protection: In a replay attack, a valid transaction is maliciously or fraudulently repeated or delayed. +With a nonce, once a transaction is executed, any attempt to execute it again will fail since the nonce will no longer +match the current state of the account. + +--- -This demonstrates one of the biggest downsides of the Accounts system. -Transactions cannot deterministically specify the initial state on which they operate. -There is only an inherent ordering between transactions from the same account. +## Shower Thought: Runtime vs STF + +> Transaction pool is entirely an **offchain operation** + +Note: + +what we said before. What does this imply? ---v -## Always Re-check On-chain +### Shower Thought: Runtime vs STF - + Notes: -None of this new pool information changes the fundamentals you learned last week. -You must execute the state transitions in full on chain. +It is commonly said that the runtime is basically your STF. +This is a good first order approximation. +It is nearly true. +---v -Most of the time you are not the block author. -When you import a block from another node, you cannot trust them to have done the pre-checks correctly. +### Shower Thought: Runtime vs STF + + + +Notes: + +But as we can see here, when we put our glasses on, actually only some of the apis are part of the stf. the non-stf +parts are runtime APIs that are called and use, but don't really contribute to the STF. typically the runtime cannot +make assumptions about these. From the PoV of the runtime, when doing the main consensus critical work (authoring, +importing) these did not happen. + +--- + +## Lecture Recap + +- Blockspace Market, Competition. +- Main tasks: + - Validate (valid and invalid) + - Order (split valid into "Ready" and "Future") + - provides and requires + - priority: Fee/tip +- By Runtime, but not in STF + +Notes: + +- Each node's pool is a local wild west. +- Because it is wild west, the transaction pool must only check static and cheap things. +- The block author won't trust the pool validation, and re-execute all checks. +- Shower Thought: Transaction queue validation is part of the runtime, but not part of the STF. + +--- + +## Additional Resources + +> Check speaker notes (click "s" πŸ˜‰) + + + +Notes: + +https://github.com/paritytech/polkadot-sdk/blob/bc53b9a03a742f8b658806a01a7bf853cb9a86cd/substrate/client/transaction-pool/src/graph/ready.rs#L149 + +Original pool PR from ages ago, old but gold: https://github.com/paritytech/substrate/issues/728 + +> Work towards a flexible transaction queue that relies **only on runtime logic to provide comprehensive dependency and queuing management**... should not be aware of the concepts of accounts, signatures, indexes or nonces. + +> Returns `Valid` if the transaction can be **statically** validated; ... the u64 is the priority used to determine which of a mutually exclusive set of transactions are better to include... Any transactions that do get included in a block should be instantly discarded (and banned) if they result in a panic execution. + +--- + +# Appendix + +--- + +## Transaction Pool Submission API + +```rust +pub enum TransactionStatus { + /// Transaction is part of the future queue. + Future, + /// Transaction is part of the ready queue. + Ready, + /// The transaction has been broadcast to the given peers. + Broadcast(Vec), + /// Transaction has been included in block with given hash. + InBlock(BlockHash), + /// The block this transaction was included in has been retracted. + Retracted(BlockHash), + /// Maximum number of finality watchers has been reached, + /// old watchers are being removed. + FinalityTimeout(BlockHash), + /// Transaction has been finalized by a finality-gadget, e.g GRANDPA + Finalized(BlockHash), + /// Transaction has been replaced in the pool, by another transaction + /// that provides the same tags. (e.g. same (sender, nonce)). + Usurped(Hash), + /// Transaction has been dropped from the pool because of the limit. + Dropped, + /// Transaction is no longer valid in the current state. + Invalid, +} +```