From df69df19c66465497e3e95cb178d361c23cd87cd Mon Sep 17 00:00:00 2001 From: frankie Date: Mon, 14 Oct 2024 12:19:21 -1000 Subject: [PATCH 1/4] add time out --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eff4f8c7..7f1695d1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,6 +15,7 @@ jobs: permissions: contents: read runs-on: ubuntu-22.04 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 From 2e063b2585ad0bf8bd4c965cd0a638761d74607b Mon Sep 17 00:00:00 2001 From: frankie Date: Mon, 14 Oct 2024 12:21:10 -1000 Subject: [PATCH 2/4] wip: siging --- src/signing/index.ts | 87 +++++++++++--------------------------- src/utils/index.ts | 17 +++++--- tests/end-to-end.test.ts | 2 +- tests/programs-dev.test.ts | 12 +++--- 4 files changed, 42 insertions(+), 76 deletions(-) diff --git a/src/signing/index.ts b/src/signing/index.ts index 86509768..3f594cc8 100644 --- a/src/signing/index.ts +++ b/src/signing/index.ts @@ -1,10 +1,9 @@ import { ApiPromise } from '@polkadot/api' -import { hexAddPrefix } from '@polkadot/util' import { Signer } from '../keys/types/internal' import { defaultAdapters } from './adapters/default' import { Adapter } from './adapters/types' import { ValidatorInfo } from '../types/internal' -import { stripHexPrefix, sendHttpPost, toHex } from '../utils' +import { stripHexPrefix, sendHttpPost, toHex, } from '../utils' import { crypto } from '../utils/crypto' import { CryptoLib } from '../utils/crypto/types' import Keyring from '../keys' @@ -199,8 +198,8 @@ export default class SignatureRequestManager { // @ts-ignore: next line const validators: string[] = (await this.substrate.query.session.validators()).toHuman() // @ts-ignore: next line - const signingGroup: string[] = (await this.substrate.query.stakingExtension.signers()).toHuman() - const validatorInfo: ValidatorInfo = await this.pickValidator(validators, signingGroup) + const signingCommittee: string[] = (await this.substrate.query.stakingExtension.signers()).toHuman() + const validatorInfo: ValidatorInfo = await this.pickValidator(validators, signingCommittee) // TO-DO: this needs to be and accounId ie hex string of the address // which means you need a new key ie device key here @@ -220,7 +219,7 @@ export default class SignatureRequestManager { const message: EncMsg = await this.formatTxRequest(txRequest) const sigs = await this.submitTransactionRequest(message) - const sig = await this.verifyAndReduceSignatures(sigs) + const sig = await this.verifyAndReduceSignatures(sigs, signingCommittee) return Uint8Array.from(atob(sig), (c) => c.charCodeAt(0)) } @@ -261,7 +260,6 @@ export default class SignatureRequestManager { hash?: string signatureVerifyingKey: string }): Promise { - console.log('validator', validator) const txRequestData: UserSignatureRequest = { message: stripHexPrefix(strippedsigRequestHash), auxilary_data: auxiliaryData, @@ -311,13 +309,10 @@ export default class SignatureRequestManager { async submitTransactionRequest (message: EncMsg): Promise { // Extract the required fields from parsedMsg const payload = JSON.parse(message.msg) - console.log('EncMsg', message) - console.log('payload', payload) const sigProof = (await sendHttpPost( `http://${message.url}/user/relay_tx`, JSON.stringify(payload) )) - console.log('fetch returned:',sigProof, new Date(Date.now())) return sigProof } @@ -328,24 +323,15 @@ export default class SignatureRequestManager { * @returns {Promise} A promise resolving to an array of validator information. */ - async pickValidator (validators: string[], signingGroup: string[]): Promise { + async pickValidator (validators: string[], signingCommittee: string[]): Promise { const relayers = validators.reduce((agg, stashKey) => { - if (signingGroup.includes(stashKey)) return agg + if (signingCommittee.includes(stashKey)) return agg agg.push(stashKey) return agg }, []) - const info = await Promise.all(validators.map(async (stashKey) => { - const i = (await this.substrate.query.stakingExtension.thresholdServers(stashKey)).toHuman() - // @ts-ignore - return {...i, stashKey, relayer: relayers.includes(stashKey)} - })) - console.log('validators', validators, info) - console.log('signingGroup', signingGroup) - console.log('relayers', relayers) // pick a relayer at random const stashKey = relayers[Math.floor(Math.random() * relayers.length)] const rawValidatorInfo = (await this.substrate.query.stakingExtension.thresholdServers(stashKey)).toHuman() - console.log('rawValidatorInfo', rawValidatorInfo) // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore const { x25519PublicKey, endpoint, tssAccount } = rawValidatorInfo @@ -363,47 +349,24 @@ export default class SignatureRequestManager { * @returns The first valid signature after verification. */ - async verifyAndReduceSignatures (sigsAndProofs: string[][]): Promise { - const seperatedSigsAndProofs = sigsAndProofs.reduce( - (a, sp) => { - if (!sp || !sp.length) return a - // the place holder is for holding an index. in the future we should notify - // the nodes or something about faulty validators - // this is really just good house keeping because you never know? - a.sigs.push(sp[0] || 'place-holder') - a.proofs.push(sp[1] || 'place-holder') - a.addresses.push(sp[2] || 'place-holder') - return a - }, - { sigs: [], proofs: [], addresses: [] } - ) - // find a valid signature - const sigMatch = seperatedSigsAndProofs.sigs.find( - (s) => s !== 'place-holder' - ) - if (!sigMatch) throw new Error('Did not receive a valid signature') - // use valid signature to see if they all match - const allSigsMatch = seperatedSigsAndProofs.sigs.every( - (s) => s === sigMatch - ) - if (!allSigsMatch) throw new Error('All signatures do not match') - // in the future. notify network of compromise? - // check to see if the tss_account signed the proof - const validated = await Promise.all( - seperatedSigsAndProofs.proofs.map( - async (proof: string, index: number): Promise => { - return await this.crypto.verifySignature( - seperatedSigsAndProofs.sigs[index], - proof, - seperatedSigsAndProofs.addresses[index] - ) - } - ) - ) - const first = validated.findIndex((v) => v) - if (first === -1) - throw new Error('Can not validate the identity of any validator') - - return seperatedSigsAndProofs.sigs[first] + async verifyAndReduceSignatures (sigsAndProofs: string[][], signingCommittee): Promise { + // take the sigs and proofs and until a valid proof is found + // keep trying + let validated = false + let sig, proof + while (!validated && !!sigsAndProofs.length) { + [sig, proof] = sigsAndProofs.pop() + let index = 0 + do { + validated = await this.crypto.verifySignature( + sig, + proof, + signingCommittee[index] + ) + ++index + } while (index < signingCommittee.length && !validated) + } + if (!validated) throw new Error('invalid signature') + return sig } } diff --git a/src/utils/index.ts b/src/utils/index.ts index 62612106..ace2110f 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -76,8 +76,6 @@ export async function sendHttpPost (url: string, data: any): Promise { method: 'POST', headers, body: data, - // 25 seconds - signal: AbortSignal.timeout(5000 * 5) }) console.log('status', response.status) if (!response.ok) { @@ -110,9 +108,16 @@ export async function sendHttpPost (url: string, data: any): Promise { } FULLRESPONSE: ${await streamResponse.text()}` ) } - return (await streamResponse.json()).Ok + const responseResult = await streamResponse.arrayBuffer() + const decoder = new TextDecoder() + const str = decoder.decode(responseResult) + const parsed = JSON.parse(str) + const oks = parsed.map(r => r.Ok) + return oks } + + /** * Converts an ArrayBuffer to a hexadecimal string. * @@ -120,10 +125,8 @@ export async function sendHttpPost (url: string, data: any): Promise { * @returns {string} The hexadecimal representation of the buffer. */ -export function buf2hex (buffer: ArrayBuffer): string { - return [...new Uint8Array(buffer)] - .map((x) => x.toString(16).padStart(2, '0')) - .join('') +export function bufferToHex (buffer: ArrayBuffer): string { + return Buffer.from(buffer).toString('hex') } export function toHex (str: any) { diff --git a/tests/end-to-end.test.ts b/tests/end-to-end.test.ts index b7ee1bfc..3c8c3948 100644 --- a/tests/end-to-end.test.ts +++ b/tests/end-to-end.test.ts @@ -19,7 +19,7 @@ const networkType = 'four-nodes' const msg = Buffer.from('Hello world: signature from entropy!').toString('hex') -test.only('End To End', async (t) => { +test('End To End', async (t) => { const run = promiseRunner(t) await run('network up', spinNetworkUp(networkType)) t.teardown(async () => { diff --git a/tests/programs-dev.test.ts b/tests/programs-dev.test.ts index 1db35208..54a5b8db 100644 --- a/tests/programs-dev.test.ts +++ b/tests/programs-dev.test.ts @@ -98,7 +98,7 @@ test('Programs#dev: all methods', async (t) => { 'auxiliaryDataSchema on chain should match what was deployed' ) - run( + await run( 'remove noopProgram', entropy.programs.dev.remove(newPointer) ) @@ -112,11 +112,11 @@ test('Programs#dev: all methods', async (t) => { // functionality to begin with so ive commented this out // for now but this needs digging // see issue https://github.com/entropyxyz/sdk/issues/414 - // t.equal( - // programsDeployedAfterRemove.length, - // 0, - // 'charlie has no deployed programs' - // ) + t.equal( + programsDeployedAfterRemove.length, + 0, + 'charlie has no deployed programs' + ) t.end() From a27ac9d5462a828b4741a25e89a34ab4987aa69b Mon Sep 17 00:00:00 2001 From: frankie Date: Tue, 15 Oct 2024 08:56:48 -1000 Subject: [PATCH 3/4] two-nodes -> four-nodes --- dev/README.md | 8 ++++---- package.json | 2 +- tests/sign.test.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/README.md b/dev/README.md index 340eb5e0..e5d03be1 100644 --- a/dev/README.md +++ b/dev/README.md @@ -53,7 +53,7 @@ docker container list You can close them down like this: ```bash -docker compose --file dev/docker-scripts/two-nodes.yaml down +docker compose --file dev/docker-scripts/four-nodes.yaml down ``` ### Gotcha 2 - ports still in use? @@ -82,7 +82,7 @@ export ENTROPY_CORE_VERSION=latest If you must do this you should run ```bash -docker compose --file dev/docker-scripts/two-nodes.yaml pull +docker compose --file dev/docker-scripts/four-nodes.yaml pull ``` ## When updating core version: @@ -93,9 +93,9 @@ docker compose --file dev/docker-scripts/two-nodes.yaml pull directly from the root directory and then call the generate types script and then the spin down script: -- `dev/bin/spin-up.sh two-nodes` +- `dev/bin/spin-up.sh four-nodes` - `dev/bin/generate-types.sh` -- `dev/bin/spin-down.sh two-nodes` +- `dev/bin/spin-down.sh four-nodes` 2. run `yarn tsc` just to make sure that went "okay" or as okay as it can be. generated types are ignored in tsc check but are used in project (kind of not diff --git a/package.json b/package.json index e0bed72a..acef068d 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "postinstall": "patch-package", "prepack": "pinst --disable", "postpack": "pinst --enable", - "removedb": "rm -rf .entropy && docker compose --file dev/docker-scripts/two-nodes.yaml down 2> /dev/null" + "removedb": "rm -rf .entropy && docker compose --file dev/docker-scripts/four-nodes.yaml down 2> /dev/null" }, "repository": { "type": "git", diff --git a/tests/sign.test.ts b/tests/sign.test.ts index 013f686a..755d7806 100644 --- a/tests/sign.test.ts +++ b/tests/sign.test.ts @@ -11,7 +11,7 @@ import { charlieSeed, } from './testing-utils' -const NETWORK_TYPE = 'two-nodes' +const NETWORK_TYPE = 'four-nodes' const msg = Buffer .from('Hello world: new signature from entropy!') From 2bbce3f9f70e3ef0db6d8403689ccd2939c7a773 Mon Sep 17 00:00:00 2001 From: frankie Date: Tue, 15 Oct 2024 09:10:45 -1000 Subject: [PATCH 4/4] throw in jumpStart if it's not a full validator network we are jump starting. otherwise jumpstart never resolves --- dev/testing-utils.mjs | 5 +++++ tests/four-nodes.test.ts | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/testing-utils.mjs b/dev/testing-utils.mjs index 0fc4bb24..1f75185f 100644 --- a/dev/testing-utils.mjs +++ b/dev/testing-utils.mjs @@ -10,6 +10,7 @@ const moduleRoot = join(__dirname, '..') // NOTE: you need to edit your /etc/hosts file to use these. See dev/README.md export async function spinNetworkUp (networkType = 'two-nodes') { + global.networkType = networkType try { execFileSync('dev/bin/spin-up.sh', [networkType], { shell: true, @@ -69,6 +70,10 @@ async function isWebSocketReady (endpoint) { } export async function jumpStartNetwork (entropy) { + // if you used spinNetworkUp check what network was used + // this is done this way so we can still use this for other + // applications + if (global.networkType && global.networkType !== 'four-nodes') throw new Error(`jump start requires four-nodes network you are running: ${global.networkType}`) await entropy.substrate.tx.registry.jumpStartNetwork().signAndSend(entropy.keyring.accounts.registration.pair) const wantedMethod = 'FinishedNetworkJumpStart' let unsub diff --git a/tests/four-nodes.test.ts b/tests/four-nodes.test.ts index 4f58e24c..a4e6233a 100644 --- a/tests/four-nodes.test.ts +++ b/tests/four-nodes.test.ts @@ -51,7 +51,6 @@ test('test the four-nodes docker script for subgroups', async (t) => { await run('jump Start Network', jumpStartNetwork(entropy)) const validators = (await run('validators', entropy.substrate.query.session.validators())).toHuman() const signingGroup = (await run('signingGroup', entropy.substrate.query.stakingExtension.signers())).toHuman() - console.log('validators, signingGroup', validators, signingGroup) t.equal(validators.length, 4, 'expecting 4 validators in validator set') t.equal(signingGroup.length, 3, 'expecting 3 validators in the signing group') await entropy.close()