Merge branch 'frankie/core#v0.3.0.rc-1' into mixmix/core-release-tweaks

.github/workflows/test.yaml

@@ -16,6 +16,7 @@ jobs:
     permissions:
       contents: read
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4

dev/README.md

@@ -53,7 +53,7 @@ docker container list
 You can close them down like this:
 
 ```bash
-docker compose --file dev/docker-scripts/two-nodes.yaml down
+docker compose --file dev/docker-scripts/four-nodes.yaml down
 ```
 
 ### Gotcha 2 - ports still in use?
@@ -82,7 +82,7 @@ export ENTROPY_CORE_VERSION=latest
 If you must do this you should run
 
 ```bash
-docker compose --file dev/docker-scripts/two-nodes.yaml pull
+docker compose --file dev/docker-scripts/four-nodes.yaml pull
 ```
 
 ## When updating core version:
@@ -93,9 +93,9 @@ docker compose --file dev/docker-scripts/two-nodes.yaml pull
    directly from the root directory and then call the generate types script and
    then the spin down script:
 
-- `dev/bin/spin-up.sh two-nodes`
+- `dev/bin/spin-up.sh four-nodes`
 - `dev/bin/generate-types.sh`
-- `dev/bin/spin-down.sh two-nodes`
+- `dev/bin/spin-down.sh four-nodes`
 
 2. run `yarn tsc` just to make sure that went "okay" or as okay as it can be.
    generated types are ignored in tsc check but are used in project (kind of not

dev/testing-utils.mjs

@@ -11,6 +11,7 @@ const SECONDS = 1000
 // NOTE: you need to edit your /etc/hosts file to use these. See dev/README.md
 
 export async function spinNetworkUp (networkType = 'two-nodes') {
+  global.networkType = networkType
   try {
     execFileSync('dev/bin/spin-up.sh', [networkType], { 
       shell: true, 
@@ -71,6 +72,11 @@ async function isWebSocketReady (endpoint) {
 
 export async function jumpStartNetwork (entropy, maxTime = 120 * SECONDS) {
   let timeout, unsub
+  // if you used spinNetworkUp check what network was used
+  // this is done this way so we can still use this for other
+  // applications
+  if (global.networkType && global.networkType !== 'four-nodes') throw new Error(`jump start requires four-nodes network you are running: ${global.networkType}`)
+  await entropy.substrate.tx.registry.jumpStartNetwork().signAndSend(entropy.keyring.accounts.registration.pair)
   const wantedMethod = 'FinishedNetworkJumpStart'
 
   const isDone = new Promise(async (res, reject) => {

src/signing/index.ts

@@ -1,10 +1,9 @@
 import { ApiPromise } from '@polkadot/api'
-import { hexAddPrefix } from '@polkadot/util'
 import { Signer } from '../keys/types/internal'
 import { defaultAdapters } from './adapters/default'
 import { Adapter } from './adapters/types'
 import { ValidatorInfo } from '../types/internal'
-import { stripHexPrefix, sendHttpPost, toHex } from '../utils'
+import { stripHexPrefix, sendHttpPost, toHex, } from '../utils'
 import { crypto } from '../utils/crypto'
 import { CryptoLib } from '../utils/crypto/types'
 import Keyring from '../keys'
@@ -199,8 +198,8 @@ export default class SignatureRequestManager {
     // @ts-ignore: next line
     const validators: string[] = (await this.substrate.query.session.validators()).toHuman()
     // @ts-ignore: next line
-    const signingGroup: string[] = (await this.substrate.query.stakingExtension.signers()).toHuman()
-    const validatorInfo: ValidatorInfo = await this.pickValidator(validators, signingGroup)
+    const signingCommittee: string[] = (await this.substrate.query.stakingExtension.signers()).toHuman()
+    const validatorInfo: ValidatorInfo = await this.pickValidator(validators, signingCommittee)
     // TO-DO: this needs to be and accounId ie hex string of the address
     // which means you need a new key ie device key here
 
@@ -220,7 +219,7 @@ export default class SignatureRequestManager {
 
     const message: EncMsg = await this.formatTxRequest(txRequest)
     const sigs = await this.submitTransactionRequest(message)
-    const sig = await this.verifyAndReduceSignatures(sigs)
+    const sig = await this.verifyAndReduceSignatures(sigs, signingCommittee)
     return Uint8Array.from(atob(sig), (c) => c.charCodeAt(0))
   }
 
@@ -261,7 +260,6 @@ export default class SignatureRequestManager {
     hash?: string
     signatureVerifyingKey: string
   }): Promise<EncMsg> {
-    console.log('validator', validator)
     const txRequestData: UserSignatureRequest = {
       message: stripHexPrefix(strippedsigRequestHash),
       auxilary_data: auxiliaryData,
@@ -311,13 +309,10 @@ export default class SignatureRequestManager {
   async submitTransactionRequest (message: EncMsg): Promise<string[][]> {
     // Extract the required fields from parsedMsg
     const payload = JSON.parse(message.msg)
-    console.log('EncMsg', message)
-    console.log('payload', payload)
     const sigProof = (await sendHttpPost(
       `http://${message.url}/user/relay_tx`,
       JSON.stringify(payload)
     ))
-    console.log('fetch returned:',sigProof, new Date(Date.now()))
     return sigProof
   }
 
@@ -328,24 +323,15 @@ export default class SignatureRequestManager {
    * @returns {Promise<ValidatorInfo[]>} A promise resolving to an array of validator information.
    */
 
-  async pickValidator (validators: string[], signingGroup: string[]): Promise<ValidatorInfo> {
+  async pickValidator (validators: string[], signingCommittee: string[]): Promise<ValidatorInfo> {
     const relayers = validators.reduce((agg, stashKey) => {
-      if (signingGroup.includes(stashKey)) return agg
+      if (signingCommittee.includes(stashKey)) return agg
       agg.push(stashKey)
       return agg
     }, [])
-    const info = await Promise.all(validators.map(async (stashKey) => {
-      const i = (await this.substrate.query.stakingExtension.thresholdServers(stashKey)).toHuman()
-      // @ts-ignore
-      return {...i, stashKey, relayer: relayers.includes(stashKey)}
-    }))
-    console.log('validators', validators, info)
-    console.log('signingGroup', signingGroup)
-    console.log('relayers', relayers)
     // pick a relayer at random
     const stashKey = relayers[Math.floor(Math.random() * relayers.length)]
     const rawValidatorInfo = (await this.substrate.query.stakingExtension.thresholdServers(stashKey)).toHuman()
-    console.log('rawValidatorInfo', rawValidatorInfo)
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore
     const { x25519PublicKey, endpoint, tssAccount } = rawValidatorInfo
@@ -363,47 +349,24 @@ export default class SignatureRequestManager {
    * @returns The first valid signature after verification.
    */
 
-  async verifyAndReduceSignatures (sigsAndProofs: string[][]): Promise<string> {
-    const seperatedSigsAndProofs = sigsAndProofs.reduce(
-      (a, sp) => {
-        if (!sp || !sp.length) return a
-        // the place holder is for holding an index. in the future we should notify
-        // the nodes or something about faulty validators
-        // this is really just good house keeping because you never know?
-        a.sigs.push(sp[0] || 'place-holder')
-        a.proofs.push(sp[1] || 'place-holder')
-        a.addresses.push(sp[2] || 'place-holder')
-        return a
-      },
-      { sigs: [], proofs: [], addresses: [] }
-    )
-    // find a valid signature
-    const sigMatch = seperatedSigsAndProofs.sigs.find(
-      (s) => s !== 'place-holder'
-    )
-    if (!sigMatch) throw new Error('Did not receive a valid signature')
-    // use valid signature to see if they all match
-    const allSigsMatch = seperatedSigsAndProofs.sigs.every(
-      (s) => s === sigMatch
-    )
-    if (!allSigsMatch) throw new Error('All signatures do not match')
-    // in the future. notify network of compromise?
-    // check to see if the tss_account signed the proof
-    const validated = await Promise.all(
-      seperatedSigsAndProofs.proofs.map(
-        async (proof: string, index: number): Promise<boolean> => {
-          return await this.crypto.verifySignature(
-            seperatedSigsAndProofs.sigs[index],
-            proof,
-            seperatedSigsAndProofs.addresses[index]
-          )
-        }
-      )
-    )
-    const first = validated.findIndex((v) => v)
-    if (first === -1)
-      throw new Error('Can not validate the identity of any validator')
-
-    return seperatedSigsAndProofs.sigs[first]
+  async verifyAndReduceSignatures (sigsAndProofs: string[][], signingCommittee): Promise<string> {
+    // take the sigs and proofs and until a valid proof is found
+    // keep trying
+    let validated = false
+    let sig, proof
+    while (!validated && !!sigsAndProofs.length) {
+      [sig, proof] = sigsAndProofs.pop()
+      let index = 0
+      do {
+        validated = await this.crypto.verifySignature(
+          sig,
+          proof,
+          signingCommittee[index]
+        )
+        ++index
+      } while (index < signingCommittee.length && !validated)
+    }
+    if (!validated) throw new Error('invalid signature')
+    return sig
   }
 }

src/utils/index.ts

@@ -76,8 +76,6 @@ export async function sendHttpPost (url: string, data: any): Promise<any> {
     method: 'POST',
     headers,
     body: data,
-    // 25 seconds
-    signal: AbortSignal.timeout(5000 * 5)
   })
   console.log('status', response.status)
   if (!response.ok) {
@@ -110,20 +108,25 @@ export async function sendHttpPost (url: string, data: any): Promise<any> {
       } FULLRESPONSE: ${await streamResponse.text()}`
     )
   }
-  return (await streamResponse.json()).Ok
+  const responseResult = await streamResponse.arrayBuffer()
+  const decoder = new TextDecoder()
+  const str = decoder.decode(responseResult)
+  const parsed = JSON.parse(str)
+  const oks = parsed.map(r => r.Ok)
+  return oks
 }
 
+
+
 /**
  * Converts an ArrayBuffer to a hexadecimal string.
  *
  * @param {ArrayBuffer} buffer - The buffer to convert.
  * @returns {string} The hexadecimal representation of the buffer.
  */
 
-export function buf2hex (buffer: ArrayBuffer): string {
-  return [...new Uint8Array(buffer)]
-    .map((x) => x.toString(16).padStart(2, '0'))
-    .join('')
+export function bufferToHex (buffer: ArrayBuffer): string {
+  return Buffer.from(buffer).toString('hex')
 }
 
 export function toHex (str: any) {

tests/four-nodes.test.ts

@@ -50,7 +50,6 @@ test('test the four-nodes docker script', async (t) => {
   await run('jump-start network', jumpStartNetwork(entropy))
   const validators = (await run('validators', entropy.substrate.query.session.validators())).toHuman()
   const signingGroup = (await run('signingGroup', entropy.substrate.query.stakingExtension.signers())).toHuman()
-  console.log('validators, signingGroup', validators, signingGroup)
   t.equal(validators.length, 4, 'expecting 4 validators in validator set')
   t.equal(signingGroup.length, 3, 'expecting 3 validators in the signing group')
   await entropy.close()

tests/programs-dev.test.ts

@@ -98,7 +98,7 @@ test('Programs#dev: all methods', async (t) => {
     'auxiliaryDataSchema on chain should match what was deployed'
   )
 
-  run(
+  await run(
     'remove noopProgram',
     entropy.programs.dev.remove(newPointer)
   )
@@ -112,11 +112,11 @@ test('Programs#dev: all methods', async (t) => {
   // functionality to begin with so ive commented this out
   // for now but this needs digging
   // see issue https://github.com/entropyxyz/sdk/issues/414
-  // t.equal(
-  //   programsDeployedAfterRemove.length,
-  //   0,
-  //   'charlie has no deployed programs'
-  // )
+  t.equal(
+    programsDeployedAfterRemove.length,
+    0,
+    'charlie has no deployed programs'
+  )
 
 
   t.end()

tests/sign.test.ts

@@ -11,7 +11,7 @@ import {
   charlieSeed,
 } from './testing-utils'
 
-const NETWORK_TYPE = 'two-nodes'
+const NETWORK_TYPE = 'four-nodes'
 
 const msg = Buffer
   .from('Hello world: new signature from entropy!')