Expose Pippenger multiplication for combining multiple sigs of same m…

…sg (#178)

* Expose Pippenger multiplication for combining multiple sigs of same msg

In many use cases, there are multiple signatures of the same message,
e.g., Ethereum attestations often share the signed `AttestationData`.

For that situation, `blst` started exposing Pippenger multiplication
to accelerate this use case. Multiscalar multiplication is much faster
than individual scalar multiplication of each signature / pubkey.

Further optimizations may be achieved with parallel tiling, see the Rust
binding code in the `npoints >= 32` situation:

- https://github.com/supranational/blst/blob/v0.3.13/bindings/rust/src/pippenger.rs

Likewise, multiple pubkeys / signatures may be loaded simultaneously
using the new `blst` APIs.

We don't do either of these additional optimizations as our architecture
does not readily support them. Pippenger multiplication alone already
offers a significant speedup until prioritizing further optimizations.

```
------------------------------------------------------------------------------------------------------------------------------------
BLS verif of 6 msgs by 6 pubkeys                                                117.232 ops/s      8530098 ns/op     20471994 cycles
BLS verif of 6 sigs of same msg by 6 pubkeys (with blinding)                    553.186 ops/s      1807711 ns/op      4338371 cycles
BLS verif of 6 sigs of same msg by 6 pubkeys                                    724.279 ops/s      1380683 ns/op      3313617 cycles
------------------------------------------------------------------------------------------------------------------------------------
BLS verif of 60 msgs by 60 pubkeys                                               11.131 ops/s     89839743 ns/op    215615251 cycles
BLS verif of 60 sigs of same msg by 60 pubkeys (with blinding)                  238.059 ops/s      4200634 ns/op     10081380 cycles
BLS verif of 60 sigs of same msg by 60 pubkeys                                  680.634 ops/s      1469219 ns/op      3526031 cycles
------------------------------------------------------------------------------------------------------------------------------------
BLS verif of 180 msgs by 180 pubkeys                                              3.887 ops/s    257298895 ns/op    617517127 cycles
BLS verif of 180 sigs of same msg by 180 pubkeys (with blinding)                166.340 ops/s      6011785 ns/op     14428186 cycles
BLS verif of 180 sigs of same msg by 180 pubkeys                                536.938 ops/s      1862413 ns/op      4469689 cycles
------------------------------------------------------------------------------------------------------------------------------------
```

* Suppress `const` warning for Windows build

* Different approach for dealing with [-Wincompatible-pointer-types]

* Extend documentation

benchmarks/bench_all.nim

@@ -1,3 +1,12 @@
+# Nim-BLSCurve
+# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
 import
   std/[os, strutils, cpuinfo],
   ../blscurve,
@@ -29,28 +38,31 @@ benchFastAggregateVerify(numKeys = 128, iters = 10)
 separator()
 
 when BLS_BACKEND == BLST:
-    var nthreads: int
-    if existsEnv"TP_NUM_THREADS":
-      nthreads = getEnv"TP_NUM_THREADS".parseInt()
-    else:
-      nthreads = countProcessors()
-
-    # Simulate Block verification (at most 6 signatures per block)
-    batchVerifyMulti(numSigs = 6, iters = 10)
-    batchVerifyMultiBatchedSerial(numSigs = 6, iters = 10)
-    batchVerifyMultiBatchedParallel(numSigs = 6, iters = 10, nthreads)
-    separator()
-
-    # Simulate 10 blocks verification
-    batchVerifyMulti(numSigs = 60, iters = 10)
-    batchVerifyMultiBatchedSerial(numSigs = 60, iters = 10)
-    batchVerifyMultiBatchedParallel(numSigs = 60, iters = 10, nthreads)
-    separator()
-
-    # Simulate 30 blocks verification
-    batchVerifyMulti(numSigs = 180, iters = 10)
-    batchVerifyMultiBatchedSerial(numSigs = 180, iters = 10)
-    batchVerifyMultiBatchedParallel(numSigs = 180, iters = 10, nthreads)
-    separator()
-
-    echo "\nUsing nthreads = ", nthreads, ". The number of threads can be changed with TP_NUM_THREADS environment variable."
+  var nthreads: int
+  if existsEnv"TP_NUM_THREADS":
+    nthreads = getEnv"TP_NUM_THREADS".parseInt()
+  else:
+    nthreads = countProcessors()
+
+  # Simulate Block verification (at most 6 signatures per block)
+  batchVerifyMulti(numSigs = 6, iters = 10)
+  batchVerifyMultiSameMessage(numSigs = 6, iters = 10)
+  batchVerifyMultiBatchedSerial(numSigs = 6, iters = 10)
+  batchVerifyMultiBatchedParallel(numSigs = 6, iters = 10, nthreads)
+  separator()
+
+  # Simulate 10 blocks verification
+  batchVerifyMulti(numSigs = 60, iters = 10)
+  batchVerifyMultiSameMessage(numSigs = 60, iters = 10)
+  batchVerifyMultiBatchedSerial(numSigs = 60, iters = 10)
+  batchVerifyMultiBatchedParallel(numSigs = 60, iters = 10, nthreads)
+  separator()
+
+  # Simulate 30 blocks verification
+  batchVerifyMulti(numSigs = 180, iters = 10)
+  batchVerifyMultiSameMessage(numSigs = 180, iters = 10)
+  batchVerifyMultiBatchedSerial(numSigs = 180, iters = 10)
+  batchVerifyMultiBatchedParallel(numSigs = 180, iters = 10, nthreads)
+  separator()
+
+  echo "\nUsing nthreads = ", nthreads, ". The number of threads can be changed with TP_NUM_THREADS environment variable."

benchmarks/bls_signature.nim

@@ -1,5 +1,5 @@
 # Nim-BLSCurve
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT))
@@ -209,11 +209,52 @@ when BLS_BACKEND == BLST:
       hashedMsg.bls_sha256_digest("msg" & $i)
       triplets.add (pk, hashedMsg, sk.sign(hashedMsg))
 
-    bench("BLS verif of " & $numSigs & " msgs by "& $numSigs & " pubkeys", iters):
+    bench("BLS verif of " & $numSigs & " msgs by " & $numSigs & " pubkeys", iters):
       for i in 0 ..< triplets.len:
         let ok = triplets[i].pubkey.verify(triplets[i].msg, triplets[i].sig)
         doAssert ok
 
+  proc batchVerifyMultiSameMessage*(numSigs, iters: int) =
+    ## Verification of N pubkeys signing the same message
+
+    var hashedMsg: array[32, byte]
+    hashedMsg.bls_sha256_digest("msg")
+
+    var
+      pks: seq[PublicKey]
+      sigs: seq[Signature]
+      multiSet {.noinit.}: MultiSignatureSet
+    for i in 0 ..< numSigs:
+      let
+        (pk, sk) = keyGen()
+        sig = sk.sign(hashedMsg)
+      pks.add pk
+      sigs.add sig
+      if i == 0:
+        multiSet = MultiSignatureSet.init((pk, hashedMsg, sig))
+      else:
+        multiSet.add((pk, hashedMsg, sig))
+
+    # With blinding (more secure, but slower)
+    var secureBlindingBytes: array[32, byte]
+    secureBlindingBytes.bls_sha256_digest("Mr F was here")
+    bench("BLS verif of " & $numSigs & " sigs of same msg by " & $numSigs & " pubkeys (with blinding)", iters):
+      let
+        triplet = multiSet.combine(secureBlindingBytes)
+        ok = triplet.pubkey.verify(triplet.message, triplet.signature)
+      doAssert ok
+
+    # Without blinding (not secure, but benched for comparison)
+    bench("BLS verif of " & $numSigs & " sigs of same msg by " & $numSigs & " pubkeys", iters):
+      var
+        pubkey {.noinit.}: PublicKey
+        signature {.noinit.}: Signature
+      let ok =
+        pubkey.aggregateAll(pks) and
+        signature.aggregateAll(sigs) and
+        pubkey.verify(hashedMsg, signature)
+      doAssert ok
+
   proc batchVerifyMultiBatchedSerial*(numSigs, iters: int) =
     ## Verification of N pubkeys signing for N messages
 
@@ -230,7 +271,7 @@ when BLS_BACKEND == BLST:
 
     var cache = BatchedBLSVerifierCache.init()
 
-    bench("Serial batch verify " & $numSigs & " msgs by "& $numSigs & " pubkeys (with blinding)", iters):
+    bench("Serial batch verify " & $numSigs & " msgs by " & $numSigs & " pubkeys (with blinding)", iters):
       secureBlindingBytes.bls_sha256_digest(secureBlindingBytes)
       let ok = cache.batchVerifySerial(batch, secureBlindingBytes)
       doAssert ok
@@ -277,16 +318,19 @@ when isMainModule:
 
     # Simulate Block verification (at most 6 signatures per block)
     batchVerifyMulti(numSigs = 6, iters = 10)
+    batchVerifyMultiSameMessage(numSigs = 6, iters = 10)
     batchVerifyMultiBatchedSerial(numSigs = 6, iters = 10)
     batchVerifyMultiBatchedParallel(numSigs = 6, iters = 10, nthreads)
 
     # Simulate 10 blocks verification
     batchVerifyMulti(numSigs = 60, iters = 10)
+    batchVerifyMultiSameMessage(numSigs = 60, iters = 10)
     batchVerifyMultiBatchedSerial(numSigs = 60, iters = 10)
     batchVerifyMultiBatchedParallel(numSigs = 60, iters = 10, nthreads)
 
     # Simulate 30 blocks verification
     batchVerifyMulti(numSigs = 180, iters = 10)
+    batchVerifyMultiSameMessage(numSigs = 180, iters = 10)
     batchVerifyMultiBatchedSerial(numSigs = 180, iters = 10)
     batchVerifyMultiBatchedParallel(numSigs = 180, iters = 10, nthreads)
 

blscurve/bls_batch_verifier.nim

@@ -1,5 +1,5 @@
 # Nim-BLSCurve
-# Copyright (c) 2018-Present Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT))
@@ -39,12 +39,26 @@ type
     ## if `signature` is the corresponding AggregateSignature
     ## on the same `message`
     ##
-    ## This assumes that `message`
-    ## is the output of a fixed size hash function.
+    ## This assumes that `message` is the output of a fixed size hash function.
     ##
     ## `pubkey` and `signature` are assumed to be grouped checked
     ## which is guaranteed at deserialization from bytes or hex
 
+  MultiSignatureSet* = object
+    ## A set of signatures that all pertain to the same `message`.
+    ##
+    ## `pubkeys` can contain aggregate publickeys (via `aggregateAll`)
+    ## if `signatures` contains the corresponding AggregateSignature
+    ## on the same `message`
+    ##
+    ## This assumes that `message` is the output of a fixed size hash function.
+    ##
+    ## `pubkeys` and `signatures` are assumed to be grouped checked
+    ## which is guaranteed at deserialization from bytes or hex
+    pubkeys: seq[PublicKey]
+    message: array[32, byte]
+    signatures: seq[Signature]
+
   BatchedBLSVerifierCache* {.requiresInit.} = object
     ## This types hold temporary contexts
     ## to batch BLS multi signatures (aggregated or individual)
@@ -57,6 +71,40 @@ type
 # Serial Batch Verifier
 # ----------------------------------------------------------------------
 
+func init*(
+       T: type MultiSignatureSet,
+       pubkeys: seq[PublicKey],
+       message: array[32, byte],
+       signatures: seq[Signature]
+     ): MultiSignatureSet =
+  doAssert pubkeys.len == signatures.len
+  doAssert pubkeys.len > 0
+  MultiSignatureSet(
+    pubkeys: pubkeys,
+    message: message,
+    signatures: signatures,
+  )
+
+func init*(T: type MultiSignatureSet, sigset: SignatureSet): MultiSignatureSet =
+  MultiSignatureSet(
+    pubkeys: @[sigset.pubkey],
+    message: sigset.message,
+    signatures: @[sigset.signature],
+  )
+
+func add*(multiSet: var MultiSignatureSet, sigset: SignatureSet) =
+  doAssert multiSet.message == sigset.message
+  multiSet.pubkeys.add sigset.pubkey
+  multiSet.signatures.add sigset.signature
+
+func combine*(
+          multiSet: MultiSignatureSet,
+          secureRandomBytes: array[32, byte]
+        ): SignatureSet =
+  let (pubkey, signature) = secureRandomBytes
+    .combine(multiSet.pubkeys, multiSet.signatures)
+  (pubkey, multiSet.message, signature)
+
 func init*(T: type BatchedBLSVerifierCache): T =
   ## Initialise the cache for single-threaded usage
   BatchedBLSVerifierCache(

blscurve/blst/blst+nim.h

@@ -0,0 +1,101 @@
+/**
+ * Copyright (c) 2024 Status Research & Development GmbH
+ * Licensed under either of
+ *  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+ *  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+ * at your option.
+ * This file may not be copied, modified, or distributed except according to
+ * those terms.
+ */
+
+#ifndef BLST_NIM_H
+#define BLST_NIM_H
+
+// Nim does not support annotating pointer destinations with C `const`.
+//
+// This leads to errors on certain platforms and toolchains
+// when interacting with APIs involving nested pointers, e.g.:
+//     expected 'const blst_p1_affine * const*'
+//     but argument is of type 'blst_p1_affine **'
+//     [-Wincompatible-pointer-types]
+//
+// To prevent these issues, offending function signatures are replaced
+// with ones that lack C `const` annotations.
+
+#define blst_p1s_to_affine blst_p1s_to_affine_replaced
+#define blst_p1s_add blst_p1s_add_replaced
+#define blst_p1s_mult_wbits_precompute blst_p1s_mult_wbits_precompute_replaced
+#define blst_p1s_mult_wbits blst_p1s_mult_wbits_replaced
+#define blst_p1s_mult_pippenger blst_p1s_mult_pippenger_replaced
+#define blst_p1s_tile_pippenger blst_p1s_tile_pippenger_replaced
+
+#define blst_p2s_to_affine blst_p2s_to_affine_replaced
+#define blst_p2s_add blst_p2s_add_replaced
+#define blst_p2s_mult_wbits_precompute blst_p2s_mult_wbits_precompute_replaced
+#define blst_p2s_mult_wbits blst_p2s_mult_wbits_replaced
+#define blst_p2s_mult_pippenger blst_p2s_mult_pippenger_replaced
+#define blst_p2s_tile_pippenger blst_p2s_tile_pippenger_replaced
+
+#define blst_miller_loop_n blst_miller_loop_n_replaced
+
+#include "../../vendor/blst/bindings/blst.h"
+
+#undef blst_p1s_to_affine
+#undef blst_p1s_add
+#undef blst_p1s_mult_wbits_precompute
+#undef blst_p1s_mult_wbits
+#undef blst_p1s_mult_pippenger
+#undef blst_p1s_tile_pippenger
+
+#undef blst_p2s_to_affine
+#undef blst_p2s_add
+#undef blst_p2s_mult_wbits_precompute
+#undef blst_p2s_mult_wbits
+#undef blst_p2s_mult_pippenger
+#undef blst_p2s_tile_pippenger
+
+#undef blst_miller_loop_n
+
+void blst_p1s_to_affine(blst_p1_affine dst[], blst_p1 *points[],
+                        size_t npoints);
+void blst_p1s_add(blst_p1 *ret, blst_p1_affine *points[],
+                                size_t npoints);
+void blst_p1s_mult_wbits_precompute(blst_p1_affine table[], size_t wbits,
+                                    blst_p1_affine *points[],
+                                    size_t npoints);
+void blst_p1s_mult_wbits(blst_p1 *ret, const blst_p1_affine table[],
+                         size_t wbits, size_t npoints,
+                         byte *scalars[], size_t nbits,
+                         limb_t *scratch);
+void blst_p1s_mult_pippenger(blst_p1 *ret, blst_p1_affine *points[],
+                             size_t npoints, byte *scalars[],
+                             size_t nbits, limb_t *scratch);
+void blst_p1s_tile_pippenger(blst_p1 *ret, blst_p1_affine *points[],
+                             size_t npoints, byte *scalars[],
+                             size_t nbits, limb_t *scratch,
+                             size_t bit0, size_t window);
+
+void blst_p2s_to_affine(blst_p2_affine dst[], blst_p2 *points[],
+                        size_t npoints);
+void blst_p2s_add(blst_p2 *ret, blst_p2_affine *points[],
+                                size_t npoints);
+void blst_p2s_mult_wbits_precompute(blst_p2_affine table[], size_t wbits,
+                                    blst_p2_affine *points[],
+                                    size_t npoints);
+void blst_p2s_mult_wbits(blst_p2 *ret, const blst_p2_affine table[],
+                         size_t wbits, size_t npoints,
+                         byte *scalars[], size_t nbits,
+                         limb_t *scratch);
+void blst_p2s_mult_pippenger(blst_p2 *ret, blst_p2_affine *points[],
+                             size_t npoints, byte *scalars[],
+                             size_t nbits, limb_t *scratch);
+void blst_p2s_tile_pippenger(blst_p2 *ret, blst_p2_affine *points[],
+                             size_t npoints, byte *scalars[],
+                             size_t nbits, limb_t *scratch,
+                             size_t bit0, size_t window);
+
+void blst_miller_loop_n(blst_fp12 *ret, blst_p2_affine *Qs[],
+                                        blst_p1_affine *Ps[],
+                                        size_t n);
+
+#endif