Merge branch 'master' into feature/ceremony-files

.github/workflows/ci-reusable.yml

@@ -32,6 +32,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: recursive
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Setup Nimbus Build System
         uses: ./.github/actions/nimbus-build-system

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ on:
 
 env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
-  nim_version: v1.6.14
+  nim_version: pinned
 
 
 concurrency:
@@ -48,6 +48,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: recursive
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Setup Nimbus Build System
         uses: ./.github/actions/nimbus-build-system

.github/workflows/nim-matrix.yml

@@ -6,7 +6,7 @@ on:
 
 env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
-  nim_version: v1.6.14, v1.6.16, v1.6.18
+  nim_version: pinned, v1.6.16, v1.6.18
 
 jobs:
   matrix:

Makefile

@@ -5,6 +5,30 @@
 # at your option. This file may not be copied, modified, or distributed except
 # according to those terms.
 
+# This is the Nim version used locally and in regular CI builds.
+# Can be a specific version tag, a branch name, or a commit hash.
+# Can be overridden by setting the NIM_COMMIT environment variable
+# before calling make.
+#
+# For readability in CI, if NIM_COMMIT is set to "pinned",
+# this will also default to the version pinned here.
+#
+# If NIM_COMMIT is set to "nimbusbuild", this will use the
+# version pinned by nimbus-build-system.
+PINNED_NIM_VERSION := v1.6.14
+
+ifeq ($(NIM_COMMIT),)
+NIM_COMMIT := $(PINNED_NIM_VERSION)
+else ifeq ($(NIM_COMMIT),pinned)
+NIM_COMMIT := $(PINNED_NIM_VERSION)
+endif
+
+ifeq ($(NIM_COMMIT),nimbusbuild)
+undefine NIM_COMMIT
+else
+export NIM_COMMIT
+endif
+
 SHELL := bash # the shell used internally by Make
 
 # used inside the included makefiles

codex.nimble

@@ -6,31 +6,4 @@ binDir = "build"
 srcDir = "."
 installFiles  = @["build.nims"]
 
-requires "nim >= 1.2.0"
-requires "asynctest >= 0.5.1 & < 0.6.0"
-requires "bearssl >= 0.1.4"
-requires "chronicles >= 0.7.2"
-requires "chronos >= 2.5.2"
-requires "confutils"
-requires "ethers >= 0.7.3 & < 0.8.0"
-requires "libbacktrace"
-requires "libp2p"
-requires "metrics"
-requires "nimcrypto >= 0.4.1"
-requires "nitro >= 0.5.1 & < 0.6.0"
-requires "presto"
-requires "protobuf_serialization >= 0.2.0 & < 0.3.0"
-requires "questionable >= 0.10.13 & < 0.11.0"
-requires "secp256k1"
-requires "serde >= 1.0.0 & < 2.0.0"
-requires "stew"
-requires "upraises >= 0.1.0 & < 0.2.0"
-requires "toml_serialization"
-requires "https://github.com/status-im/lrucache.nim#1.2.2"
-requires "leopard >= 0.1.0 & < 0.2.0"
-requires "blscurve"
-requires "libp2pdht"
-requires "eth"
-requires "https://github.com/codex-storage/nim-poseidon2.git >= 0.1.0 & < 0.2.0"
-
 include "build.nims"

codex/codex.nim

@@ -297,7 +297,7 @@ proc new*(
       taskpool = taskpool)
 
     restServer = RestServerRef.new(
-      codexNode.initRestApi(config, repoStore),
+      codexNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin),
       initTAddress(config.apiBindAddress , config.apiPort),
       bufferSize = (1024 * 64),
       maxRequestBodySize = int.high)

codex/conf.nim

@@ -198,6 +198,12 @@ type
       name: "api-port"
       abbr: "p" }: Port
 
+    apiCorsAllowedOrigin* {.
+      desc: "The REST Api CORS allowed origin for downloading data. '*' will allow all origins, '' will allow none.",
+      defaultValue: string.none
+      defaultValueDesc: "Disallow all cross origin requests to download data"
+      name: "api-cors-origin" }: Option[string]
+
     repoKind* {.
       desc: "Backend for main repo store (fs, sqlite, leveldb)"
       defaultValueDesc: "fs"

codex/erasure/erasure.nim

@@ -120,7 +120,7 @@ proc getPendingBlocks(
         CatchableError,
         "Future for block id not found, tree cid: " & $manifest.treeCid & ", index: " & $index)
 
-  Iter.new(genNext, isFinished)
+  AsyncIter[(?!bt.Block, int)].new(genNext, isFinished)
 
 proc prepareEncodingData(
   self: Erasure,
@@ -440,8 +440,7 @@ proc decode*(
   if treeCid != encoded.originalTreeCid:
     return failure("Original tree root differs from the tree root computed out of recovered data")
 
-  let idxIter = Iter
-    .fromItems(recoveredIndices)
+  let idxIter = Iter[Natural].new(recoveredIndices)
     .filter((i: Natural) => i < tree.leavesCount)
 
   if err =? (await self.store.putSomeProofs(tree, idxIter)).errorOption:

codex/indexingstrategy.nim

@@ -39,22 +39,9 @@ func checkIteration(self: IndexingStrategy, iteration: int): void {.raises: [Ind
       IndexingError,
       "Indexing iteration can't be greater than or equal to iterations.")
 
-proc getIter(first, last, step: int): Iter[int] =
-  var
-    finish = false
-    cur = first
-
-  func get(): int =
-    result = cur
-    cur += step
-
-    if cur > last:
-      finish = true
-
-  func isFinished(): bool =
-    finish
-
-  Iter.new(get, isFinished)
+func getIter(first, last, step: int): Iter[int] =
+  {.cast(noSideEffect).}:
+    Iter[int].new(first, last, step)
 
 func getLinearIndicies(
   self: IndexingStrategy,

codex/node.nim

@@ -157,10 +157,8 @@ proc updateExpiry*(
 
   try:
     let
-      ensuringFutures = Iter
-        .fromSlice(0..<manifest.blocksCount)
-        .mapIt(
-          self.networkStore.localStore.ensureExpiry( manifest.treeCid, it, expiry ))
+      ensuringFutures = Iter[int].new(0..<manifest.blocksCount)
+        .mapIt(self.networkStore.localStore.ensureExpiry( manifest.treeCid, it, expiry ))
     await allFuturesThrowing(ensuringFutures)
   except CancelledError as exc:
     raise exc
@@ -209,7 +207,7 @@ proc fetchBatched*(
 
   trace "Fetching blocks in batches of", size = batchSize
 
-  let iter = Iter.fromSlice(0..<manifest.blocksCount)
+  let iter = Iter[int].new(0..<manifest.blocksCount)
   self.fetchBatched(manifest.treeCid, iter, batchSize, onBatch)
 
 proc streamSingleBlock(