feat: configure max aggregate pieces (#109)

This PR allows configuration of the maximum number of pieces in an aggregate. See storacha/w3up#1566 for further context.
storacha · Oct 20, 2024 · f9746a2 · f9746a2
1 parent 56605b2
commit f9746a2
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 6 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/functions/package.json b/packages/functions/package.json
@@ -12,7 +12,7 @@
     "@ucanto/transport": "^9.1.1",
     "@w3filecoin/core": "*",
     "@web3-storage/data-segment": "^5.3.0",
-    "@web3-storage/filecoin-api": "^7.3.1",
+    "@web3-storage/filecoin-api": "^7.3.2",
     "@web3-storage/filecoin-client": "^3.3.4",
     "lru-cache": "^11.0.1",
     "multiformats": "^13.3.0",

diff --git a/packages/functions/src/aggregator/handle-aggregate-insert-to-piece-accept-queue.js b/packages/functions/src/aggregator/handle-aggregate-insert-to-piece-accept-queue.js
@@ -105,6 +105,9 @@ function getEnv () {
     pieceAcceptQueueUrl: mustGetEnv('PIECE_ACCEPT_QUEUE_URL'),
     pieceAcceptQueueRegion: mustGetEnv('AWS_REGION'),
     maxAggregateSize: Number.parseInt(mustGetEnv('MAX_AGGREGATE_SIZE')),
+    maxAggregatePieces: process.env.MAX_AGGREGATE_PIECES
+      ? Number.parseInt(process.env.MAX_AGGREGATE_PIECES)
+      : undefined,
     minAggregateSize: Number.parseInt(mustGetEnv('MIN_AGGREGATE_SIZE')),
     minUtilizationFactor: Number.parseInt(mustGetEnv('MIN_UTILIZATION_FACTOR')),
   }

diff --git a/packages/functions/src/aggregator/handle-buffer-queue-message.js b/packages/functions/src/aggregator/handle-buffer-queue-message.js
@@ -108,7 +108,8 @@ function getContext () {
     aggregateOfferQueueRegion,
     maxAggregateSize,
     minAggregateSize,
-    minUtilizationFactor
+    minUtilizationFactor,
+    maxAggregatePieces
   } = getEnv()
 
   let customHasherUsed = false
@@ -130,6 +131,7 @@ function getContext () {
     ),
     config: {
       maxAggregateSize,
+      maxAggregatePieces,
       minAggregateSize,
       minUtilizationFactor,
       prependBufferedPieces: [{
@@ -169,6 +171,9 @@ function getEnv () {
     aggregateOfferQueueUrl: mustGetEnv('AGGREGATE_OFFER_QUEUE_URL'),
     aggregateOfferQueueRegion: mustGetEnv('AWS_REGION'),
     maxAggregateSize: Number.parseInt(mustGetEnv('MAX_AGGREGATE_SIZE')),
+    maxAggregatePieces: process.env.MAX_AGGREGATE_PIECES
+      ? Number.parseInt(process.env.MAX_AGGREGATE_PIECES)
+      : undefined,
     minAggregateSize: Number.parseInt(mustGetEnv('MIN_AGGREGATE_SIZE')),
     minUtilizationFactor: Number.parseInt(mustGetEnv('MIN_UTILIZATION_FACTOR')),
   }

diff --git a/stacks/aggregator-stack.js b/stacks/aggregator-stack.js
@@ -19,6 +19,7 @@ export function AggregatorStack({ stack, app }) {
     AGGREGATOR_HOSTED_ZONE,
     AGGREGATOR_DID,
     MAX_AGGREGATE_SIZE,
+    MAX_AGGREGATE_PIECES,
     MIN_AGGREGATE_SIZE,
     MIN_UTILIZATION_FACTOR,
     AGGREGATOR_PROOF
@@ -184,6 +185,7 @@ export function AggregatorStack({ stack, app }) {
         BUFFER_STORE_BUCKET_NAME: aggregatorBufferStoreBucket.bucketName,
         AGGREGATE_OFFER_QUEUE_URL: aggregateOfferQueue.queueUrl,
         MAX_AGGREGATE_SIZE,
+        MAX_AGGREGATE_PIECES,
         MIN_AGGREGATE_SIZE,
         MIN_UTILIZATION_FACTOR,
       },
@@ -242,6 +244,7 @@ export function AggregatorStack({ stack, app }) {
           BUFFER_STORE_BUCKET_NAME: aggregatorBufferStoreBucket.bucketName,
           PIECE_ACCEPT_QUEUE_URL: pieceAcceptQueue.queueUrl,
           MAX_AGGREGATE_SIZE,
+          MAX_AGGREGATE_PIECES,
           MIN_AGGREGATE_SIZE,
           MIN_UTILIZATION_FACTOR,
         },

diff --git a/stacks/config.js b/stacks/config.js
@@ -133,6 +133,7 @@ export function getAggregatorEnv(stack) {
     AGGREGATOR_HOSTED_ZONE: process.env.AGGREGATOR_HOSTED_ZONE,
     AGGREGATOR_DID: mustGetEnv('AGGREGATOR_DID'),
     MAX_AGGREGATE_SIZE: process.env.MAX_AGGREGATE_SIZE || defaultMaxAggregateSize,
+    MAX_AGGREGATE_PIECES: process.env.MAX_AGGREGATE_PIECES ?? '',
     MIN_AGGREGATE_SIZE: process.env.MIN_AGGREGATE_SIZE || defaultMinAggregateSize,
     MIN_UTILIZATION_FACTOR: process.env.MIN_UTILIZATION_FACTOR || defaultMinUtilizationFactor,
     AGGREGATOR_PROOF: process.env.AGGREGATOR_PROOF ?? '',