feat: sqs based architecture data stack

.env.tpl

@@ -6,3 +6,17 @@
 
 # uncomment to set SENTRY_DSN
 # SENTRY_DSN = ''
+
+# Aggregator DID
+DID=""
+
+# just for a local test
+PRIVATE_KEY=""
+
+# Dealer service
+DEALER_DID=""
+DEALER_URL=""
+
+# Ucan log
+UCAN_LOG_URL=""
+UCAN_LOG_BASIC_AUTH=""

README.md

@@ -33,6 +33,19 @@ Deployment is managed by [seed.run]. PR's are deployed automatically to `https:/
 
 The `main` branch is deployed to https://staging.filecoin.web3.storage and staging builds are promoted to prod manually via the UI at https://console.seed.run
 
+### Local dev
+
+You can use `sst` to create a custom dev deployment on aws, with a local dev console for debugging.
+
+To do that **you need**
+
+- An AWS account with the AWS CLI configured locally
+- Copy `.env.tpl` to `.env.local`
+
+Then run `npm dev` to deploy dev services to your aws account and start dev console
+
+See: https://docs.sst.dev for more info on how things get deployed.
+
 ### Environment Variables
 
 Ensure the following variables are set in the env when deploying
@@ -41,6 +54,64 @@ Ensure the following variables are set in the env when deploying
 
 The root domain to deploy the API to. e.g `filecoin.web3.storage`. The value should match a hosted zone configured in route53 that your aws account has access to.
 
+#### `DID`
+
+[DID](https://www.w3.org/TR/did-core/) of the ucanto server running for the Aggregator service. e.g. `did:key:abc..`. Optional: if omitted, a `did:key` will be derrived from `PRIVATE_KEY`
+
+#### `BROKER_DID`
+
+[DID](https://www.w3.org/TR/did-core/) of the ucanto server running a Broker service. e.g. `did:web:spade.storage`.
+
+#### `BROKER_URL`
+
+URL of the ucanto server running a Broker service. e.g. `https://spade-proxy.web3.storage`
+
+#### `UCAN_LOG_URL`
+
+URL of the UCAN log server. e.g. `https://up.web3.storage/ucan`
+
+### Secrets
+
+Set production secrets in aws SSM via [`sst secrets`](https://docs.sst.dev/config#sst-secrets). The region must be set to the one you deploy that stage to
+
+```sh
+# set `PRIVATE_KEY` for prod
+$ npx sst secrets set --region us-west-2 --stage prod PRIVATE_KEY "MgCblCY...="
+```
+
+To set a fallback value for `staging` or an ephmeral PR build use [`sst secrets set-fallback`](https://docs.sst.dev/config#fallback-values)
+
+```sh
+# set `PRIVATE_KEY` for any stage in us-east-2
+$ npx sst secrets set --fallback --region us-east-2 PRIVATE_KEY "MgCZG7...="
+```
+
+**note** The fallback value can only be inherited by stages deployed in the same AWS account and region.
+
+Confirm the secret value using [`sst secrets list`](https://docs.sst.dev/config#sst-secrets)
+
+```sh
+$ npx sst secrets list --region us-east-2
+PRIVATE_KEY MgCZG7...= (fallback)
+
+$ npx sst secrets list --region us-west-2 --stage prod
+PRIVATE_KEY M...=
+```
+
+#### `PRIVATE_KEY`
+
+The [`multibase`](https://github.com/multiformats/multibase) encoded ED25519 keypair used as the signing key for the upload-api.
+
+Generated by [@ucanto/principal `EdSigner`](https://github.com/web3-storage/ucanto) via [`ucan-key`](https://www.npmjs.com/package/ucan-key)
+
+_Example:_ `MgCZG7EvaA...1pX9as=`
+
+#### `UCAN_LOG_BASIC_AUTH`
+
+The HTTP Basic auth token for the UCAN Invocation entrypoint, where UCAN invocations can be stored and proxied to the UCAN Stream.
+
+_Example:_ `MgCZG7EvaA...1pX9as=`
+
 </p>
 
 [SST]: https://sst.dev

docs/architecture.md

@@ -140,6 +140,10 @@ interface Piece {
 interface PieceBuffer {
   // Pieces inside the buffer
   pieces: BufferedPiece[]
+  // identifier of storefront `did:web:web3.storage`
+  storefront: string
+  // identifier of group within storefront
+  group: string
 }
 
 interface BufferedPiece {
@@ -172,6 +176,10 @@ interface Aggregate {
   task: Link
   // number of milliseconds elapsed since the epoch when aggregate was submitted
   insertedAt: number
+  // identifier of storefront `did:web:web3.storage`
+  storefront: string
+  // identifier of group within storefront
+  group: string
   // known status of the aggregate (a secondary index)
   stat: AggregateStatus
 }