diff --git a/src/components/SidebarMigrations.tsx b/src/components/SidebarMigrations.tsx
new file mode 100644
index 0000000..fd27a01
--- /dev/null
+++ b/src/components/SidebarMigrations.tsx
@@ -0,0 +1,37 @@
+import { useRouter } from 'next/navigation'
+import { useMigrations } from './MigrationsProvider'
+import { H2 } from './Text'
+import { PlusCircleIcon } from '@heroicons/react/20/solid'
+import { DidIcon } from './DidIcon'
+import { Migration } from '@/lib/migrations/api'
+
+export const SidebarMigrations = () => {
+ const [{ migrations }] = useMigrations()
+ const router = useRouter()
+ return (
+ <>
+
+
Migrations
+
+
+ )
+}
+
+const MigrationsList = ({ migrations }: { migrations: Migration[] }) => {
+ const router = useRouter()
+ if (!migrations.length) {
+ return
No running migrations
+ }
+ const selected = window.location.pathname.split('/').pop()
+ return migrations.map(m => {
+ return (
+
+ )
+ })
+}
diff --git a/src/lib/migrations/api.ts b/src/lib/migrations/api.ts
new file mode 100644
index 0000000..c636572
--- /dev/null
+++ b/src/lib/migrations/api.ts
@@ -0,0 +1,53 @@
+import { CARLink, DIDKey, Link, UnknownLink } from '@w3ui/react'
+
+export type MigrationID = string
+
+export type MigrationSource = 'classic.nft.storage' | 'old.web3.storage'
+
+export interface MigrationSourceConfiguration {
+ /** API token for data source */
+ token: string
+ /** Cursor for resuming migration. */
+ cursor?: string
+}
+
+export interface MigrationConfiguration {
+ /** Data source */
+ source: MigrationSource
+ /** API token for data source */
+ token: string
+ /** Target space to migrate data to */
+ space: DIDKey
+}
+
+export interface MigrationProgress {
+ /** The current item being migrated. */
+ current?: UnknownLink
+ /** Cursor for resuming migration. */
+ cursor?: string
+ /** The number of items that are still to be migrated. */
+ pending: number
+ /** The number of items that have been migrated successfully. */
+ succeeded: number
+ /** The items that failed to migrate. */
+ failed: UnknownLink[]
+}
+
+export interface Migration extends MigrationConfiguration {
+ id: MigrationID
+ /** The progress of the migration. */
+ progress?: MigrationProgress
+}
+
+export interface Shard {
+ link: CARLink
+ size: () => Promise
+ bytes: () => Promise
+}
+
+export interface Upload {
+ root: UnknownLink
+ shards: Shard[]
+}
+
+export interface UploadsSource extends AsyncIterable {}
diff --git a/src/lib/migrations/index.ts b/src/lib/migrations/index.ts
new file mode 100644
index 0000000..100e50d
--- /dev/null
+++ b/src/lib/migrations/index.ts
@@ -0,0 +1,130 @@
+import { ConnectionView, DIDKey, Proof, Service, Signer } from '@w3ui/react'
+import retry, { AbortError } from 'p-retry'
+import * as StoreCapabilities from '@web3-storage/capabilities/store'
+import * as UploadCapabilities from '@web3-storage/capabilities/upload'
+import { UploadsSource, Shard, Upload, MigrationSource, MigrationSourceConfiguration } from './api'
+import { NFTStorageMigrator } from './nft-storage'
+
+const REQUEST_RETRIES = 3
+
+export const create = (source: MigrationSource, config: MigrationSourceConfiguration) => {
+ switch (source) {
+ case 'classic.nft.storage':
+ return new NFTStorageMigrator(config)
+ default:
+ throw new Error(`not implemented`)
+ }
+}
+
+export const migrate = async ({
+ signal,
+ uploads,
+ issuer,
+ space,
+ proofs,
+ connection,
+ onStoreAdd,
+ onUploadAdd,
+ onError
+}: {
+ signal: AbortSignal
+ uploads: UploadsSource
+ issuer: Signer
+ space: DIDKey
+ proofs: Proof[]
+ connection: ConnectionView
+ onStoreAdd: (upload: Upload, shard: Shard) => unknown
+ onUploadAdd: (upload: Upload) => unknown
+ onError: (err: Error, upload: Upload, shard?: Shard) => unknown
+}) => {
+ for await (const upload of uploads) {
+ let allShardsStored = true
+
+ for (const shard of upload.shards) {
+ if (signal.aborted) return
+
+ try {
+ const result = await retry(async () => {
+ const receipt = await StoreCapabilities.add
+ .invoke({
+ issuer,
+ audience: connection.id,
+ with: space,
+ nb: { link: shard.link, size: await shard.size() },
+ proofs
+ })
+ .execute(connection)
+
+ if (!receipt.out.ok) {
+ throw receipt.out.error
+ }
+ return receipt.out.ok
+ }, { onFailedAttempt: console.warn, retries: REQUEST_RETRIES })
+
+ if (signal.aborted) return
+ onStoreAdd(upload, shard)
+
+ if (result.status === 'done') {
+ continue
+ }
+
+ const res = await retry(async () => {
+ try {
+ const res = await fetch(result.url, {
+ method: 'PUT',
+ body: await shard.bytes(),
+ headers: result.headers,
+ signal,
+ // @ts-expect-error
+ duplex: 'half',
+ })
+ if (res.status >= 400 && res.status < 500) {
+ throw new AbortError(`upload failed: ${res.status}`)
+ }
+ return res
+ } catch (err) {
+ if (signal?.aborted === true) {
+ throw new AbortError('upload aborted')
+ }
+ throw err
+ }
+ }, { onFailedAttempt: console.warn, retries: REQUEST_RETRIES })
+
+ if (!res.ok) {
+ throw new Error(`upload failed: ${res.status}`)
+ }
+ } catch (err: any) {
+ if (signal.aborted) return
+ onError(err, upload, shard)
+ allShardsStored = false
+ break
+ }
+ }
+
+ // do no register an upload if not all the shards uploaded successfully
+ if (!allShardsStored) continue
+
+ try {
+ const receipt = await UploadCapabilities.add.invoke({
+ issuer,
+ audience: connection.id,
+ proofs,
+ with: space,
+ nb: {
+ root: upload.root,
+ shards: upload.shards.map(s => s.link),
+ },
+ }).execute(connection)
+
+ if (receipt.out.error) {
+ throw receipt.out.error
+ }
+
+ if (signal.aborted) return
+ onUploadAdd(upload)
+ } catch (err: any) {
+ if (signal.aborted) return
+ onError(err, upload)
+ }
+ }
+}
diff --git a/src/lib/migrations/nft-storage.ts b/src/lib/migrations/nft-storage.ts
new file mode 100644
index 0000000..3bef7bf
--- /dev/null
+++ b/src/lib/migrations/nft-storage.ts
@@ -0,0 +1,115 @@
+import { NFTStorage } from 'nft.storage'
+import * as Link from 'multiformats/link'
+import { MigrationSourceConfiguration, Shard, Upload } from './api'
+
+export const checkToken = async (token: string) => {
+ const client = new NFTStorage({ token })
+ try {
+ await client.status(crypto.randomUUID())
+ } catch (err: any) {
+ if (!err.message.startsWith('Invalid CID')) {
+ throw err
+ }
+ }
+}
+
+interface Service {
+ endpoint: URL
+ token: string
+}
+
+interface ListOptions {
+ before?: string
+}
+
+export class NFTStorageMigrator {
+ #token
+ #cursor
+ #client
+ #started
+
+ constructor ({ token, cursor }: MigrationSourceConfiguration) {
+ this.#token = token
+ this.#cursor = cursor
+ this.#client = new NFTStorage({ token })
+ this.#started = false
+ }
+
+ async count () {
+ const headers = NFTStorage.auth({ token: this.#token })
+ const res = await fetch(this.#client.endpoint, { headers })
+ const count = res.headers.get('count')
+ if (!count) throw new Error('missing count in headers')
+ return parseInt(count)
+ }
+
+ [Symbol.asyncIterator] () {
+ return this.list()
+ }
+
+ async* list () {
+ const listPage = async ({ endpoint, token }: Service, { before }: ListOptions) => {
+ const params = new URLSearchParams()
+ if (before) params.append('before', before)
+ params.append('limit', String(1000))
+ const url = new URL(`?${params}`, endpoint)
+ return fetch(url, { headers: NFTStorage.auth({ token }) })
+ }
+
+ for await (const res of paginator(listPage, this.#client, {})) {
+ for (const raw of res.value) {
+ if (this.#cursor && !this.#started) {
+ if (raw.cid === this.#cursor) {
+ this.#started = true
+ }
+ continue
+ }
+
+ const shards: Shard[] = []
+ for (const p of raw.parts) {
+ shards.push({
+ link: Link.parse(p),
+ size: async () => {
+ const res = await fetch(`https://w3s.link/ipfs/${p}`, { method: 'HEAD' })
+ if (!res.ok) throw new Error(`failed to get size: ${p}`, { cause: { status: res.status } })
+ const contentLength = res.headers.get('Content-Length')
+ if (!contentLength) throw new Error('missing content length')
+ return parseInt(contentLength)
+ },
+ bytes: async () => {
+ // TODO: fetch from /complete?
+ throw new Error('not implemented')
+ }
+ })
+ }
+
+ yield { root: Link.parse(raw.cid), shards } as Upload
+ }
+ }
+ }
+}
+
+/**
+ * Follow before with last item, to fetch all the things.
+ */
+async function * paginator (fn: (service: Service, opts: ListOptions) => Promise, service: Service, opts: ListOptions) {
+ let res = await fn(service, opts)
+ if (!res.ok) {
+ if (res.status === 429) {
+ throw new Error('rate limited')
+ }
+ const errorMessage = await res.json()
+ throw new Error(`${res.status} ${res.statusText} ${errorMessage ? '- ' + errorMessage.message : ''}`)
+ }
+ let body = await res.json()
+ yield body
+
+ // Iterate through next pages
+ while (body && body.value.length) {
+ // Get before timestamp with less 1ms
+ const before = (new Date((new Date(body.value[body.value.length-1].created)).getTime() - 1)).toISOString()
+ res = await fn(service, { ...opts, before })
+ body = await res.json()
+ yield body
+ }
+}
diff --git a/src/lib/migrations/store.ts b/src/lib/migrations/store.ts
new file mode 100644
index 0000000..f5f86cd
--- /dev/null
+++ b/src/lib/migrations/store.ts
@@ -0,0 +1,45 @@
+import * as dagJSON from '@ipld/dag-json'
+import { Migration, MigrationConfiguration, MigrationID } from './api'
+
+export class MigrationsStorage {
+ load () {
+ if (typeof localStorage === 'undefined') { // in dev there is SSR
+ return []
+ }
+ const ids: string[] = dagJSON.parse(localStorage.getItem('migrations') ?? '[]')
+ const migrations: Migration[] = []
+ for (const id of ids) {
+ try {
+ const migration: Migration = dagJSON.parse(localStorage.getItem(`migration.${id}`) ?? '')
+ migrations.push(migration)
+ } catch (err) {
+ console.error(`failed to load migration: ${id}`, err)
+ }
+ }
+ return migrations
+ }
+
+ create (config: MigrationConfiguration) {
+ const migration: Migration = { id: crypto.randomUUID(), ...config }
+ const ids: string[] = dagJSON.parse(localStorage.getItem('migrations') ?? '[]')
+ localStorage.setItem(`migration.${migration.id}`, dagJSON.stringify(migration))
+ localStorage.setItem('migrations', dagJSON.stringify([...ids, migration.id]))
+ return { id: migration.id }
+ }
+
+ read (id: MigrationID) {
+ const raw = localStorage.getItem(`migration.${id}`) ?? ''
+ if (!raw) throw new Error('migration not found')
+ return dagJSON.parse(raw)
+ }
+
+ update (migration: Migration) {
+ localStorage.setItem(`migration.${migration.id}`, dagJSON.stringify(migration))
+ }
+
+ delete (id: MigrationID) {
+ const ids: string[] = dagJSON.parse(localStorage.getItem('migrations') ?? '[]')
+ localStorage.setItem('migrations', dagJSON.stringify(ids.filter(i => i !== id)))
+ localStorage.removeItem(`migration.${id}`)
+ }
+}
diff --git a/src/lib/migrations/web3-storage.ts b/src/lib/migrations/web3-storage.ts
new file mode 100644
index 0000000..8b80731
--- /dev/null
+++ b/src/lib/migrations/web3-storage.ts
@@ -0,0 +1,6 @@
+// @ts-expect-error not sure why types not working for old client!?
+import { Web3Storage } from 'web3.storage'
+
+export const checkToken = async (token: string) => {
+ await new Web3Storage({ token }).list().next()
+}
+