From 3fc800a645f891a5b8fb254bd8c949d156ee8462 Mon Sep 17 00:00:00 2001 From: Damjan Dimitrov Date: Tue, 6 Feb 2024 15:50:36 +0100 Subject: [PATCH 1/2] Modify file upload implementation, add CLI options --- packages/cli/package.json | 2 +- .../src/modules/storage/storage.commands.ts | 3 ++ .../src/modules/storage/storage.service.ts | 15 +++++- packages/sdk/package.json | 2 +- .../sdk/src/modules/storage/storage-bucket.ts | 10 ++-- packages/sdk/src/tests/hosting.test.ts | 27 +++++----- packages/sdk/src/types/storage.ts | 2 + packages/sdk/src/util/file-utils.ts | 51 +++++++------------ 8 files changed, 54 insertions(+), 58 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 504ce45..c74682a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@apillon/cli", "description": "▶◀ Apillon CLI tools ▶◀", - "version": "1.2.0", + "version": "1.2.1", "author": "Apillon", "license": "MIT", "main": "./dist/index.js", diff --git a/packages/cli/src/modules/storage/storage.commands.ts b/packages/cli/src/modules/storage/storage.commands.ts index 23c79c6..fa378d8 100644 --- a/packages/cli/src/modules/storage/storage.commands.ts +++ b/packages/cli/src/modules/storage/storage.commands.ts @@ -65,6 +65,9 @@ export function createStorageCommands(cli: Command) { .description('Upload contents of a local folder to bucket') .argument('', 'path to local folder') .requiredOption('-b, --bucket-uuid ', 'UUID of destination bucket') + .option('-w, --wrap', 'Wrap uploaded files to an IPFS directory') + .option('-p, --path ', 'Path to upload files to') + .option('--await', 'await file CIDs to be resolved') .action(async function (path: string) { await uploadFromFolder(path, this.optsWithGlobals()); }); diff --git a/packages/cli/src/modules/storage/storage.service.ts b/packages/cli/src/modules/storage/storage.service.ts index 9c0f78b..30366e8 100644 --- a/packages/cli/src/modules/storage/storage.service.ts +++ b/packages/cli/src/modules/storage/storage.service.ts @@ -44,11 +44,22 @@ export async function uploadFromFolder( path: string, optsWithGlobals: GlobalOptions, ) { + if (!!optsWithGlobals.wrap && !optsWithGlobals.path) { + console.error( + 'Error: path option (-p, --path) is required when --wrap option is supplied', + ); + return; + } await withErrorHandler(async () => { console.log(`Uploading files from folder: ${path}`); - await new Storage(optsWithGlobals) + const files = await new Storage(optsWithGlobals) .bucket(optsWithGlobals.bucketUuid) - .uploadFromFolder(path); + .uploadFromFolder(path, { + wrapWithDirectory: !!optsWithGlobals.wrap, + directoryPath: optsWithGlobals.path, + awaitCid: !!optsWithGlobals.await, + }); + console.log(files); }); } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 73a6940..e3f2bc2 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,7 +1,7 @@ { "name": "@apillon/sdk", "description": "▶◀ Apillon SDK for NodeJS ▶◀", - "version": "2.0.0", + "version": "2.0.1", "author": "Apillon", "license": "MIT", "main": "./dist/index.js", diff --git a/packages/sdk/src/modules/storage/storage-bucket.ts b/packages/sdk/src/modules/storage/storage-bucket.ts index 7e8ad1a..3040f93 100644 --- a/packages/sdk/src/modules/storage/storage-bucket.ts +++ b/packages/sdk/src/modules/storage/storage-bucket.ts @@ -1,6 +1,7 @@ import { Directory } from './directory'; import { FileMetadata, + FileUploadResult, IBucketFilesRequest, ICreateIpns, IFileUploadRequest, @@ -121,7 +122,7 @@ export class StorageBucket extends ApillonModel { public async uploadFromFolder( folderPath: string, params?: IFileUploadRequest, - ): Promise { + ): Promise { const { files: uploadedFiles, sessionUuid } = await uploadFiles( folderPath, this.API_PREFIX, @@ -143,7 +144,7 @@ export class StorageBucket extends ApillonModel { public async uploadFiles( files: FileMetadata[], params?: IFileUploadRequest, - ): Promise { + ): Promise { const { files: uploadedFiles, sessionUuid } = await uploadFiles( null, this.API_PREFIX, @@ -179,8 +180,8 @@ export class StorageBucket extends ApillonModel { private async resolveFileCIDs( sessionUuid: string, limit: number, - ): Promise { - let resolvedFiles: FileMetadata[] = []; + ): Promise { + let resolvedFiles: FileUploadResult[] = []; // Resolve CIDs for each file let retryTimes = 0; @@ -192,7 +193,6 @@ export class StorageBucket extends ApillonModel { fileUuid: file.uuid, CID: file.CID, CIDv1: file.CIDv1, - content: null, }), ); diff --git a/packages/sdk/src/tests/hosting.test.ts b/packages/sdk/src/tests/hosting.test.ts index 4787af7..f03a53e 100644 --- a/packages/sdk/src/tests/hosting.test.ts +++ b/packages/sdk/src/tests/hosting.test.ts @@ -51,21 +51,18 @@ describe('Hosting tests', () => { ); try { console.time('File upload complete'); - await hosting.website(websiteUuid).uploadFiles( - [ - { - fileName: 'index.html', - contentType: 'text/html', - content: html, - }, - { - fileName: 'style.css', - contentType: 'text/css', - content: css, - }, - ], - { wrapWithDirectory: true, directoryPath: 'main/subdir' }, - ); + await hosting.website(websiteUuid).uploadFiles([ + { + fileName: 'index.html', + contentType: 'text/html', + content: html, + }, + { + fileName: 'style.css', + contentType: 'text/css', + content: css, + }, + ]); console.timeEnd('File upload complete'); // console.log(content); } catch (e) { diff --git a/packages/sdk/src/types/storage.ts b/packages/sdk/src/types/storage.ts index 2135834..e7efb0a 100644 --- a/packages/sdk/src/types/storage.ts +++ b/packages/sdk/src/types/storage.ts @@ -50,6 +50,8 @@ export interface FileMetadata { CID?: string; } +export type FileUploadResult = Omit; + export interface IFileUploadRequest { /** * Wrap uploaded files to IPFS directory. diff --git a/packages/sdk/src/util/file-utils.ts b/packages/sdk/src/util/file-utils.ts index 9f05eff..6389457 100644 --- a/packages/sdk/src/util/file-utils.ts +++ b/packages/sdk/src/util/file-utils.ts @@ -62,25 +62,12 @@ async function uploadFilesToS3( throw new Error(`Can't find file ${link.path}${link.fileName}!`); } uploadWorkers.push( - new Promise(async (resolve, reject) => { - // If uploading from local folder read file, otherwise directly upload content - if (file.index) { - fs.readFile(file.index, async (err, data) => { - if (err) { - reject(err.message); - } - // const respS3 = - await s3Api.put(link.url, data); - // console.log(respS3); - - ApillonLogger.log(`File uploaded: ${file.fileName}`); - resolve(); - }); - } else if (file.content) { - await s3Api.put(link.url, file.content); - ApillonLogger.log(`File uploaded: ${file.fileName}`); - resolve(); - } + new Promise(async (resolve, _reject) => { + // If uploading from local folder then read file, otherwise directly upload content + const content = file.index ? fs.readFileSync(file.index) : file.content; + await s3Api.put(link.url, content); + ApillonLogger.log(`File uploaded: ${file.fileName}`); + resolve(); }), ); } @@ -114,23 +101,19 @@ export async function uploadFiles( ApillonLogger.log(`Total files to upload: ${files.length}`); // Split files into chunks for parallel uploading - const fileChunkSize = 50; + const fileChunkSize = 200; const sessionUuid = uuidv4(); + const uploadedFiles = []; - const uploadedFiles = await Promise.all( - chunkify(files, fileChunkSize).map(async (fileGroup) => { - const data = await ApillonApi.post( - `${apiPrefix}/upload`, - { - files: fileGroup, - sessionUuid, - }, - ); - - await uploadFilesToS3(data.files, fileGroup); - return data.files; - }), - ); + for (const fileGroup of chunkify(files, fileChunkSize)) { + const { files } = await ApillonApi.post( + `${apiPrefix}/upload`, + { files: fileGroup, sessionUuid }, + ); + + await uploadFilesToS3(files, fileGroup); + uploadedFiles.push(files); + } ApillonLogger.logWithTime('File upload complete.'); From d29924417b9742a54fbde1f6aa74f4dbd53fa90a Mon Sep 17 00:00:00 2001 From: Damjan Dimitrov Date: Wed, 7 Feb 2024 16:14:45 +0100 Subject: [PATCH 2/2] Update CLI storage upload README --- packages/cli/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 9ca2409..7f02e4b 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -373,11 +373,14 @@ Upload contents of a local folder to specified bucket. - ``: Path to the folder containing your files. - `-b, --bucket-uuid `: UUID of the bucket to upload files to. +- `-w, --wrap`: Wrap uploaded files to an IPFS directory +- `-p, --path `: Path to upload files to (e.g. main/subdir). Required when --wrap is supplied. +- `--await`: await file CIDs to be resolved **Example** ```sh -apillon storage upload ./my_folder --bucket-uuid "123e4567-e89b-12d3-a456-426655440000" +apillon storage upload ./my_folder --bucket-uuid "123e4567-e89b-12d3-a456-426655440000" --wrap --path "main/subdir" ``` #### `storage get-file`