diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dd06ae482..4371eaf1a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -218,7 +218,7 @@ services: NEXT_PUBLIC_POSTHOG_KEY: ${POSTHOG_KEY} NEXT_PUBLIC_POSTHOG_HOST: ${POSTHOG_HOST} NEXT_PUBLIC_DISTRIBUTION: ${DISTRIBUTION} - NEXT_PUBLIC_BACKEND_DOMAIN: https://localhost:3000 + NEXT_PUBLIC_BACKEND_DOMAIN: http://localhost:3000 NEXT_PUBLIC_MAGIC_LINK_DOMAIN: ${NEXT_PUBLIC_MAGIC_LINK_DOMAIN} NEXT_PUBLIC_WEBAPP_DOMAIN: ${NEXT_PUBLIC_WEBAPP_DOMAIN} NEXT_PUBLIC_REDIRECT_WEBHOOK_INGRESS: ${REDIRECT_TUNNEL_INGRESS} @@ -238,22 +238,22 @@ services: # Developer tools # # # # # # # # # # # - pgadmin: - image: dpage/pgadmin4 - container_name: pgadmin4_container - restart: always - ports: - - "8888:80" - environment: - PGADMIN_DEFAULT_EMAIL: user-name@domain-name.com - PGADMIN_DEFAULT_PASSWORD: strong-password - networks: - - backend - depends_on: - postgres: - condition: service_healthy - volumes: - - pgadmin-data:/var/lib/pgadmin + # pgadmin: + # image: dpage/pgadmin4 + # container_name: pgadmin4_container + # restart: always + # ports: + # - "8888:80" + # environment: + # PGADMIN_DEFAULT_EMAIL: user-name@domain-name.com + # PGADMIN_DEFAULT_PASSWORD: strong-password + # networks: + # - backend + # depends_on: + # postgres: + # condition: service_healthy + # volumes: + # - pgadmin-data:/var/lib/pgadmin # ngrok: # image: ngrok/ngrok:latest diff --git a/packages/api/src/filestorage/file/services/sharepoint/index.ts b/packages/api/src/filestorage/file/services/sharepoint/index.ts index dfd3cb0fe..7e7643920 100644 --- a/packages/api/src/filestorage/file/services/sharepoint/index.ts +++ b/packages/api/src/filestorage/file/services/sharepoint/index.ts @@ -482,24 +482,45 @@ export class SharepointService implements IFileService { } catch (error: any) { attempts++; + // Handle rate limiting + if (error.response && error.response.status === 429) { + const retryAfter: number = this.getRetryAfter( + error.response.headers['retry-after'], + ); + const delayTime: number = Math.max(retryAfter * 1000, backoff); + + this.logger.warn( + `Rate limit hit. Retrying request in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`, + ); + + await this.delay(delayTime); + backoff *= 2; // Exponential backoff + continue; + } + + // Handle timeout errors if ( - (error.response && error.response.status === 429) || - (error.response && error.response.status >= 500) || error.code === 'ECONNABORTED' || error.code === 'ETIMEDOUT' || error.response?.code === 'ETIMEDOUT' ) { - const retryAfter = this.getRetryAfter( - error.response?.headers['retry-after'], + const delayTime: number = backoff; + + this.logger.warn( + `Request timeout. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`, ); - const delayTime: number = Math.max(retryAfter * 1000, backoff); + + await this.delay(delayTime); + backoff *= 2; + continue; + } + + // Handle server errors (500+) + if (error.response && error.response.status >= 500) { + const delayTime: number = backoff; this.logger.warn( - `Request failed with ${ - error.code || error.response?.status - }. Retrying in ${delayTime}ms (Attempt ${attempts}/${ - this.MAX_RETRIES - })`, + `Server error ${error.response.status}. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`, ); await this.delay(delayTime); @@ -507,7 +528,11 @@ export class SharepointService implements IFileService { continue; } - this.logger.error(`Request failed: ${error.message}`, error); + // handle 410 gone errors + if (error.response?.status === 410 && config.url.includes('delta')) { + // todo: handle 410 gone errors + } + throw error; } } diff --git a/packages/api/src/filestorage/file/services/sharepoint/mappers.ts b/packages/api/src/filestorage/file/services/sharepoint/mappers.ts index 899638c53..eb2023d0c 100644 --- a/packages/api/src/filestorage/file/services/sharepoint/mappers.ts +++ b/packages/api/src/filestorage/file/services/sharepoint/mappers.ts @@ -88,48 +88,25 @@ export class SharepointFileMapper implements IFileMapper { } } - const opts: any = {}; - if (file.permissions?.length) { - const permissions = await this.coreUnificationService.unify< - OriginalPermissionOutput[] - >({ - sourceObject: file.permissions, - targetType: FileStorageObject.permission, - providerName: 'sharepoint', - vertical: 'filestorage', - connectionId, - customFieldMappings: [], - }); - opts.permissions = permissions; - - // shared link - if (file.permissions.some((p) => p.link)) { - const sharedLinks = - await this.coreUnificationService.unify({ - sourceObject: file.permissions.find((p) => p.link), - targetType: FileStorageObject.sharedlink, - providerName: 'sharepoint', - vertical: 'filestorage', - connectionId, - customFieldMappings: [], - }); - opts.shared_links = sharedLinks; - } - } - - // todo: handle folder - return { remote_id: file.id, remote_data: file, + remote_folder_id: file.parentReference?.id, + remote_drive_id: file.driveId || file?.parentReference?.driveId || null, + remote_created_at: file.createdDateTime + ? new Date(file.createdDateTime) + : null, + remote_modified_at: file.lastModifiedDateTime + ? new Date(file.lastModifiedDateTime) + : null, + remote_was_deleted: file.deleted ? true : false, name: file.name, file_url: file.webUrl, mime_type: file.file.mimeType, size: file.size.toString(), folder_id: null, - // permission: opts.permissions?.[0] || null, - permissions: null, - shared_link: opts.shared_links?.[0] || null, + permissions: file.internal_permissions, + shared_link: null, field_mappings, }; } diff --git a/packages/api/src/filestorage/folder/services/sharepoint/index.ts b/packages/api/src/filestorage/folder/services/sharepoint/index.ts index 558bc5dbd..57d02f3ac 100644 --- a/packages/api/src/filestorage/folder/services/sharepoint/index.ts +++ b/packages/api/src/filestorage/folder/services/sharepoint/index.ts @@ -646,24 +646,45 @@ export class SharepointService implements IFolderService { } catch (error: any) { attempts++; + // Handle rate limiting + if (error.response && error.response.status === 429) { + const retryAfter: number = this.getRetryAfter( + error.response.headers['retry-after'], + ); + const delayTime: number = Math.max(retryAfter * 1000, backoff); + + this.logger.warn( + `Rate limit hit. Retrying request in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`, + ); + + await this.delay(delayTime); + backoff *= 2; // Exponential backoff + continue; + } + + // Handle timeout errors if ( - (error.response && error.response.status === 429) || - (error.response && error.response.status >= 500) || error.code === 'ECONNABORTED' || error.code === 'ETIMEDOUT' || error.response?.code === 'ETIMEDOUT' ) { - const retryAfter = this.getRetryAfter( - error.response?.headers['retry-after'], + const delayTime: number = backoff; + + this.logger.warn( + `Request timeout. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`, ); - const delayTime: number = Math.max(retryAfter * 1000, backoff); + + await this.delay(delayTime); + backoff *= 2; + continue; + } + + // Handle server errors (500+) + if (error.response && error.response.status >= 500) { + const delayTime: number = backoff; this.logger.warn( - `Request failed with ${ - error.code || error.response?.status - }. Retrying in ${delayTime}ms (Attempt ${attempts}/${ - this.MAX_RETRIES - })`, + `Server error ${error.response.status}. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`, ); await this.delay(delayTime); @@ -671,7 +692,11 @@ export class SharepointService implements IFolderService { continue; } - this.logger.error(`Request failed: ${error.message}`, error); + // handle 410 gone errors + if (error.response?.status === 410 && config.url.includes('delta')) { + // todo: handle 410 gone errors + } + throw error; } } diff --git a/packages/api/src/filestorage/group/services/sharepoint/index.ts b/packages/api/src/filestorage/group/services/sharepoint/index.ts index 9377c12a5..2f45529ef 100644 --- a/packages/api/src/filestorage/group/services/sharepoint/index.ts +++ b/packages/api/src/filestorage/group/services/sharepoint/index.ts @@ -111,10 +111,11 @@ export class SharepointService implements IGroupService { group: SharepointGroupOutput, connection: connections, ) { + const url = connection.account_url.replace(/\/sites\/.+$/, ''); const config: AxiosRequestConfig = { timeout: 10000, method: 'get', - url: `${connection.account_url}/groups/${group.id}/members`, + url: `${url}/groups/${group.id}/members`, headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${this.cryptoService.decrypt(