Skip to content

Commit

Permalink
fix: update backend domain in docker-compose and enhance error handli…
Browse files Browse the repository at this point in the history 
…ng in Sharepoint services
  • Loading branch information
@amuwal
amuwal committed Dec 30, 2024
1 parent fbdbc75 commit 21c8f85
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 74 deletions.
34 changes: 17 additions & 17 deletions docker-compose.dev.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ services:
NEXT_PUBLIC_POSTHOG_KEY: ${POSTHOG_KEY}
NEXT_PUBLIC_POSTHOG_HOST: ${POSTHOG_HOST}
NEXT_PUBLIC_DISTRIBUTION: ${DISTRIBUTION}
NEXT_PUBLIC_BACKEND_DOMAIN: https://localhost:3000
NEXT_PUBLIC_BACKEND_DOMAIN: http://localhost:3000
NEXT_PUBLIC_MAGIC_LINK_DOMAIN: ${NEXT_PUBLIC_MAGIC_LINK_DOMAIN}
NEXT_PUBLIC_WEBAPP_DOMAIN: ${NEXT_PUBLIC_WEBAPP_DOMAIN}
NEXT_PUBLIC_REDIRECT_WEBHOOK_INGRESS: ${REDIRECT_TUNNEL_INGRESS}
Expand All @@ -238,22 +238,22 @@ services:
# Developer tools #
# # # # # # # # # #

pgadmin:
image: dpage/pgadmin4
container_name: pgadmin4_container
restart: always
ports:
- "8888:80"
environment:
PGADMIN_DEFAULT_EMAIL: user-name@domain-name.com
PGADMIN_DEFAULT_PASSWORD: strong-password
networks:
- backend
depends_on:
postgres:
condition: service_healthy
volumes:
- pgadmin-data:/var/lib/pgadmin
# pgadmin:
# image: dpage/pgadmin4
# container_name: pgadmin4_container
# restart: always
# ports:
# - "8888:80"
# environment:
# PGADMIN_DEFAULT_EMAIL: user-name@domain-name.com
# PGADMIN_DEFAULT_PASSWORD: strong-password
# networks:
# - backend
# depends_on:
# postgres:
# condition: service_healthy
# volumes:
# - pgadmin-data:/var/lib/pgadmin

# ngrok:
# image: ngrok/ngrok:latest
Expand Down
47 changes: 36 additions & 11 deletions packages/api/src/filestorage/file/services/sharepoint/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,32 +482,57 @@ export class SharepointService implements IFileService {
} catch (error: any) {
attempts++;

// Handle rate limiting
if (error.response && error.response.status === 429) {
const retryAfter: number = this.getRetryAfter(
error.response.headers['retry-after'],
);
const delayTime: number = Math.max(retryAfter * 1000, backoff);

this.logger.warn(
`Rate limit hit. Retrying request in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
);

await this.delay(delayTime);
backoff *= 2; // Exponential backoff
continue;
}

// Handle timeout errors
if (
(error.response && error.response.status === 429) ||
(error.response && error.response.status >= 500) ||
error.code === 'ECONNABORTED' ||
error.code === 'ETIMEDOUT' ||
error.response?.code === 'ETIMEDOUT'
) {
const retryAfter = this.getRetryAfter(
error.response?.headers['retry-after'],
const delayTime: number = backoff;

this.logger.warn(
`Request timeout. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
);
const delayTime: number = Math.max(retryAfter * 1000, backoff);

await this.delay(delayTime);
backoff *= 2;
continue;
}

// Handle server errors (500+)
if (error.response && error.response.status >= 500) {
const delayTime: number = backoff;

this.logger.warn(
`Request failed with ${
error.code || error.response?.status
}. Retrying in ${delayTime}ms (Attempt ${attempts}/${
this.MAX_RETRIES
})`,
`Server error ${error.response.status}. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
);

await this.delay(delayTime);
backoff *= 2;
continue;
}

this.logger.error(`Request failed: ${error.message}`, error);
// handle 410 gone errors
if (error.response?.status === 410 && config.url.includes('delta')) {
// todo: handle 410 gone errors
}

throw error;
}
}
Expand Down
45 changes: 11 additions & 34 deletions packages/api/src/filestorage/file/services/sharepoint/mappers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,48 +88,25 @@ export class SharepointFileMapper implements IFileMapper {
}
}

const opts: any = {};
if (file.permissions?.length) {
const permissions = await this.coreUnificationService.unify<
OriginalPermissionOutput[]
>({
sourceObject: file.permissions,
targetType: FileStorageObject.permission,
providerName: 'sharepoint',
vertical: 'filestorage',
connectionId,
customFieldMappings: [],
});
opts.permissions = permissions;

// shared link
if (file.permissions.some((p) => p.link)) {
const sharedLinks =
await this.coreUnificationService.unify<OriginalSharedLinkOutput>({
sourceObject: file.permissions.find((p) => p.link),
targetType: FileStorageObject.sharedlink,
providerName: 'sharepoint',
vertical: 'filestorage',
connectionId,
customFieldMappings: [],
});
opts.shared_links = sharedLinks;
}
}

// todo: handle folder

return {
remote_id: file.id,
remote_data: file,
remote_folder_id: file.parentReference?.id,
remote_drive_id: file.driveId || file?.parentReference?.driveId || null,
remote_created_at: file.createdDateTime
? new Date(file.createdDateTime)
: null,
remote_modified_at: file.lastModifiedDateTime
? new Date(file.lastModifiedDateTime)
: null,
remote_was_deleted: file.deleted ? true : false,
name: file.name,
file_url: file.webUrl,
mime_type: file.file.mimeType,
size: file.size.toString(),
folder_id: null,
// permission: opts.permissions?.[0] || null,
permissions: null,
shared_link: opts.shared_links?.[0] || null,
permissions: file.internal_permissions,
shared_link: null,
field_mappings,
};
}
Expand Down
47 changes: 36 additions & 11 deletions packages/api/src/filestorage/folder/services/sharepoint/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -646,32 +646,57 @@ export class SharepointService implements IFolderService {
} catch (error: any) {
attempts++;

// Handle rate limiting
if (error.response && error.response.status === 429) {
const retryAfter: number = this.getRetryAfter(
error.response.headers['retry-after'],
);
const delayTime: number = Math.max(retryAfter * 1000, backoff);

this.logger.warn(
`Rate limit hit. Retrying request in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
);

await this.delay(delayTime);
backoff *= 2; // Exponential backoff
continue;
}

// Handle timeout errors
if (
(error.response && error.response.status === 429) ||
(error.response && error.response.status >= 500) ||
error.code === 'ECONNABORTED' ||
error.code === 'ETIMEDOUT' ||
error.response?.code === 'ETIMEDOUT'
) {
const retryAfter = this.getRetryAfter(
error.response?.headers['retry-after'],
const delayTime: number = backoff;

this.logger.warn(
`Request timeout. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
);
const delayTime: number = Math.max(retryAfter * 1000, backoff);

await this.delay(delayTime);
backoff *= 2;
continue;
}

// Handle server errors (500+)
if (error.response && error.response.status >= 500) {
const delayTime: number = backoff;

this.logger.warn(
`Request failed with ${
error.code || error.response?.status
}. Retrying in ${delayTime}ms (Attempt ${attempts}/${
this.MAX_RETRIES
})`,
`Server error ${error.response.status}. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
);

await this.delay(delayTime);
backoff *= 2;
continue;
}

this.logger.error(`Request failed: ${error.message}`, error);
// handle 410 gone errors
if (error.response?.status === 410 && config.url.includes('delta')) {
// todo: handle 410 gone errors
}

throw error;
}
}
Expand Down
3 changes: 2 additions & 1 deletion packages/api/src/filestorage/group/services/sharepoint/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,11 @@ export class SharepointService implements IGroupService {
group: SharepointGroupOutput,
connection: connections,
) {
const url = connection.account_url.replace(/\/sites\/.+$/, '');
const config: AxiosRequestConfig = {
timeout: 10000,
method: 'get',
url: `${connection.account_url}/groups/${group.id}/members`,
url: `${url}/groups/${group.id}/members`,
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.cryptoService.decrypt(
Expand Down

0 comments on commit 21c8f85

Please sign in to comment.