From f82a8f2e311551b3d3e18c8082b8418bdb95b68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C5=A1per=20Grom?= Date: Thu, 12 Sep 2024 15:19:44 +0200 Subject: [PATCH 1/3] Github connect invitations (#2603) Co-authored-by: garrrikkotua --- .../helpers/getGithubInstallations.ts | 9 + .../helpers/githubConnectInstallation.ts | 9 + backend/src/api/integration/index.ts | 10 + .../U1725439490__installed-github-orgs.sql | 0 .../V1725439490__installed-github-orgs.sql | 10 + .../githubInstallationsRepository.ts | 20 ++ backend/src/services/integrationService.ts | 34 +++ .../components/github-connect-modal.vue | 221 ++++++++++++++++++ .../github/components/github-connect.vue | 42 +--- .../components/integration-list.vue | 6 + .../integration/integration-service.js | 23 ++ .../apps/webhook_api/src/routes/github.ts | 20 +- .../src/old/apps/webhook_api/webhooks.repo.ts | 39 ++++ 13 files changed, 410 insertions(+), 33 deletions(-) create mode 100644 backend/src/api/integration/helpers/getGithubInstallations.ts create mode 100644 backend/src/api/integration/helpers/githubConnectInstallation.ts create mode 100644 backend/src/database/migrations/U1725439490__installed-github-orgs.sql create mode 100644 backend/src/database/migrations/V1725439490__installed-github-orgs.sql create mode 100644 backend/src/database/repositories/githubInstallationsRepository.ts create mode 100644 frontend/src/integrations/github/components/github-connect-modal.vue diff --git a/backend/src/api/integration/helpers/getGithubInstallations.ts b/backend/src/api/integration/helpers/getGithubInstallations.ts new file mode 100644 index 0000000000..08448c107d --- /dev/null +++ b/backend/src/api/integration/helpers/getGithubInstallations.ts @@ -0,0 +1,9 @@ +import Permissions from '../../../security/permissions' +import IntegrationService from '../../../services/integrationService' +import PermissionChecker from '../../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + const payload = await new IntegrationService(req).getGithubInstallations() + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/helpers/githubConnectInstallation.ts b/backend/src/api/integration/helpers/githubConnectInstallation.ts new file mode 100644 index 0000000000..d7da9b0f43 --- /dev/null +++ b/backend/src/api/integration/helpers/githubConnectInstallation.ts @@ -0,0 +1,9 @@ +import Permissions from '../../../security/permissions' +import IntegrationService from '../../../services/integrationService' +import PermissionChecker from '../../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) + const payload = await new IntegrationService(req).connectGithubInstallation(req.body.installId) + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/integration/index.ts b/backend/src/api/integration/index.ts index e90faedeb8..35541a2d25 100644 --- a/backend/src/api/integration/index.ts +++ b/backend/src/api/integration/index.ts @@ -189,6 +189,16 @@ export default (app) => { safeWrap(require('./helpers/jiraConnectOrUpdate').default), ) + app.get( + '/tenant/:tenantId/github-installations', + safeWrap(require('./helpers/getGithubInstallations').default), + ) + + app.post( + '/tenant/:tenantId/github-connect-installation', + safeWrap(require('./helpers/githubConnectInstallation').default), + ) + app.get('/gitlab/:tenantId/connect', safeWrap(require('./helpers/gitlabAuthenticate').default)) app.get( diff --git a/backend/src/database/migrations/U1725439490__installed-github-orgs.sql b/backend/src/database/migrations/U1725439490__installed-github-orgs.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/database/migrations/V1725439490__installed-github-orgs.sql b/backend/src/database/migrations/V1725439490__installed-github-orgs.sql new file mode 100644 index 0000000000..c8d963932a --- /dev/null +++ b/backend/src/database/migrations/V1725439490__installed-github-orgs.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS "githubInstallations" ( + "id" UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + "installationId" VARCHAR(255) NOT NULL UNIQUE, + "type" VARCHAR(255) NOT NULL, + "numRepos" INTEGER NOT NULL, + "login" VARCHAR(255) NOT NULL, + "avatarUrl" TEXT, + "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); diff --git a/backend/src/database/repositories/githubInstallationsRepository.ts b/backend/src/database/repositories/githubInstallationsRepository.ts new file mode 100644 index 0000000000..c2f27cebd6 --- /dev/null +++ b/backend/src/database/repositories/githubInstallationsRepository.ts @@ -0,0 +1,20 @@ +import { QueryTypes } from 'sequelize' +import { IRepositoryOptions } from './IRepositoryOptions' +import SequelizeRepository from './sequelizeRepository' + +export default class GithubInstallationsRepository { + static async getInstallations(options: IRepositoryOptions) { + const transaction = SequelizeRepository.getTransaction(options) + const seq = SequelizeRepository.getSequelize(options) + + return seq.query( + ` + select * from "githubInstallations" + `, + { + transaction, + type: QueryTypes.SELECT, + }, + ) + } +} diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 79533752f6..028afd1fbb 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -63,6 +63,7 @@ import SearchSyncService from './searchSyncService' import { IRepositoryOptions } from '@/database/repositories/IRepositoryOptions' import IntegrationProgressRepository from '@/database/repositories/integrationProgressRepository' import { IntegrationProgress } from '@/serverless/integrations/types/regularTypes' +import GithubInstallationsRepository from '@/database/repositories/githubInstallationsRepository' import { fetchGitlabUserProjects, fetchGitlabGroupProjects, @@ -433,6 +434,39 @@ export default class IntegrationService { return integration } + async connectGithubInstallation(installId: string) { + const installToken = await IntegrationService.getInstallToken(installId) + const repos = await getInstalledRepositories(installToken) + const githubOwner = IntegrationService.extractOwner(repos, this.options) + + let orgAvatar + try { + const response = await request('GET /users/{user}', { + user: githubOwner, + }) + orgAvatar = response.data.avatar_url + } catch (err) { + this.options.log.warn(err, 'Error while fetching GitHub user!') + } + + const integration = await this.createOrUpdateGithubIntegration( + { + platform: PlatformType.GITHUB, + token: installToken, + settings: { updateMemberAttributes: true, orgAvatar }, + integrationIdentifier: installId, + status: 'mapping', + }, + repos, + ) + + return integration + } + + async getGithubInstallations() { + return GithubInstallationsRepository.getInstallations(this.options) + } + /** * Creates or updates a GitHub integration, handling large repos data * @param integrationData The integration data to create or update diff --git a/frontend/src/integrations/github/components/github-connect-modal.vue b/frontend/src/integrations/github/components/github-connect-modal.vue new file mode 100644 index 0000000000..f08fd42ad3 --- /dev/null +++ b/frontend/src/integrations/github/components/github-connect-modal.vue @@ -0,0 +1,221 @@ + + + + + diff --git a/frontend/src/integrations/github/components/github-connect.vue b/frontend/src/integrations/github/components/github-connect.vue index 1bd0bbef44..87b0a1cd45 100644 --- a/frontend/src/integrations/github/components/github-connect.vue +++ b/frontend/src/integrations/github/components/github-connect.vue @@ -1,11 +1,15 @@ diff --git a/frontend/src/modules/integration/components/integration-list.vue b/frontend/src/modules/integration/components/integration-list.vue index f16f3174b1..dc4e510e10 100644 --- a/frontend/src/modules/integration/components/integration-list.vue +++ b/frontend/src/modules/integration/components/integration-list.vue @@ -137,6 +137,12 @@ onMounted(async () => { }); showGitlabDialog.value = false; } else { + const state = params.get('state'); + + if (state === 'noconnect') { + return; + } + showGithubDialog.value = true; await store.dispatch('integration/doGithubConnect', { code, diff --git a/frontend/src/modules/integration/integration-service.js b/frontend/src/modules/integration/integration-service.js index 1dc05c4bc5..0a2f183125 100644 --- a/frontend/src/modules/integration/integration-service.js +++ b/frontend/src/modules/integration/integration-service.js @@ -519,6 +519,29 @@ export class IntegrationService { return response.data; } + static async githubConnectInstallation(installId) { + const tenantId = AuthService.getTenantId(); + + const response = await authAxios.post( + `/tenant/${tenantId}/github-connect-installation`, + { + installId, + }, + ); + + return response.data; + } + + static async getGithubInstallations() { + const tenantId = AuthService.getTenantId(); + + const response = await authAxios.get( + `/tenant/${tenantId}/github-installations`, + ); + + return response.data; + } + static async gitlabConnect(code, state) { const tenantId = AuthService.getTenantId(); const response = await authAxios.get(`/gitlab/${tenantId}/callback`, { diff --git a/services/apps/webhook_api/src/routes/github.ts b/services/apps/webhook_api/src/routes/github.ts index cc276efe04..22839d25b6 100644 --- a/services/apps/webhook_api/src/routes/github.ts +++ b/services/apps/webhook_api/src/routes/github.ts @@ -26,8 +26,26 @@ export const installGithubRoutes = async (app: express.Express) => { throw new Error400BadRequest('Missing installation id!') } const identifier = data.installation.id.toString() - // load integration from database to verify that it exists const repo = new WebhooksRepository(req.dbStore, req.log) + + if (event === 'installation' && data.action === 'created') { + await repo.addGithubInstallation( + identifier, + data.installation.account.type, + data.installation.account.login, + data.installation.account.avatar_url, + data.repositories.length, + ) + res.sendStatus(204) + return + } + if (event === 'installation' && data.action === 'deleted') { + await repo.deleteGithubInstallation(identifier) + res.sendStatus(204) + return + } + + // load integration from database to verify that it exists const integration = await repo.findIntegrationByIdentifier(PlatformType.GITHUB, identifier) if (integration) { diff --git a/services/libs/data-access-layer/src/old/apps/webhook_api/webhooks.repo.ts b/services/libs/data-access-layer/src/old/apps/webhook_api/webhooks.repo.ts index e0d8c3c503..ff9f883a68 100644 --- a/services/libs/data-access-layer/src/old/apps/webhook_api/webhooks.repo.ts +++ b/services/libs/data-access-layer/src/old/apps/webhook_api/webhooks.repo.ts @@ -94,6 +94,45 @@ export class WebhooksRepository extends RepositoryBase { return result } + public async addGithubInstallation( + installationId: string, + type: string, + login: string, + avatarUrl: string | null, + numRepos: number, + ): Promise { + await this.db().none( + ` + INSERT INTO "githubInstallations" ("installationId", "type", "login", "avatarUrl", "numRepos") + VALUES ($(installationId), $(type), $(login), $(avatarUrl), $(numRepos)) + ON CONFLICT ("installationId") DO UPDATE + SET "type" = EXCLUDED."type", + "login" = EXCLUDED."login", + "avatarUrl" = EXCLUDED."avatarUrl", + "numRepos" = EXCLUDED."numRepos", + "updatedAt" = CURRENT_TIMESTAMP + `, + { + installationId, + type, + login, + avatarUrl, + numRepos, + }, + ) + } + + public async deleteGithubInstallation(installationId: string): Promise { + await this.db().none( + ` + DELETE FROM "githubInstallations" WHERE "installationId" = $(installationId) + `, + { + installationId, + }, + ) + } + public async findIntegrationById(id: string): Promise { const result = await this.db().oneOrNone( ` From 85b8782e6e7c52011710b7aa0ddae03503fb2855 Mon Sep 17 00:00:00 2001 From: Yeganathan S Date: Thu, 12 Sep 2024 21:08:49 +0530 Subject: [PATCH 2/3] remove `joinedAt` in OpenSearch member cleanup (#2609) --- .../src/service/member.sync.service.ts | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/services/libs/opensearch/src/service/member.sync.service.ts b/services/libs/opensearch/src/service/member.sync.service.ts index ee8b4bd612..03b7b3715d 100644 --- a/services/libs/opensearch/src/service/member.sync.service.ts +++ b/services/libs/opensearch/src/service/member.sync.service.ts @@ -146,10 +146,10 @@ export class MemberSyncService { }, } - const sort = [{ date_joinedAt: 'asc' }] - const include = ['date_joinedAt', 'uuid_memberId'] + const sort = [{ _id: 'asc' }] + const include = ['uuid_memberId'] const pageSize = 500 - let lastJoinedAt: string + let lastId: string let results = (await this.openSearchService.search( OpenSearchIndex.MEMBERS, @@ -159,7 +159,7 @@ export class MemberSyncService { sort, undefined, include, - )) as ISearchHit<{ date_joinedAt: string; uuid_memberId: string }>[] + )) as ISearchHit<{ uuid_memberId: string }>[] let processed = 0 const idsToRemove: string[] = [] @@ -187,17 +187,16 @@ export class MemberSyncService { processed += results.length this.log.warn({ tenantId }, `Processed ${processed} members while cleaning up tenant!`) - // use last joinedAt to get the next page - lastJoinedAt = results[results.length - 1]._source.date_joinedAt + lastId = results[results.length - 1]._id results = (await this.openSearchService.search( OpenSearchIndex.MEMBERS, query, undefined, pageSize, sort, - lastJoinedAt, + lastId, include, - )) as ISearchHit<{ date_joinedAt: string; uuid_memberId: string }>[] + )) as ISearchHit<{ uuid_memberId: string }>[] } // Remove any remaining IDs that were not processed @@ -222,10 +221,9 @@ export class MemberSyncService { }, } - const sort = [{ date_joinedAt: 'asc' }] - const include = ['date_joinedAt'] + const sort = [{ _id: 'asc' }] const pageSize = 10 - let lastJoinedAt: string + let lastId: string let results = (await this.openSearchService.search( OpenSearchIndex.MEMBERS, @@ -234,24 +232,23 @@ export class MemberSyncService { pageSize, sort, undefined, - include, - )) as ISearchHit<{ date_joinedAt: string }>[] + undefined, + )) as ISearchHit[] while (results.length > 0) { const ids = results.map((r) => r._id) await this.openSearchService.bulkRemoveFromIndex(ids, OpenSearchIndex.MEMBERS) - // use last joinedAt to get the next page - lastJoinedAt = results[results.length - 1]._source.date_joinedAt + lastId = results[results.length - 1]._id results = (await this.openSearchService.search( OpenSearchIndex.MEMBERS, query, undefined, pageSize, sort, - lastJoinedAt, - include, - )) as ISearchHit<{ date_joinedAt: string }>[] + lastId, + undefined, + )) as ISearchHit[] } } From 571ed815d3341ba999bdfbeea110563e3b53b143 Mon Sep 17 00:00:00 2001 From: garrrikkotua Date: Thu, 12 Sep 2024 19:53:23 +0400 Subject: [PATCH 3/3] hot fix github invitations --- frontend/src/modules/integration/integration-service.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/modules/integration/integration-service.js b/frontend/src/modules/integration/integration-service.js index 0a2f183125..0ffc256934 100644 --- a/frontend/src/modules/integration/integration-service.js +++ b/frontend/src/modules/integration/integration-service.js @@ -526,6 +526,7 @@ export class IntegrationService { `/tenant/${tenantId}/github-connect-installation`, { installId, + ...getSegments(), }, );